From edd862c21a572b8c58d497b0e96bb44e07649aba Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 11 Nov 2024 17:22:14 -0800 Subject: [PATCH 001/506] init module --- lib/spack/spack/install_trees.py | 13 +++++++++++++ lib/spack/spack/main.py | 5 +++++ 2 files changed, 18 insertions(+) create mode 100644 lib/spack/spack/install_trees.py diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_trees.py new file mode 100644 index 00000000000000..b6b01a04cba447 --- /dev/null +++ b/lib/spack/spack/install_trees.py @@ -0,0 +1,13 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +""" +For tracking install trees. +""" + +# Default install tree is ~// +# That has a config scope, add it in config.py + +# Also look in /etc/spack/ for install trees diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index fc5423b5a26a98..52a54a8fabb9ec 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -501,6 +501,11 @@ def make_argument_parser(**kwargs): parser.add_argument( "--print-shell-vars", action="store", help="print info needed by setup-env.*sh" ) + parser.add_argument( + '--install-root', dest='install_root', + action='store', default=None, + help='specify non-default install tree' + ) return parser From 9ee021d7dccab3e4187f5e31beba30401b52470a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 09:03:13 -0800 Subject: [PATCH 002/506] moved a few things out of install_trees into paths (I think logic can be simplified by 'intercepting' at path level --- lib/spack/spack/install_trees.py | 29 +++++++++++++++++++--- lib/spack/spack/paths.py | 42 +++++++++++++++++++++++++++++--- 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_trees.py index b6b01a04cba447..72466c10c40be5 100644 --- a/lib/spack/spack/install_trees.py +++ b/lib/spack/spack/install_trees.py @@ -7,7 +7,30 @@ For tracking install trees. """ -# Default install tree is ~// -# That has a config scope, add it in config.py +import spack.paths as paths +import spack.util.hash as hash -# Also look in /etc/spack/ for install trees +import pathlib + + +def use_new_root(): + internal_opt_dir = pathlib.Path(paths.internal_opt_path) + # If the spack-internal opt/ dir is nonempty, we are using the old + # root + return not (internal_opt_dir.is_dir() and any(internal_opt_dir.iterdir())) + + + +def user_root(): + """Default install tree and config scope. + + Applies when $spack/opt is not an install tree. + + ~// + """ + spack_prefix = paths.prefix + return pathlib.Path(paths.user_config_path, hash.b32_hash(spack_prefix)[:7]) + + +def shared_trees(): + root_dir = pathlib.Path(paths.system_config_path) / "install-trees" diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 84583cd552f531..035b575bbc293f 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -11,8 +11,10 @@ """ import os from pathlib import PurePath +import pathlib import llnl.util.filesystem +import spack.util.hash as hash #: This file lives in $prefix/lib/spack/spack/__file__ prefix = str(PurePath(llnl.util.filesystem.ancestor(__file__, 4))) @@ -42,10 +44,23 @@ operating_system_path = os.path.join(module_path, "operating_systems") test_path = os.path.join(module_path, "test") hooks_path = os.path.join(module_path, "hooks") -opt_path = os.path.join(prefix, "opt") share_path = os.path.join(prefix, "share", "spack") etc_path = os.path.join(prefix, "etc", "spack") +def user_root(): + """Default install tree and config scope. + + Applies when $spack/opt is not an install tree. + + ~// + """ + spack_prefix = prefix + return pathlib.Path(user_config_path, hash.b32_hash(spack_prefix)[:7]) + + +def shared_trees(): + root_dir = pathlib.Path(system_config_path) / "install-trees" + # # Things in $spack/etc/spack # @@ -54,10 +69,31 @@ # # Things in $spack/var/spack # -var_path = os.path.join(prefix, "var", "spack") +read_var_path = os.path.join(prefix, "var", "spack") + +def dir_is_occupied(x, except_for=None): + x = pathlib.Path(x) + except_for = except_for or set() + return not (x.is_dir() and bool(set(x.iterdir()) - except_for)) + +internal_opt_path = os.path.join(prefix, "opt") + +if dir_is_occupied(internal_opt_path): + opt_path = internal_opt_path +else: + opt_path = os.path.join(str(user_root()), "opt") + +if dir_is_occupied(read_var_path, except_for={"repos"}): + var_path = read_var_path +else: + var_path = os.path.join(str(user_root()), "var", "spack") + +# TODO: also check share_path/{lmod, tcl} +# TODO: can use new-style locations if user explicitly specifies --install-root +# TODO: can shutil.mv everything in {opt, var} except installs into new user root # read-only things in $spack/var/spack -repos_path = os.path.join(var_path, "repos") +repos_path = os.path.join(read_var_path, "repos") packages_path = os.path.join(repos_path, "builtin") mock_packages_path = os.path.join(repos_path, "builtin.mock") From d93974697f2b735319e10a590bd4207c4c07fd1e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 09:42:49 -0800 Subject: [PATCH 003/506] relocate paths code --- lib/spack/spack/paths.py | 55 ++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 035b575bbc293f..5a5f613a56a9dd 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -19,6 +19,33 @@ #: This file lives in $prefix/lib/spack/spack/__file__ prefix = str(PurePath(llnl.util.filesystem.ancestor(__file__, 4))) +# Below paths pull configuration from the host environment. +# +# There are three environment variables you can use to isolate spack from +# the host environment: +# - `SPACK_USER_CONFIG_PATH`: override `~/.spack` location (for config and caches) +# - `SPACK_SYSTEM_CONFIG_PATH`: override `/etc/spack` configuration scope. +# - `SPACK_DISABLE_LOCAL_CONFIG`: disable both of these locations. + + +# User configuration and caches in $HOME/.spack +def _get_user_config_path(): + return os.path.expanduser(os.getenv("SPACK_USER_CONFIG_PATH") or "~%s.spack" % os.sep) + + +# Configuration in /etc/spack on the system +def _get_system_config_path(): + return os.path.expanduser( + os.getenv("SPACK_SYSTEM_CONFIG_PATH") or os.sep + os.path.join("etc", "spack") + ) + + +#: User configuration location +user_config_path = _get_user_config_path() + +#: System configuration location +system_config_path = _get_system_config_path() + #: synonym for prefix spack_root = prefix @@ -145,34 +172,6 @@ def _get_user_cache_path(): #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) default_misc_cache_path = os.path.join(user_cache_path, "cache") - -# Below paths pull configuration from the host environment. -# -# There are three environment variables you can use to isolate spack from -# the host environment: -# - `SPACK_USER_CONFIG_PATH`: override `~/.spack` location (for config and caches) -# - `SPACK_SYSTEM_CONFIG_PATH`: override `/etc/spack` configuration scope. -# - `SPACK_DISABLE_LOCAL_CONFIG`: disable both of these locations. - - -# User configuration and caches in $HOME/.spack -def _get_user_config_path(): - return os.path.expanduser(os.getenv("SPACK_USER_CONFIG_PATH") or "~%s.spack" % os.sep) - - -# Configuration in /etc/spack on the system -def _get_system_config_path(): - return os.path.expanduser( - os.getenv("SPACK_SYSTEM_CONFIG_PATH") or os.sep + os.path.join("etc", "spack") - ) - - -#: User configuration location -user_config_path = _get_user_config_path() - -#: System configuration location -system_config_path = _get_system_config_path() - #: Recorded directory where spack command was originally invoked spack_working_dir = None From 65b26ca40c6b66e86e35e403941d88c3b3e11b65 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 09:56:10 -0800 Subject: [PATCH 004/506] installs should still default to inside spack prefix --- lib/spack/spack/install_trees.py | 19 ------------------- lib/spack/spack/modules/common.py | 2 +- lib/spack/spack/paths.py | 4 ++++ lib/spack/spack/store.py | 4 ++-- 4 files changed, 7 insertions(+), 22 deletions(-) diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_trees.py index 72466c10c40be5..0742a7bde5a62a 100644 --- a/lib/spack/spack/install_trees.py +++ b/lib/spack/spack/install_trees.py @@ -13,24 +13,5 @@ import pathlib -def use_new_root(): - internal_opt_dir = pathlib.Path(paths.internal_opt_path) - # If the spack-internal opt/ dir is nonempty, we are using the old - # root - return not (internal_opt_dir.is_dir() and any(internal_opt_dir.iterdir())) - - - -def user_root(): - """Default install tree and config scope. - - Applies when $spack/opt is not an install tree. - - ~// - """ - spack_prefix = paths.prefix - return pathlib.Path(paths.user_config_path, hash.b32_hash(spack_prefix)[:7]) - - def shared_trees(): root_dir = pathlib.Path(paths.system_config_path) / "install-trees" diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py index c770564edddb0a..6122d5cab41439 100644 --- a/lib/spack/spack/modules/common.py +++ b/lib/spack/spack/modules/common.py @@ -218,7 +218,7 @@ def root_path(name, module_set_name): # Merge config values into the defaults so we prefer configured values roots = spack.config.merge_yaml(defaults, roots) - path = roots.get(name, os.path.join(spack.paths.share_path, name)) + path = roots.get(name, os.path.join(spack.paths.modules_base, name)) return spack.util.path.canonicalize_path(path) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 5a5f613a56a9dd..4f7cb4c5a83f84 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -107,14 +107,18 @@ def dir_is_occupied(x, except_for=None): if dir_is_occupied(internal_opt_path): opt_path = internal_opt_path + modules_base = share_path else: opt_path = os.path.join(str(user_root()), "opt") + modules_base = os.path.join(str(user_root()), "modules") if dir_is_occupied(read_var_path, except_for={"repos"}): var_path = read_var_path else: var_path = os.path.join(str(user_root()), "var", "spack") +installs_base = os.path.join(internal_opt_path, "spack") + # TODO: also check share_path/{lmod, tcl} # TODO: can use new-style locations if user explicitly specifies --install-root # TODO: can shutil.mv everything in {opt, var} except installs into new user root diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index abd7d9250007c0..8c874934ab8d71 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -35,8 +35,8 @@ import spack.spec import spack.util.path -#: default installation root, relative to the Spack install path -DEFAULT_INSTALL_TREE_ROOT = os.path.join(spack.paths.opt_path, "spack") +#: default installation root +DEFAULT_INSTALL_TREE_ROOT = spack.paths.installs_base def parse_install_tree(config_dict): From 428a6eff2568c411328b690dd09a11feeac10a7b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 10:25:16 -0800 Subject: [PATCH 005/506] consolidate docs + move method --- lib/spack/spack/paths.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 4f7cb4c5a83f84..c2089dc6389ea6 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -19,27 +19,27 @@ #: This file lives in $prefix/lib/spack/spack/__file__ prefix = str(PurePath(llnl.util.filesystem.ancestor(__file__, 4))) -# Below paths pull configuration from the host environment. -# -# There are three environment variables you can use to isolate spack from -# the host environment: -# - `SPACK_USER_CONFIG_PATH`: override `~/.spack` location (for config and caches) -# - `SPACK_SYSTEM_CONFIG_PATH`: override `/etc/spack` configuration scope. -# - `SPACK_DISABLE_LOCAL_CONFIG`: disable both of these locations. - # User configuration and caches in $HOME/.spack +# Override w/ `SPACK_USER_CONFIG_PATH` def _get_user_config_path(): return os.path.expanduser(os.getenv("SPACK_USER_CONFIG_PATH") or "~%s.spack" % os.sep) # Configuration in /etc/spack on the system +# Override w/ `SPACK_SYSTEM_CONFIG_PATH` def _get_system_config_path(): return os.path.expanduser( os.getenv("SPACK_SYSTEM_CONFIG_PATH") or os.sep + os.path.join("etc", "spack") ) +def dir_is_occupied(x, except_for=None): + x = pathlib.Path(x) + except_for = except_for or set() + return not (x.is_dir() and bool(set(x.iterdir()) - except_for)) + + #: User configuration location user_config_path = _get_user_config_path() @@ -98,11 +98,6 @@ def shared_trees(): # read_var_path = os.path.join(prefix, "var", "spack") -def dir_is_occupied(x, except_for=None): - x = pathlib.Path(x) - except_for = except_for or set() - return not (x.is_dir() and bool(set(x.iterdir()) - except_for)) - internal_opt_path = os.path.join(prefix, "opt") if dir_is_occupied(internal_opt_path): From 731818ca84df211ef0a4e60da124713425a9cf8b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 10:31:27 -0800 Subject: [PATCH 006/506] update comment --- lib/spack/spack/paths.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index c2089dc6389ea6..03864445910794 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -124,10 +124,8 @@ def shared_trees(): mock_packages_path = os.path.join(repos_path, "builtin.mock") # -# Writable things in $spack/var/spack -# TODO: Deprecate these, as we want a read-only spack prefix by default. -# TODO: These should probably move to user cache, or some other location. -# +# Writable things in $var_path +# In older spack, this is $spack/var/spack # fetch cache for downloaded files default_fetch_cache_path = os.path.join(var_path, "cache") From 2577e1a9328c1d433ebeea54de00c1a9f3987d25 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 11:03:16 -0800 Subject: [PATCH 007/506] selection of install tree, universal install tree config --- lib/spack/spack/config.py | 3 +++ lib/spack/spack/install_trees.py | 33 ++++++++++++++++++++++++++++---- lib/spack/spack/main.py | 4 ++++ lib/spack/spack/paths.py | 10 +++++----- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index afd8f30baccbb9..04874b33396aa0 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -40,6 +40,7 @@ from llnl.util import filesystem, lang, tty import spack.error +import spack.install_trees import spack.paths import spack.platforms import spack.schema @@ -848,6 +849,8 @@ def create() -> Configuration: # Python package's can register configuration scopes via entry_points configuration_paths.extend(config_paths_from_entry_points()) + configuration_paths.append(("install", spack.install_trees.install_tree_config())) + # User configuration can override both spack defaults and site config # This is disabled if user asks for no local configuration. if not disable_local_config: diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_trees.py index 0742a7bde5a62a..78741a8e0b06f7 100644 --- a/lib/spack/spack/install_trees.py +++ b/lib/spack/spack/install_trees.py @@ -8,10 +8,35 @@ """ import spack.paths as paths -import spack.util.hash as hash -import pathlib +import os +install_root = None -def shared_trees(): - root_dir = pathlib.Path(paths.system_config_path) / "install-trees" +_old_installs_base = os.path.join(paths.internal_opt_path, "spack") + +default_install_base = os.path.join(paths.per_spack_user_root, "installs") + +def install_tree(): + """ + Selecting install-tree + + - if user specifies --install-root, we use that + - if $user_root has one, we use that + - if $old_install_path is occupied, we use that + - otherwise, we use $user_root + """ + if install_root: + return install_root + elif paths.dir_is_occupied(default_install_base): + return default_install_base + elif paths.dir_is_occupied(_old_installs_base): + return _old_installs_base + else: + return default_install_base + + +def install_tree_config(): + root = install_tree() + cfgs = os.path.join(root, "configs") + return cfgs \ No newline at end of file diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 52a54a8fabb9ec..61cd6d8425aef5 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -39,6 +39,7 @@ import spack.config import spack.environment as ev import spack.error +import spack.install_trees import spack.modules import spack.paths import spack.platforms @@ -573,6 +574,9 @@ def setup_main_options(args): if args.color is not None: color.set_color_when(args.color) + if args.install_root: + spack.install_trees.install_root = args.install_root + def allows_unknown_args(command): """Implements really simple argument injection for unknown arguments. diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 03864445910794..c00551fc8aa1e5 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -88,6 +88,8 @@ def user_root(): def shared_trees(): root_dir = pathlib.Path(system_config_path) / "install-trees" +per_spack_user_root = str(user_root()) + # # Things in $spack/etc/spack # @@ -104,15 +106,13 @@ def shared_trees(): opt_path = internal_opt_path modules_base = share_path else: - opt_path = os.path.join(str(user_root()), "opt") - modules_base = os.path.join(str(user_root()), "modules") + opt_path = os.path.join(per_spack_user_root, "opt") + modules_base = os.path.join(per_spack_user_root, "modules") if dir_is_occupied(read_var_path, except_for={"repos"}): var_path = read_var_path else: - var_path = os.path.join(str(user_root()), "var", "spack") - -installs_base = os.path.join(internal_opt_path, "spack") + var_path = os.path.join(per_spack_user_root, "var", "spack") # TODO: also check share_path/{lmod, tcl} # TODO: can use new-style locations if user explicitly specifies --install-root From 0a18c640b78b22067722336ca70116af5fe5cc95 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 11:05:28 -0800 Subject: [PATCH 008/506] alias for install-tree --- lib/spack/spack/install_trees.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_trees.py index 78741a8e0b06f7..ae29dc02ae54cf 100644 --- a/lib/spack/spack/install_trees.py +++ b/lib/spack/spack/install_trees.py @@ -17,6 +17,11 @@ default_install_base = os.path.join(paths.per_spack_user_root, "installs") +alias = { + ".root": _old_installs_base, + ".user": default_install_base, +} + def install_tree(): """ Selecting install-tree @@ -27,7 +32,9 @@ def install_tree(): - otherwise, we use $user_root """ if install_root: - return install_root + # User can e.g. say --install-root=.root to refer to + # the installation tree inside of the Spack prefix + return alias.get(install_root, install_root) elif paths.dir_is_occupied(default_install_base): return default_install_base elif paths.dir_is_occupied(_old_installs_base): From 6230f256c3d937cabcdee8df04c7c1d7ca0c36ff Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 11:08:00 -0800 Subject: [PATCH 009/506] make use of install tree selection in store --- lib/spack/spack/store.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 8c874934ab8d71..1c9ecf58922c8b 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -31,12 +31,13 @@ import spack.database import spack.directory_layout import spack.error +import spack.install_trees import spack.paths import spack.spec import spack.util.path #: default installation root -DEFAULT_INSTALL_TREE_ROOT = spack.paths.installs_base +DEFAULT_INSTALL_TREE_ROOT = spack.install_trees.install_tree() def parse_install_tree(config_dict): From 517384a889d1e7f6de7896b4f42228db10b16dc3 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 11:15:44 -0800 Subject: [PATCH 010/506] make install cfg scope preferred for editing --- lib/spack/spack/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 04874b33396aa0..29f131904d2098 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -455,6 +455,10 @@ def writable_scopes(self) -> Generator[ConfigScope, None, None]: def highest_precedence_scope(self) -> ConfigScope: """Writable scope with highest precedence.""" + if "install" in self.scopes: + # Preference is to write into config scope associated with + # the installation tree, if it exists for this Configuration + return self.scopes["install"] return next(s for s in reversed(self.scopes.values()) if s.writable) # type: ignore def highest_precedence_non_platform_scope(self) -> ConfigScope: From 77093d5db02ae0500f72cc13bc56942356e46d69 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 11:20:01 -0800 Subject: [PATCH 011/506] style edits --- lib/spack/spack/install_trees.py | 17 ++++++++++------- lib/spack/spack/main.py | 8 +++++--- lib/spack/spack/paths.py | 7 +++---- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_trees.py index ae29dc02ae54cf..21c9710bb2d20d 100644 --- a/lib/spack/spack/install_trees.py +++ b/lib/spack/spack/install_trees.py @@ -7,9 +7,10 @@ For tracking install trees. """ -import spack.paths as paths - import os +import pathlib + +import spack.paths as paths install_root = None @@ -17,10 +18,8 @@ default_install_base = os.path.join(paths.per_spack_user_root, "installs") -alias = { - ".root": _old_installs_base, - ".user": default_install_base, -} +alias = {".root": _old_installs_base, ".user": default_install_base} + def install_tree(): """ @@ -46,4 +45,8 @@ def install_tree(): def install_tree_config(): root = install_tree() cfgs = os.path.join(root, "configs") - return cfgs \ No newline at end of file + return cfgs + + +def shared_trees(): + return pathlib.Path(paths.system_config_path) / "install-trees" diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 61cd6d8425aef5..9e0dec52095fd8 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -503,9 +503,11 @@ def make_argument_parser(**kwargs): "--print-shell-vars", action="store", help="print info needed by setup-env.*sh" ) parser.add_argument( - '--install-root', dest='install_root', - action='store', default=None, - help='specify non-default install tree' + "--install-root", + dest="install_root", + action="store", + default=None, + help="specify non-default install tree", ) return parser diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index c00551fc8aa1e5..2a90ab50479b2f 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -10,10 +10,11 @@ dependencies. """ import os -from pathlib import PurePath import pathlib +from pathlib import PurePath import llnl.util.filesystem + import spack.util.hash as hash #: This file lives in $prefix/lib/spack/spack/__file__ @@ -74,6 +75,7 @@ def dir_is_occupied(x, except_for=None): share_path = os.path.join(prefix, "share", "spack") etc_path = os.path.join(prefix, "etc", "spack") + def user_root(): """Default install tree and config scope. @@ -85,9 +87,6 @@ def user_root(): return pathlib.Path(user_config_path, hash.b32_hash(spack_prefix)[:7]) -def shared_trees(): - root_dir = pathlib.Path(system_config_path) / "install-trees" - per_spack_user_root = str(user_root()) # From a77cc9ed2d8a5c4d1e0d3769e1f10e3ae5b03632 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 11:40:59 -0800 Subject: [PATCH 012/506] add option for hook on modifications to spack install prefix --- lib/spack/spack/install_trees.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_trees.py index 21c9710bb2d20d..cec6bda0df0cae 100644 --- a/lib/spack/spack/install_trees.py +++ b/lib/spack/spack/install_trees.py @@ -48,5 +48,25 @@ def install_tree_config(): return cfgs +def _guard_writes(event, args): + if event == "open": + path, mode = args[:2] + abs_path = os.path.abspath(path) + intent_to_modify = set(mode) & set("wax") + if abs_path.startswith(paths.prefix) and intent_to_modify: + pass + elif event == "shutil.copyfile": + src, dst = args[:2] + abs_dst = os.path.abspath(dst) + if abs_dst.startswith(paths.prefix): + pass + + +def guard_writes_into_spack(): + import sys + + sys.addaudithook(_guard_writes) + + def shared_trees(): return pathlib.Path(paths.system_config_path) / "install-trees" From 2603b7655371059b976d9952c875132ed689d545 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 15:17:26 -0800 Subject: [PATCH 013/506] frame inspection --- lib/spack/spack/install_trees.py | 45 +++++++++++++++++++++++++++++--- lib/spack/spack/main.py | 2 ++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_trees.py index cec6bda0df0cae..bb4af3148f278b 100644 --- a/lib/spack/spack/install_trees.py +++ b/lib/spack/spack/install_trees.py @@ -7,8 +7,10 @@ For tracking install trees. """ +import inspect import os import pathlib +import warnings import spack.paths as paths @@ -48,18 +50,55 @@ def install_tree_config(): return cfgs +def _most_recent_internal_call(): + """If called within an audit for a Python library function, finds + the most recent spot within Spack's source code that generated + the call. + """ + + stack = inspect.stack() + this_file = str(pathlib.Path(__file__).resolve()) + for frame in stack: + frame_loc = pathlib.Path(frame.filename).resolve() + if str(frame_loc) != this_file: + return frame_loc, frame.lineno + + return None, None + + +_recorded_accesses = set() + + +def _attempted_modify_internal(msg): + loc, line = _most_recent_internal_call() + if loc: + if (loc, line) not in _recorded_accesses: + _recorded_accesses.add((loc, line)) + msg += f" at {loc}:{line}" + warnings.warn(msg) + else: + msg += " (no location)" + warnings.warn(msg) + + def _guard_writes(event, args): if event == "open": path, mode = args[:2] + if not mode: + # Some internal Python libs can call open(..., mode=None) + return + if not isinstance(path, str): + # Skip instances of open() that function like fdopen + return abs_path = os.path.abspath(path) - intent_to_modify = set(mode) & set("wax") + intent_to_modify = bool(set(mode) & set("wax")) if abs_path.startswith(paths.prefix) and intent_to_modify: - pass + _attempted_modify_internal(f"Open {path} in mode {mode}") elif event == "shutil.copyfile": src, dst = args[:2] abs_dst = os.path.abspath(dst) if abs_dst.startswith(paths.prefix): - pass + _attempted_modify_internal(f"Copy targets {abs_dst}") def guard_writes_into_spack(): diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 9e0dec52095fd8..2cc6de3845b6f5 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -965,6 +965,8 @@ def _main(argv=None): cmd_name = args.command[0] cmd_name, args.command = resolve_alias(cmd_name, args.command) + spack.install_trees.guard_writes_into_spack() + # set up a bootstrap context, if asked. # bootstrap context needs to include parsing the command, b/c things # like `ConstraintAction` and `ConfigSetAction` happen at parse time. From 1d2aa0eb09e34b24b283d6424086b204a995367c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 15:21:44 -0800 Subject: [PATCH 014/506] style edits --- lib/spack/spack/install_trees.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_trees.py index bb4af3148f278b..693dfba8402cb2 100644 --- a/lib/spack/spack/install_trees.py +++ b/lib/spack/spack/install_trees.py @@ -52,8 +52,8 @@ def install_tree_config(): def _most_recent_internal_call(): """If called within an audit for a Python library function, finds - the most recent spot within Spack's source code that generated - the call. + the most recent spot within Spack's source code that generated + the call. """ stack = inspect.stack() From d446f0ac0a801ca86888c212cbd67d4d4f416f0e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 16:28:49 -0800 Subject: [PATCH 015/506] move misc cache to per-spack user root --- lib/spack/spack/paths.py | 5 +++-- var/spack/{ => read-gpg}/gpg.mock/README.md | 0 var/spack/{ => read-gpg}/gpg.mock/data/content.txt | 0 var/spack/{ => read-gpg}/gpg.mock/data/content.txt.asc | 0 var/spack/{ => read-gpg}/gpg.mock/keys/external.key | 0 var/spack/{ => read-gpg}/gpg.mock/keys/package-signing-key | 0 var/spack/{ => read-gpg}/gpg/README.md | 0 7 files changed, 3 insertions(+), 2 deletions(-) rename var/spack/{ => read-gpg}/gpg.mock/README.md (100%) rename var/spack/{ => read-gpg}/gpg.mock/data/content.txt (100%) rename var/spack/{ => read-gpg}/gpg.mock/data/content.txt.asc (100%) rename var/spack/{ => read-gpg}/gpg.mock/keys/external.key (100%) rename var/spack/{ => read-gpg}/gpg.mock/keys/package-signing-key (100%) rename var/spack/{ => read-gpg}/gpg/README.md (100%) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 2a90ab50479b2f..7e0e6024c50d62 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -108,7 +108,7 @@ def user_root(): opt_path = os.path.join(per_spack_user_root, "opt") modules_base = os.path.join(per_spack_user_root, "modules") -if dir_is_occupied(read_var_path, except_for={"repos"}): +if dir_is_occupied(read_var_path, except_for={"repos", "read-gpg"}): var_path = read_var_path else: var_path = os.path.join(per_spack_user_root, "var", "spack") @@ -166,7 +166,8 @@ def _get_user_cache_path(): default_user_bootstrap_path = os.path.join(user_cache_path, "bootstrap") #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) -default_misc_cache_path = os.path.join(user_cache_path, "cache") +#: placed in per-spack-instance user root +default_misc_cache_path = os.path.join(per_spack_user_root, "cache") #: Recorded directory where spack command was originally invoked spack_working_dir = None diff --git a/var/spack/gpg.mock/README.md b/var/spack/read-gpg/gpg.mock/README.md similarity index 100% rename from var/spack/gpg.mock/README.md rename to var/spack/read-gpg/gpg.mock/README.md diff --git a/var/spack/gpg.mock/data/content.txt b/var/spack/read-gpg/gpg.mock/data/content.txt similarity index 100% rename from var/spack/gpg.mock/data/content.txt rename to var/spack/read-gpg/gpg.mock/data/content.txt diff --git a/var/spack/gpg.mock/data/content.txt.asc b/var/spack/read-gpg/gpg.mock/data/content.txt.asc similarity index 100% rename from var/spack/gpg.mock/data/content.txt.asc rename to var/spack/read-gpg/gpg.mock/data/content.txt.asc diff --git a/var/spack/gpg.mock/keys/external.key b/var/spack/read-gpg/gpg.mock/keys/external.key similarity index 100% rename from var/spack/gpg.mock/keys/external.key rename to var/spack/read-gpg/gpg.mock/keys/external.key diff --git a/var/spack/gpg.mock/keys/package-signing-key b/var/spack/read-gpg/gpg.mock/keys/package-signing-key similarity index 100% rename from var/spack/gpg.mock/keys/package-signing-key rename to var/spack/read-gpg/gpg.mock/keys/package-signing-key diff --git a/var/spack/gpg/README.md b/var/spack/read-gpg/gpg/README.md similarity index 100% rename from var/spack/gpg/README.md rename to var/spack/read-gpg/gpg/README.md From f2e581a052a79f9430c50d38a6592011dc462c7b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 17:14:05 -0800 Subject: [PATCH 016/506] several bugfixes --- etc/spack/defaults/config.yaml | 1 - lib/spack/spack/install_trees.py | 11 ++++++++--- lib/spack/spack/paths.py | 2 +- lib/spack/spack/store.py | 5 +---- lib/spack/spack/util/path.py | 2 ++ 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index b9c4aee64eee3c..85c106a5ad43f4 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -17,7 +17,6 @@ config: # This is the path to the root of the Spack install tree. # You can use $spack here to refer to the root of the spack instance. install_tree: - root: $spack/opt/spack projections: all: "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}" # install_tree can include an optional padded length (int or boolean) diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_trees.py index 693dfba8402cb2..992e55c37812b0 100644 --- a/lib/spack/spack/install_trees.py +++ b/lib/spack/spack/install_trees.py @@ -93,12 +93,17 @@ def _guard_writes(event, args): abs_path = os.path.abspath(path) intent_to_modify = bool(set(mode) & set("wax")) if abs_path.startswith(paths.prefix) and intent_to_modify: - _attempted_modify_internal(f"Open {path} in mode {mode}") + _attempted_modify_internal(f"Open {path} in mode [{mode}]") elif event == "shutil.copyfile": - src, dst = args[:2] + _, dst = args[:2] abs_dst = os.path.abspath(dst) if abs_dst.startswith(paths.prefix): - _attempted_modify_internal(f"Copy targets {abs_dst}") + _attempted_modify_internal(f"copy dst {abs_dst}") + elif event == "os.mkdir": + path = args[0] + abs_path = os.path.abspath(path) + if abs_path.startswith(paths.prefix): + _attempted_modify_internal(f"mkdir {abs_path}") def guard_writes_into_spack(): diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 7e0e6024c50d62..e5caa6c949d67d 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -38,7 +38,7 @@ def _get_system_config_path(): def dir_is_occupied(x, except_for=None): x = pathlib.Path(x) except_for = except_for or set() - return not (x.is_dir() and bool(set(x.iterdir()) - except_for)) + return x.is_dir() and bool(set(x.iterdir()) - except_for) #: User configuration location diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 1c9ecf58922c8b..bfc70487e284dc 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -36,9 +36,6 @@ import spack.spec import spack.util.path -#: default installation root -DEFAULT_INSTALL_TREE_ROOT = spack.install_trees.install_tree() - def parse_install_tree(config_dict): """Parse config settings and return values relevant to the store object. @@ -83,7 +80,7 @@ def parse_install_tree(config_dict): projections = {"all": all_projection} else: - unpadded_root = install_tree.get("root", DEFAULT_INSTALL_TREE_ROOT) + unpadded_root = install_tree.get("root", spack.install_trees.install_tree()) unpadded_root = spack.util.path.canonicalize_path(unpadded_root) padded_length = install_tree.get("padded_length", False) diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 5e7fa3a797b995..d5548437d27748 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -65,6 +65,7 @@ def replacements(): "user": lambda: get_user(), "tempdir": lambda: tempfile.gettempdir(), "user_cache_path": lambda: spack.paths.user_cache_path, + "per_spack_user": lambda: spack.paths.per_spack_user_root, "architecture": lambda: arch, "arch": lambda: arch, "platform": lambda: arch.platform, @@ -160,6 +161,7 @@ def substitute_config_variables(path): - $tempdir Default temporary directory returned by tempfile.gettempdir() - $user The current user's username - $user_cache_path The user cache directory (~/.spack, unless overridden) + - $per_spack_user Each Spack install has its own dir in ~ - $architecture The spack architecture triple for the current system - $arch The spack architecture triple for the current system - $platform The spack platform for the current system From 53afc712a6599efff90c6e59bea41ee2bf3f3cc4 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 17:45:28 -0800 Subject: [PATCH 017/506] check for r+ --- lib/spack/spack/install_trees.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_trees.py index 992e55c37812b0..6d919a15803bd9 100644 --- a/lib/spack/spack/install_trees.py +++ b/lib/spack/spack/install_trees.py @@ -91,7 +91,7 @@ def _guard_writes(event, args): # Skip instances of open() that function like fdopen return abs_path = os.path.abspath(path) - intent_to_modify = bool(set(mode) & set("wax")) + intent_to_modify = bool((set(mode) & set("wax")) or "r+" in mode) if abs_path.startswith(paths.prefix) and intent_to_modify: _attempted_modify_internal(f"Open {path} in mode [{mode}]") elif event == "shutil.copyfile": From 6cb1868017ff0e6cd165b028c4f073561bea7ec6 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 12 Nov 2024 17:48:06 -0800 Subject: [PATCH 018/506] add todo comment --- lib/spack/spack/install_trees.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_trees.py index 6d919a15803bd9..4d8aca6b68ef17 100644 --- a/lib/spack/spack/install_trees.py +++ b/lib/spack/spack/install_trees.py @@ -82,6 +82,12 @@ def _attempted_modify_internal(msg): def _guard_writes(event, args): + # Note: this doesn't catch files opened in "r" mode and then + # later upgraded to "w" mode (e.g. our locks). I think to track + # that properly we would need to (a) patch builtins.open to + # map all paths to FDs as they are opened (and delete on close) + # and (b) audit fcntl.fcntl events, using reverse mapping on + # FD to check associated path if event == "open": path, mode = args[:2] if not mode: From 877797f5ca271098b10759a216d75b859ac0d827 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 13 Nov 2024 11:32:20 -0800 Subject: [PATCH 019/506] download cache is now in global user prefix --- etc/spack/defaults/config.yaml | 4 ---- lib/spack/spack/install_trees.py | 2 +- lib/spack/spack/paths.py | 12 ++++-------- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 85c106a5ad43f4..43f060437e8c3b 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -75,10 +75,6 @@ config: # name/hash. test_stage: $user_cache_path/test - # Cache directory for already downloaded source tarballs and archived - # repositories. This can be purged with `spack clean --downloads`. - source_cache: $spack/var/spack/cache - ## Directory where spack managed environments are created and stored # environments_root: $spack/var/spack/environments diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_trees.py index 4d8aca6b68ef17..41ce669d4c8db7 100644 --- a/lib/spack/spack/install_trees.py +++ b/lib/spack/spack/install_trees.py @@ -100,7 +100,7 @@ def _guard_writes(event, args): intent_to_modify = bool((set(mode) & set("wax")) or "r+" in mode) if abs_path.startswith(paths.prefix) and intent_to_modify: _attempted_modify_internal(f"Open {path} in mode [{mode}]") - elif event == "shutil.copyfile": + elif event in ["shutil.copyfile", "os.rename", "shutil.move"]: _, dst = args[:2] abs_dst = os.path.abspath(dst) if abs_dst.startswith(paths.prefix): diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index e5caa6c949d67d..65b7eea39381a0 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -114,7 +114,6 @@ def user_root(): var_path = os.path.join(per_spack_user_root, "var", "spack") # TODO: also check share_path/{lmod, tcl} -# TODO: can use new-style locations if user explicitly specifies --install-root # TODO: can shutil.mv everything in {opt, var} except installs into new user root # read-only things in $spack/var/spack @@ -122,13 +121,8 @@ def user_root(): packages_path = os.path.join(repos_path, "builtin") mock_packages_path = os.path.join(repos_path, "builtin.mock") -# -# Writable things in $var_path -# In older spack, this is $spack/var/spack -# fetch cache for downloaded files -default_fetch_cache_path = os.path.join(var_path, "cache") - -# GPG paths. +# GPG keys may be written into $spack/var/spack for older instances +# of spack, but otherwise are written into $per_spack_user gpg_keys_path = os.path.join(var_path, "gpg") mock_gpg_data_path = os.path.join(var_path, "gpg.mock", "data") mock_gpg_keys_path = os.path.join(var_path, "gpg.mock", "keys") @@ -150,6 +144,8 @@ def _get_user_cache_path(): user_cache_path = str(PurePath(_get_user_cache_path())) +default_fetch_cache_path = os.path.join(user_cache_path, "downloads") + #: junit, cdash, etc. reports about builds reports_path = os.path.join(user_cache_path, "reports") From 71100d64345c9fa5fe1d311cdd07f45a31208c7e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 13 Nov 2024 14:01:22 -0800 Subject: [PATCH 020/506] proper handling of module location defaults; gpg.mock is read only, so put it back in its old location --- etc/spack/defaults/modules.yaml | 4 ---- lib/spack/spack/modules/common.py | 4 ---- lib/spack/spack/paths.py | 16 ++++++++++------ var/spack/{read-gpg => }/gpg.mock/README.md | 0 .../{read-gpg => }/gpg.mock/data/content.txt | 0 .../{read-gpg => }/gpg.mock/data/content.txt.asc | 0 .../{read-gpg => }/gpg.mock/keys/external.key | 0 .../gpg.mock/keys/package-signing-key | 0 var/spack/read-gpg/gpg/README.md | 5 ----- 9 files changed, 10 insertions(+), 19 deletions(-) rename var/spack/{read-gpg => }/gpg.mock/README.md (100%) rename var/spack/{read-gpg => }/gpg.mock/data/content.txt (100%) rename var/spack/{read-gpg => }/gpg.mock/data/content.txt.asc (100%) rename var/spack/{read-gpg => }/gpg.mock/keys/external.key (100%) rename var/spack/{read-gpg => }/gpg.mock/keys/package-signing-key (100%) delete mode 100644 var/spack/read-gpg/gpg/README.md diff --git a/etc/spack/defaults/modules.yaml b/etc/spack/defaults/modules.yaml index 75ec3661174378..ee89695b227671 100644 --- a/etc/spack/defaults/modules.yaml +++ b/etc/spack/defaults/modules.yaml @@ -36,10 +36,6 @@ modules: # These are configurations for the module set named "default" default: - # Where to install modules - roots: - tcl: $spack/share/spack/modules - lmod: $spack/share/spack/lmod # What type of modules to use ("tcl" and/or "lmod") enable: [] diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py index 6122d5cab41439..9a9f496a74b8ce 100644 --- a/lib/spack/spack/modules/common.py +++ b/lib/spack/spack/modules/common.py @@ -211,13 +211,9 @@ def root_path(name, module_set_name): Returns: root folder for module file installation """ - defaults = {"lmod": "$spack/share/spack/lmod", "tcl": "$spack/share/spack/modules"} # Root folders where the various module files should be written roots = spack.config.get(f"modules:{module_set_name}:roots", {}) - # Merge config values into the defaults so we prefer configured values - roots = spack.config.merge_yaml(defaults, roots) - path = roots.get(name, os.path.join(spack.paths.modules_base, name)) return spack.util.path.canonicalize_path(path) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 65b7eea39381a0..de405b240f7f85 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -103,17 +103,21 @@ def user_root(): if dir_is_occupied(internal_opt_path): opt_path = internal_opt_path - modules_base = share_path else: opt_path = os.path.join(per_spack_user_root, "opt") - modules_base = os.path.join(per_spack_user_root, "modules") -if dir_is_occupied(read_var_path, except_for={"repos", "read-gpg"}): +if dir_is_occupied(read_var_path, except_for={"repos", "gpg.mock"}): var_path = read_var_path else: var_path = os.path.join(per_spack_user_root, "var", "spack") -# TODO: also check share_path/{lmod, tcl} +modules_base = None +for module_dir in ["lmod", "modules"]: + if dir_is_occupied(os.path.join(share_path, module_dir)): + modules_base = share_path +if not modules_base: + modules_base = os.path.join(per_spack_user_root, "modules") + # TODO: can shutil.mv everything in {opt, var} except installs into new user root # read-only things in $spack/var/spack @@ -124,8 +128,8 @@ def user_root(): # GPG keys may be written into $spack/var/spack for older instances # of spack, but otherwise are written into $per_spack_user gpg_keys_path = os.path.join(var_path, "gpg") -mock_gpg_data_path = os.path.join(var_path, "gpg.mock", "data") -mock_gpg_keys_path = os.path.join(var_path, "gpg.mock", "keys") +mock_gpg_data_path = os.path.join(read_var_path, "gpg.mock", "data") +mock_gpg_keys_path = os.path.join(read_var_path, "gpg.mock", "keys") gpg_path = os.path.join(opt_path, "spack", "gpg") diff --git a/var/spack/read-gpg/gpg.mock/README.md b/var/spack/gpg.mock/README.md similarity index 100% rename from var/spack/read-gpg/gpg.mock/README.md rename to var/spack/gpg.mock/README.md diff --git a/var/spack/read-gpg/gpg.mock/data/content.txt b/var/spack/gpg.mock/data/content.txt similarity index 100% rename from var/spack/read-gpg/gpg.mock/data/content.txt rename to var/spack/gpg.mock/data/content.txt diff --git a/var/spack/read-gpg/gpg.mock/data/content.txt.asc b/var/spack/gpg.mock/data/content.txt.asc similarity index 100% rename from var/spack/read-gpg/gpg.mock/data/content.txt.asc rename to var/spack/gpg.mock/data/content.txt.asc diff --git a/var/spack/read-gpg/gpg.mock/keys/external.key b/var/spack/gpg.mock/keys/external.key similarity index 100% rename from var/spack/read-gpg/gpg.mock/keys/external.key rename to var/spack/gpg.mock/keys/external.key diff --git a/var/spack/read-gpg/gpg.mock/keys/package-signing-key b/var/spack/gpg.mock/keys/package-signing-key similarity index 100% rename from var/spack/read-gpg/gpg.mock/keys/package-signing-key rename to var/spack/gpg.mock/keys/package-signing-key diff --git a/var/spack/read-gpg/gpg/README.md b/var/spack/read-gpg/gpg/README.md deleted file mode 100644 index 122d24f84126b4..00000000000000 --- a/var/spack/read-gpg/gpg/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# GPG Keys - -This directory contains keys that should be trusted by this installation of -Spack. They are imported when running `spack gpg init`, but may also be -imported manually with `spack gpg trust path/to/key`. From 21287cb400becafb9d0a0134ed8597e23b41db34 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 13 Nov 2024 14:54:44 -0800 Subject: [PATCH 021/506] partial refactor (moving away from opt/var_path variables) --- lib/spack/spack/cmd/clean.py | 4 +- lib/spack/spack/environment/environment.py | 2 +- lib/spack/spack/install_trees.py | 16 ++++---- lib/spack/spack/paths.py | 48 ++++++++++++---------- lib/spack/spack/test/cmd/clean.py | 17 ++++---- 5 files changed, 45 insertions(+), 42 deletions(-) diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 59d650fd12a7cc..b46b203c249b3b 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -17,7 +17,7 @@ import spack.store import spack.util.path from spack.cmd.common import arguments -from spack.paths import lib_path, var_path +import spack.paths as paths description = "remove temporary build files and/or downloaded archives" section = "build" @@ -73,7 +73,7 @@ def setup_parser(subparser): def remove_python_cache(): - for directory in [lib_path, var_path]: + for directory in [paths.lib_path, paths.repos_path]: for root, dirs, files in os.walk(directory): for f in files: if f.endswith(".pyc") or f.endswith(".pyo"): diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index b61332a0abc92c..8d00cb58b3db6c 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -68,7 +68,7 @@ #: default path where environments are stored in the spack tree -default_env_path = os.path.join(spack.paths.var_path, "environments") +default_env_path = spack.paths.envs_path #: Name of the input yaml file for an environment diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_trees.py index 41ce669d4c8db7..90afa868428854 100644 --- a/lib/spack/spack/install_trees.py +++ b/lib/spack/spack/install_trees.py @@ -16,11 +16,11 @@ install_root = None -_old_installs_base = os.path.join(paths.internal_opt_path, "spack") +_old_install_base = os.path.join("prefix", "opt", "spack") -default_install_base = os.path.join(paths.per_spack_user_root, "installs") +_default_install_base = os.path.join(paths.per_spack_user_root, "installs") -alias = {".root": _old_installs_base, ".user": default_install_base} +alias = {".root": _old_install_base, ".user": _default_install_base} def install_tree(): @@ -36,12 +36,12 @@ def install_tree(): # User can e.g. say --install-root=.root to refer to # the installation tree inside of the Spack prefix return alias.get(install_root, install_root) - elif paths.dir_is_occupied(default_install_base): - return default_install_base - elif paths.dir_is_occupied(_old_installs_base): - return _old_installs_base + elif paths.dir_is_occupied(_default_install_base): + return _default_install_base + elif paths.dir_is_occupied(_old_install_base): + return _old_install_base else: - return default_install_base + return _default_install_base def install_tree_config(): diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index de405b240f7f85..bab6573f734a19 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -76,6 +76,17 @@ def dir_is_occupied(x, except_for=None): etc_path = os.path.join(prefix, "etc", "spack") +# +# Things in $spack/etc/spack +# +default_license_dir = os.path.join(etc_path, "licenses") + +# +# Things in $spack/var/spack +# +read_var_path = os.path.join(prefix, "var", "spack") + + def user_root(): """Default install tree and config scope. @@ -89,22 +100,9 @@ def user_root(): per_spack_user_root = str(user_root()) -# -# Things in $spack/etc/spack -# -default_license_dir = os.path.join(etc_path, "licenses") - -# -# Things in $spack/var/spack -# -read_var_path = os.path.join(prefix, "var", "spack") - -internal_opt_path = os.path.join(prefix, "opt") - -if dir_is_occupied(internal_opt_path): - opt_path = internal_opt_path -else: - opt_path = os.path.join(per_spack_user_root, "opt") +#: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) +#: placed in per-spack-instance user root +default_misc_cache_path = os.path.join(per_spack_user_root, "cache") if dir_is_occupied(read_var_path, except_for={"repos", "gpg.mock"}): var_path = read_var_path @@ -118,7 +116,13 @@ def user_root(): if not modules_base: modules_base = os.path.join(per_spack_user_root, "modules") -# TODO: can shutil.mv everything in {opt, var} except installs into new user root +old_envs_path = os.path.join(var_path, "environments") +if dir_is_occupied(old_envs_path): + envs_path = old_envs_path +else: + envs_path = os.path.join(per_spack_user_root, "environments") + +# TODO: we could shutil.mv resources from old paths to new paths # read-only things in $spack/var/spack repos_path = os.path.join(read_var_path, "repos") @@ -130,8 +134,6 @@ def user_root(): gpg_keys_path = os.path.join(var_path, "gpg") mock_gpg_data_path = os.path.join(read_var_path, "gpg.mock", "data") mock_gpg_keys_path = os.path.join(read_var_path, "gpg.mock", "keys") -gpg_path = os.path.join(opt_path, "spack", "gpg") - # Below paths are where Spack can write information for the user. # Some are caches, some are not exactly caches. @@ -165,9 +167,11 @@ def _get_user_cache_path(): #: bootstrap store for bootstrapping clingo and other tools default_user_bootstrap_path = os.path.join(user_cache_path, "bootstrap") -#: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) -#: placed in per-spack-instance user root -default_misc_cache_path = os.path.join(per_spack_user_root, "cache") +old_gpg_path = os.path.join("prefix", "opt" "spack", "gpg") +if dir_is_occupied(old_gpg_path): + gpg_path = old_gpg_path +else: + gpg_path = os.path.join(user_cache_path, "gpg") #: Recorded directory where spack command was originally invoked spack_working_dir = None diff --git a/lib/spack/spack/test/cmd/clean.py b/lib/spack/spack/test/cmd/clean.py index 8b671e495ee008..c4b1962b60f9b2 100644 --- a/lib/spack/spack/test/cmd/clean.py +++ b/lib/spack/spack/test/cmd/clean.py @@ -14,6 +14,7 @@ import spack.environment as ev import spack.main import spack.package_base +import spack.paths import spack.spec import spack.stage import spack.store @@ -106,19 +107,17 @@ def _check_files(directory): assert not os.path.exists(fs.join_path(directory, "__pycache__")) source_dir = fs.join_path(tmpdir, "lib", "spack", "spack") - var_dir = fs.join_path(tmpdir, "var", "spack", "stuff") + repos_dir = fs.join_path(tmpdir, "var", "spack", "repos") - for d in [source_dir, var_dir]: + for d in [source_dir, repos_dir]: _setup_files(d) - # Patching the path variables from-import'd by spack.cmd.clean is needed - # to ensure the paths used by the command for this test reflect the - # temporary directory locations and not those from spack.paths when - # the clean command's module was imported. - monkeypatch.setattr(spack.cmd.clean, "lib_path", source_dir) - monkeypatch.setattr(spack.cmd.clean, "var_path", var_dir) + # spack.cmd.clean references paths from spack.paths: we want to + # update them for the duration of this test. + monkeypatch.setattr(spack.paths, "lib_path", source_dir) + monkeypatch.setattr(spack.paths, "repos_path", repos_dir) spack.cmd.clean.remove_python_cache() - for d in [source_dir, var_dir]: + for d in [source_dir, repos_dir]: _check_files(d) From 4bf2226dbe50fc2efce294c857e642cf7ddd9408 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 13 Nov 2024 15:38:45 -0800 Subject: [PATCH 022/506] reorg to remove distinct read_var_path (was confusing) --- lib/spack/spack/install_trees.py | 2 +- lib/spack/spack/paths.py | 25 ++++++++++++------------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_trees.py index 90afa868428854..c2f188e19e1f20 100644 --- a/lib/spack/spack/install_trees.py +++ b/lib/spack/spack/install_trees.py @@ -16,7 +16,7 @@ install_root = None -_old_install_base = os.path.join("prefix", "opt", "spack") +_old_install_base = os.path.join(paths.prefix, "opt", "spack") _default_install_base = os.path.join(paths.per_spack_user_root, "installs") diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index bab6573f734a19..24a8ee64a85574 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -84,7 +84,7 @@ def dir_is_occupied(x, except_for=None): # # Things in $spack/var/spack # -read_var_path = os.path.join(prefix, "var", "spack") +var_path = os.path.join(prefix, "var", "spack") def user_root(): @@ -104,11 +104,6 @@ def user_root(): #: placed in per-spack-instance user root default_misc_cache_path = os.path.join(per_spack_user_root, "cache") -if dir_is_occupied(read_var_path, except_for={"repos", "gpg.mock"}): - var_path = read_var_path -else: - var_path = os.path.join(per_spack_user_root, "var", "spack") - modules_base = None for module_dir in ["lmod", "modules"]: if dir_is_occupied(os.path.join(share_path, module_dir)): @@ -124,16 +119,14 @@ def user_root(): # TODO: we could shutil.mv resources from old paths to new paths -# read-only things in $spack/var/spack -repos_path = os.path.join(read_var_path, "repos") +# $spack/var/spack is generally read-only. Older instances may +# write gpg keys or environments into ...var/ +repos_path = os.path.join(var_path, "repos") packages_path = os.path.join(repos_path, "builtin") mock_packages_path = os.path.join(repos_path, "builtin.mock") -# GPG keys may be written into $spack/var/spack for older instances -# of spack, but otherwise are written into $per_spack_user -gpg_keys_path = os.path.join(var_path, "gpg") -mock_gpg_data_path = os.path.join(read_var_path, "gpg.mock", "data") -mock_gpg_keys_path = os.path.join(read_var_path, "gpg.mock", "keys") +mock_gpg_data_path = os.path.join(var_path, "gpg.mock", "data") +mock_gpg_keys_path = os.path.join(var_path, "gpg.mock", "keys") # Below paths are where Spack can write information for the user. # Some are caches, some are not exactly caches. @@ -173,6 +166,12 @@ def _get_user_cache_path(): else: gpg_path = os.path.join(user_cache_path, "gpg") +old_gpg_keys_path = os.path.join(var_path, "gpg") +if dir_is_occupied(old_gpg_keys_path): + gpg_keys_path = old_gpg_keys_path +else: + gpg_keys_path = os.path.join(user_cache_path, "gpg-keys") + #: Recorded directory where spack command was originally invoked spack_working_dir = None From f7ec052ef192d3e7e2d8f7373e3e6f8f9a83433a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 13 Nov 2024 15:50:01 -0800 Subject: [PATCH 023/506] organization is now a bit more complex --- lib/spack/spack/paths.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 24a8ee64a85574..34e58ddffe3c07 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -74,16 +74,7 @@ def dir_is_occupied(x, except_for=None): hooks_path = os.path.join(module_path, "hooks") share_path = os.path.join(prefix, "share", "spack") etc_path = os.path.join(prefix, "etc", "spack") - - -# -# Things in $spack/etc/spack -# default_license_dir = os.path.join(etc_path, "licenses") - -# -# Things in $spack/var/spack -# var_path = os.path.join(prefix, "var", "spack") From 692c5cfbd55c0f9c6e43688ec26311efc2e36bb9 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 13 Nov 2024 15:56:57 -0800 Subject: [PATCH 024/506] style fix --- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/paths.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index b46b203c249b3b..4333e467405580 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -13,11 +13,11 @@ import spack.caches import spack.cmd import spack.config +import spack.paths as paths import spack.stage import spack.store import spack.util.path from spack.cmd.common import arguments -import spack.paths as paths description = "remove temporary build files and/or downloaded archives" section = "build" diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 34e58ddffe3c07..1a8f177eb5b9bf 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -119,6 +119,7 @@ def user_root(): mock_gpg_data_path = os.path.join(var_path, "gpg.mock", "data") mock_gpg_keys_path = os.path.join(var_path, "gpg.mock", "keys") + # Below paths are where Spack can write information for the user. # Some are caches, some are not exactly caches. # From 5f99f99f332a5b6525fe7ba83b714be30f4bce85 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 Nov 2024 12:20:49 -0800 Subject: [PATCH 025/506] update misc-cache in config; change default behavior for install scheme --- etc/spack/defaults/config.yaml | 2 +- lib/spack/spack/install_trees.py | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 43f060437e8c3b..543138922a7bad 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -82,7 +82,7 @@ config: # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` - misc_cache: $user_cache_path/cache + misc_cache: $per_spack_user/cache # Timeout in seconds used for downloading sources etc. This only applies diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_trees.py index c2f188e19e1f20..6e7d0705536f10 100644 --- a/lib/spack/spack/install_trees.py +++ b/lib/spack/spack/install_trees.py @@ -16,11 +16,11 @@ install_root = None -_old_install_base = os.path.join(paths.prefix, "opt", "spack") +_in_spack = os.path.join(paths.prefix, "opt", "spack") -_default_install_base = os.path.join(paths.per_spack_user_root, "installs") +_in_user = os.path.join(paths.per_spack_user_root, "installs") -alias = {".root": _old_install_base, ".user": _default_install_base} +alias = {".root": _in_spack, ".user": _in_user} def install_tree(): @@ -36,12 +36,14 @@ def install_tree(): # User can e.g. say --install-root=.root to refer to # the installation tree inside of the Spack prefix return alias.get(install_root, install_root) - elif paths.dir_is_occupied(_default_install_base): - return _default_install_base - elif paths.dir_is_occupied(_old_install_base): - return _old_install_base + elif paths.dir_is_occupied(_in_user): + return _in_user else: - return _default_install_base + # In the future, we may only choose the in-spack + # scheme if that install tree is populated, but for + # now even new instances of Spack should install + # into the Spack prefix + return _in_spack def install_tree_config(): From 27d1b49a78997c739b918bb85deffa57add692bf Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 Nov 2024 12:51:22 -0800 Subject: [PATCH 026/506] renames --- lib/spack/spack/config.py | 2 +- .../{install_trees.py => install_scheme.py} | 19 +++++++++---------- lib/spack/spack/main.py | 10 +++++----- lib/spack/spack/store.py | 4 ++-- 4 files changed, 17 insertions(+), 18 deletions(-) rename lib/spack/spack/{install_trees.py => install_scheme.py} (90%) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 29f131904d2098..e0a55f8d9ed1a1 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -853,7 +853,7 @@ def create() -> Configuration: # Python package's can register configuration scopes via entry_points configuration_paths.extend(config_paths_from_entry_points()) - configuration_paths.append(("install", spack.install_trees.install_tree_config())) + configuration_paths.append(("install", spack.install_trees.config())) # User configuration can override both spack defaults and site config # This is disabled if user asks for no local configuration. diff --git a/lib/spack/spack/install_trees.py b/lib/spack/spack/install_scheme.py similarity index 90% rename from lib/spack/spack/install_trees.py rename to lib/spack/spack/install_scheme.py index 6e7d0705536f10..602bf95c4e328a 100644 --- a/lib/spack/spack/install_trees.py +++ b/lib/spack/spack/install_scheme.py @@ -14,7 +14,7 @@ import spack.paths as paths -install_root = None +scheme = None _in_spack = os.path.join(paths.prefix, "opt", "spack") @@ -23,19 +23,18 @@ alias = {".root": _in_spack, ".user": _in_user} -def install_tree(): +def default_install_location(): """ Selecting install-tree - - if user specifies --install-root, we use that - - if $user_root has one, we use that - - if $old_install_path is occupied, we use that - - otherwise, we use $user_root + - if user specifies --install-scheme, we use that + - if there is config or installs in $per_spack user, use that + - otherwise, we are managing installs inside of Spack """ - if install_root: + if scheme: # User can e.g. say --install-root=.root to refer to # the installation tree inside of the Spack prefix - return alias.get(install_root, install_root) + return alias.get(scheme, scheme) elif paths.dir_is_occupied(_in_user): return _in_user else: @@ -46,8 +45,8 @@ def install_tree(): return _in_spack -def install_tree_config(): - root = install_tree() +def config(): + root = default_install_location() cfgs = os.path.join(root, "configs") return cfgs diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 2cc6de3845b6f5..528d11dfca07c9 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -39,7 +39,7 @@ import spack.config import spack.environment as ev import spack.error -import spack.install_trees +import spack.install_scheme import spack.modules import spack.paths import spack.platforms @@ -503,8 +503,8 @@ def make_argument_parser(**kwargs): "--print-shell-vars", action="store", help="print info needed by setup-env.*sh" ) parser.add_argument( - "--install-root", - dest="install_root", + "--install-scheme", + dest="install_scheme", action="store", default=None, help="specify non-default install tree", @@ -576,8 +576,8 @@ def setup_main_options(args): if args.color is not None: color.set_color_when(args.color) - if args.install_root: - spack.install_trees.install_root = args.install_root + if args.install_scheme: + spack.install_scheme.scheme = args.install_scheme def allows_unknown_args(command): diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index bfc70487e284dc..1c16fc814555d2 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -31,7 +31,7 @@ import spack.database import spack.directory_layout import spack.error -import spack.install_trees +import spack.install_scheme import spack.paths import spack.spec import spack.util.path @@ -80,7 +80,7 @@ def parse_install_tree(config_dict): projections = {"all": all_projection} else: - unpadded_root = install_tree.get("root", spack.install_trees.install_tree()) + unpadded_root = install_tree.get("root", spack.install_scheme.default_install_location()) unpadded_root = spack.util.path.canonicalize_path(unpadded_root) padded_length = install_tree.get("padded_length", False) From 6fd3fd1dc01b4dbece291e0c9beaf48054ccbd03 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 Nov 2024 13:42:00 -0800 Subject: [PATCH 027/506] if we cant write to spack, we write in the user home --- lib/spack/spack/install_scheme.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/install_scheme.py index 602bf95c4e328a..f14ce215a37c17 100644 --- a/lib/spack/spack/install_scheme.py +++ b/lib/spack/spack/install_scheme.py @@ -30,6 +30,7 @@ def default_install_location(): - if user specifies --install-scheme, we use that - if there is config or installs in $per_spack user, use that - otherwise, we are managing installs inside of Spack + - ... unless the Spack prefix is not writable by us """ if scheme: # User can e.g. say --install-root=.root to refer to @@ -37,6 +38,9 @@ def default_install_location(): return alias.get(scheme, scheme) elif paths.dir_is_occupied(_in_user): return _in_user + elif not os.access(paths.prefix, os.W_OK): + # If we can't write into the Spack prefix we use ~ + return _in_user else: # In the future, we may only choose the in-spack # scheme if that install tree is populated, but for From 702045e7a4ea5d7658837174aeea143bf5754967 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 Nov 2024 13:43:34 -0800 Subject: [PATCH 028/506] reference error --- lib/spack/spack/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index e0a55f8d9ed1a1..9af92e9a6ce9be 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -40,7 +40,7 @@ from llnl.util import filesystem, lang, tty import spack.error -import spack.install_trees +import spack.install_scheme import spack.paths import spack.platforms import spack.schema @@ -853,7 +853,7 @@ def create() -> Configuration: # Python package's can register configuration scopes via entry_points configuration_paths.extend(config_paths_from_entry_points()) - configuration_paths.append(("install", spack.install_trees.config())) + configuration_paths.append(("install", spack.install_scheme.config())) # User configuration can override both spack defaults and site config # This is disabled if user asks for no local configuration. From e647c21841b968e1ad98643f2324e45901fe722c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 Nov 2024 14:04:00 -0800 Subject: [PATCH 029/506] shared tree concept no longer applies --- lib/spack/spack/install_scheme.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/install_scheme.py index f14ce215a37c17..e9bc1f6de33499 100644 --- a/lib/spack/spack/install_scheme.py +++ b/lib/spack/spack/install_scheme.py @@ -121,7 +121,3 @@ def guard_writes_into_spack(): import sys sys.addaudithook(_guard_writes) - - -def shared_trees(): - return pathlib.Path(paths.system_config_path) / "install-trees" From c622e5e056c6f796dc6b4c723bd82d42ef59ab61 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 Nov 2024 14:23:22 -0800 Subject: [PATCH 030/506] internalize some state; make it possible to ask what install scheme we are using (abstractly, not as a dir) --- lib/spack/spack/install_scheme.py | 58 +++++++++++++++++++++++-------- lib/spack/spack/main.py | 2 +- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/install_scheme.py index e9bc1f6de33499..d57fd093fb1ad4 100644 --- a/lib/spack/spack/install_scheme.py +++ b/lib/spack/spack/install_scheme.py @@ -12,15 +12,47 @@ import pathlib import warnings +import spack.error import spack.paths as paths -scheme = None +_scheme = None _in_spack = os.path.join(paths.prefix, "opt", "spack") _in_user = os.path.join(paths.per_spack_user_root, "installs") -alias = {".root": _in_spack, ".user": _in_user} +class InstallScheme: + ROOT = 0 + USER = 1 + +aliases = {"root": InstallScheme.ROOT, "user": InstallScheme.USER} + + +def set(alias): + if alias in aliases: + _scheme = aliases[alias] + + +def scheme(): + return _determine_scheme() + + +def _determine_scheme(): + global _scheme + if _scheme: + pass + elif paths.dir_is_occupied(_in_user): + _scheme = InstallScheme.USER + elif not os.access(paths.prefix, os.W_OK): + # If we can't write into the Spack prefix we use ~ + _scheme = InstallScheme.USER + else: + # In the future, we may only choose the in-spack + # scheme if that install tree is populated, but for + # now even new instances of Spack should install + # into the Spack prefix + _scheme = InstallScheme.ROOT + return _scheme def default_install_location(): @@ -32,21 +64,13 @@ def default_install_location(): - otherwise, we are managing installs inside of Spack - ... unless the Spack prefix is not writable by us """ - if scheme: - # User can e.g. say --install-root=.root to refer to - # the installation tree inside of the Spack prefix - return alias.get(scheme, scheme) - elif paths.dir_is_occupied(_in_user): + chosen = scheme() + if chosen == InstallScheme.USER: return _in_user - elif not os.access(paths.prefix, os.W_OK): - # If we can't write into the Spack prefix we use ~ - return _in_user - else: - # In the future, we may only choose the in-spack - # scheme if that install tree is populated, but for - # now even new instances of Spack should install - # into the Spack prefix + elif chosen == InstallScheme.ROOT: return _in_spack + else: + raise InstallSchemeError(f"Unset or unexpected scheme: {chosen}") def config(): @@ -55,6 +79,10 @@ def config(): return cfgs +class InstallSchemeError(spack.error.SpackError): + pass + + def _most_recent_internal_call(): """If called within an audit for a Python library function, finds the most recent spot within Spack's source code that generated diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 528d11dfca07c9..ad3e560f27e24e 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -577,7 +577,7 @@ def setup_main_options(args): color.set_color_when(args.color) if args.install_scheme: - spack.install_scheme.scheme = args.install_scheme + spack.install_scheme.set(args.install_scheme) def allows_unknown_args(command): From 88ca2d1fbb9d4b1d14ce290e2ccc95279100ba32 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 Nov 2024 14:47:00 -0800 Subject: [PATCH 031/506] allow an admin to add config that applies to end users but not to the admin --- lib/spack/spack/config.py | 2 ++ lib/spack/spack/paths.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 9af92e9a6ce9be..3bc025ad0a6fd7 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -845,6 +845,8 @@ def create() -> Configuration: # This is disabled if user asks for no local configuration. if not disable_local_config: configuration_paths.append(("system", spack.paths.system_config_path)) + if spack.install_scheme.scheme() == spack.install_scheme.InstallScheme.USER: + configuration_paths.append(("admin-customer", spack.paths.admin_customer_cfg)) # Site configuration is per spack instance, for sites or projects # No site-level configs should be checked into spack by default. diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 1a8f177eb5b9bf..f628cc9ca7f330 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -47,6 +47,10 @@ def dir_is_occupied(x, except_for=None): #: System configuration location system_config_path = _get_system_config_path() +#: When Spack is provided by an admin to a user, the admin can +#: provide a config that only applies for the end-users +admin_customer_cfg = os.path.join(system_config_path, "admin-customer") + #: synonym for prefix spack_root = prefix From e8fa98d51dbb52b8da530c8cc05b1eed0e25f7e5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 Nov 2024 14:52:43 -0800 Subject: [PATCH 032/506] ref errors --- lib/spack/spack/install_scheme.py | 4 +++- lib/spack/spack/main.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/install_scheme.py index d57fd093fb1ad4..7119fba2aff969 100644 --- a/lib/spack/spack/install_scheme.py +++ b/lib/spack/spack/install_scheme.py @@ -21,14 +21,16 @@ _in_user = os.path.join(paths.per_spack_user_root, "installs") + class InstallScheme: ROOT = 0 USER = 1 + aliases = {"root": InstallScheme.ROOT, "user": InstallScheme.USER} -def set(alias): +def set_scheme(alias): if alias in aliases: _scheme = aliases[alias] diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index ad3e560f27e24e..323c84be81d6af 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -577,7 +577,7 @@ def setup_main_options(args): color.set_color_when(args.color) if args.install_scheme: - spack.install_scheme.set(args.install_scheme) + spack.install_scheme.set_scheme(args.install_scheme) def allows_unknown_args(command): @@ -965,7 +965,7 @@ def _main(argv=None): cmd_name = args.command[0] cmd_name, args.command = resolve_alias(cmd_name, args.command) - spack.install_trees.guard_writes_into_spack() + spack.install_scheme.guard_writes_into_spack() # set up a bootstrap context, if asked. # bootstrap context needs to include parsing the command, b/c things From 50bdf3c7b4ee843a44d12063e701abdfd1558591 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 Nov 2024 15:25:29 -0800 Subject: [PATCH 033/506] global ref error --- lib/spack/spack/install_scheme.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/install_scheme.py index 7119fba2aff969..d9a216ed3fe5a9 100644 --- a/lib/spack/spack/install_scheme.py +++ b/lib/spack/spack/install_scheme.py @@ -31,6 +31,7 @@ class InstallScheme: def set_scheme(alias): + global _scheme if alias in aliases: _scheme = aliases[alias] From 2aaf4ba73035724fccad44d65d85a3cb06a2adc0 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 Nov 2024 16:30:29 -0800 Subject: [PATCH 034/506] install-scheme does not dictate whether we want end-user config --- lib/spack/spack/config.py | 5 ++++- lib/spack/spack/main.py | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3bc025ad0a6fd7..3236e93675454f 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -819,6 +819,9 @@ def _add_command_line_scopes(cfg: Configuration, command_line_scopes: List[str]) cfg.push_scope(scope) +end_user_system_scope = True + + def create() -> Configuration: """Singleton Configuration instance. @@ -845,7 +848,7 @@ def create() -> Configuration: # This is disabled if user asks for no local configuration. if not disable_local_config: configuration_paths.append(("system", spack.paths.system_config_path)) - if spack.install_scheme.scheme() == spack.install_scheme.InstallScheme.USER: + if end_user_system_scope: configuration_paths.append(("admin-customer", spack.paths.admin_customer_cfg)) # Site configuration is per spack instance, for sites or projects diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 323c84be81d6af..b5f8527e1a0ed1 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -507,7 +507,10 @@ def make_argument_parser(**kwargs): dest="install_scheme", action="store", default=None, - help="specify non-default install tree", + help="specify install scheme", + ) + parser.add_argument( + "--disable-end-user-config", action="store_true", help="Disable system config scope for end users" ) return parser @@ -579,6 +582,9 @@ def setup_main_options(args): if args.install_scheme: spack.install_scheme.set_scheme(args.install_scheme) + if args.disable_end_user_config: + spack.config.end_user_system_scope = False + def allows_unknown_args(command): """Implements really simple argument injection for unknown arguments. From 84b392562e700aa6deaf636183cdb6b051d9f234 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 Nov 2024 16:32:12 -0800 Subject: [PATCH 035/506] style edit --- lib/spack/spack/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index b5f8527e1a0ed1..7e0721a6b49a11 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -510,7 +510,9 @@ def make_argument_parser(**kwargs): help="specify install scheme", ) parser.add_argument( - "--disable-end-user-config", action="store_true", help="Disable system config scope for end users" + "--disable-end-user-config", + action="store_true", + help="Disable system config scope for end users", ) return parser From 55566146da22c1d41a187b59d1c1a2a7a0d71364 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 Nov 2024 22:48:11 -0800 Subject: [PATCH 036/506] more comments, change install scheme names --- lib/spack/spack/install_scheme.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/install_scheme.py index d9a216ed3fe5a9..3166b15f51be5f 100644 --- a/lib/spack/spack/install_scheme.py +++ b/lib/spack/spack/install_scheme.py @@ -23,17 +23,19 @@ class InstallScheme: - ROOT = 0 - USER = 1 + IN_SPACK = 0 + OUT_OF_SPACK = 1 -aliases = {"root": InstallScheme.ROOT, "user": InstallScheme.USER} +aliases = {"in-spack": InstallScheme.IN_SPACK, "out-of-spack": InstallScheme.OUT_OF_SPACK} def set_scheme(alias): global _scheme if alias in aliases: _scheme = aliases[alias] + else: + raise InstallSchemeError(f"Unknown install scheme: {alias}") def scheme(): @@ -41,20 +43,22 @@ def scheme(): def _determine_scheme(): + """Automatically choose an install scheme if it has not been set. + """ global _scheme if _scheme: pass elif paths.dir_is_occupied(_in_user): - _scheme = InstallScheme.USER + _scheme = InstallScheme.OUT_OF_SPACK elif not os.access(paths.prefix, os.W_OK): # If we can't write into the Spack prefix we use ~ - _scheme = InstallScheme.USER + _scheme = InstallScheme.OUT_OF_SPACK else: # In the future, we may only choose the in-spack # scheme if that install tree is populated, but for # now even new instances of Spack should install # into the Spack prefix - _scheme = InstallScheme.ROOT + _scheme = InstallScheme.IN_SPACK return _scheme @@ -68,9 +72,9 @@ def default_install_location(): - ... unless the Spack prefix is not writable by us """ chosen = scheme() - if chosen == InstallScheme.USER: + if chosen == InstallScheme.OUT_OF_SPACK: return _in_user - elif chosen == InstallScheme.ROOT: + elif chosen == InstallScheme.IN_SPACK: return _in_spack else: raise InstallSchemeError(f"Unset or unexpected scheme: {chosen}") From 9d01978759f7a064ddcb58c623d7d575001c8223 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 Nov 2024 23:57:22 -0800 Subject: [PATCH 037/506] style fix --- lib/spack/spack/install_scheme.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/install_scheme.py index 3166b15f51be5f..ad7d72aaf54633 100644 --- a/lib/spack/spack/install_scheme.py +++ b/lib/spack/spack/install_scheme.py @@ -43,8 +43,7 @@ def scheme(): def _determine_scheme(): - """Automatically choose an install scheme if it has not been set. - """ + """Automatically choose an install scheme if it has not been set.""" global _scheme if _scheme: pass From d29ae402a50450d5d680ab00683ec81fd7f28caf Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 15 Nov 2024 00:15:21 -0800 Subject: [PATCH 038/506] bool me once; shame on me --- lib/spack/spack/install_scheme.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/install_scheme.py index ad7d72aaf54633..5db137cea876df 100644 --- a/lib/spack/spack/install_scheme.py +++ b/lib/spack/spack/install_scheme.py @@ -23,8 +23,8 @@ class InstallScheme: - IN_SPACK = 0 - OUT_OF_SPACK = 1 + IN_SPACK = 1 + OUT_OF_SPACK = 2 aliases = {"in-spack": InstallScheme.IN_SPACK, "out-of-spack": InstallScheme.OUT_OF_SPACK} From 5dbf1f53c38b4bc69c7ab2ed1e75a48eb2855e1c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 18 Nov 2024 13:07:17 -0800 Subject: [PATCH 039/506] sys.audithook is python >= 3.8 only; also update module docstring --- lib/spack/spack/install_scheme.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/install_scheme.py index 5db137cea876df..705e351be13146 100644 --- a/lib/spack/spack/install_scheme.py +++ b/lib/spack/spack/install_scheme.py @@ -4,7 +4,12 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ -For tracking install trees. +This module tracks whether we are installing packages into the Spack +prefix or outside of it. If we are installing outside of Spack, that +is controlled by ``$per_spack_user/installs/config``. By default, +when installing outside of Spack, packages are installed into +``$per_spack_user``; you can change that with +``config:install_tree:root``. """ import inspect @@ -153,5 +158,5 @@ def _guard_writes(event, args): def guard_writes_into_spack(): import sys - - sys.addaudithook(_guard_writes) + if sys.version_info[:2] >= (3, 8): + sys.addaudithook(_guard_writes) From bfd1bdce0786bf8c196617a9810390da584a1272 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 18 Nov 2024 13:13:04 -0800 Subject: [PATCH 040/506] style edit --- lib/spack/spack/install_scheme.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/install_scheme.py index 705e351be13146..73ae2a99cb4a22 100644 --- a/lib/spack/spack/install_scheme.py +++ b/lib/spack/spack/install_scheme.py @@ -158,5 +158,6 @@ def _guard_writes(event, args): def guard_writes_into_spack(): import sys + if sys.version_info[:2] >= (3, 8): sys.addaudithook(_guard_writes) From b15b0be36ae08590fad28c64fb5a987511e5bd1c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 18 Nov 2024 13:31:49 -0800 Subject: [PATCH 041/506] redundant import --- lib/spack/spack/store.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 1c16fc814555d2..752cb6ab3d56ce 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -32,7 +32,6 @@ import spack.directory_layout import spack.error import spack.install_scheme -import spack.paths import spack.spec import spack.util.path From 35f872204981ab7153c3e201bea4003c0711af27 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 18 Nov 2024 13:36:03 -0800 Subject: [PATCH 042/506] try disabling vermin check given that audithook is conditioned on python version --- lib/spack/spack/install_scheme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/install_scheme.py index 73ae2a99cb4a22..54aa3a312e4944 100644 --- a/lib/spack/spack/install_scheme.py +++ b/lib/spack/spack/install_scheme.py @@ -160,4 +160,4 @@ def guard_writes_into_spack(): import sys if sys.version_info[:2] >= (3, 8): - sys.addaudithook(_guard_writes) + sys.addaudithook(_guard_writes) # novermin From 4faedc9d473f91e25d5086870527be21bdf608b8 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 18 Nov 2024 14:13:06 -0800 Subject: [PATCH 043/506] update bash completion --- share/spack/spack-completion.bash | 2 +- share/spack/spack-completion.fish | 56 +++++++++++++++++-------------- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index d8c58143c97a34..107dffa91ade34 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -399,7 +399,7 @@ SPACK_ALIASES="concretise:concretize;containerise:containerize;rm:remove" _spack() { if $list_options then - SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -b --bootstrap -p --profile --sorted-profile --lines -v --verbose --stacktrace -t --backtrace -V --version --print-shell-vars" + SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -b --bootstrap -p --profile --sorted-profile --lines -v --verbose --stacktrace -t --backtrace -V --version --print-shell-vars --install-scheme --disable-end-user-config" else SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean clone commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse logs maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" fi diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index afea0b1a57af6c..f8dfd06a385ca0 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -347,7 +347,7 @@ complete -c spack --erase # Everything below here is auto-generated. # spack -set -g __fish_spack_optspecs_spack h/help H/all-help color= c/config= C/config-scope= d/debug timestamp pdb e/env= D/env-dir= E/no-env use-env-repo k/insecure l/enable-locks L/disable-locks m/mock b/bootstrap p/profile sorted-profile= lines= v/verbose stacktrace t/backtrace V/version print-shell-vars= +set -g __fish_spack_optspecs_spack h/help H/all-help color= c/config= C/config-scope= d/debug timestamp pdb e/env= D/env-dir= E/no-env use-env-repo k/insecure l/enable-locks L/disable-locks m/mock b/bootstrap p/profile sorted-profile= lines= v/verbose stacktrace t/backtrace V/version print-shell-vars= install-scheme= disable-end-user-config complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a add -d 'add a spec to an environment' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a arch -d 'print architecture information about this machine' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a audit -d 'audit configuration files, packages, etc.' @@ -479,6 +479,10 @@ complete -c spack -n '__fish_spack_using_command ' -s V -l version -f -a version complete -c spack -n '__fish_spack_using_command ' -s V -l version -d 'show version number and exit' complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -f -a print_shell_vars complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -d 'print info needed by setup-env.*sh' +complete -c spack -n '__fish_spack_using_command ' -l install-scheme -r -f -a install_scheme +complete -c spack -n '__fish_spack_using_command ' -l install-scheme -r -d 'specify install scheme' +complete -c spack -n '__fish_spack_using_command ' -l disable-end-user-config -f -a disable_end_user_config +complete -c spack -n '__fish_spack_using_command ' -l disable-end-user-config -d 'Disable system config scope for end users' # spack add set -g __fish_spack_optspecs_spack_add h/help l/list-name= @@ -603,7 +607,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_enable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap enable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap disable @@ -611,7 +615,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_disable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap disable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap reset @@ -626,14 +630,14 @@ set -g __fish_spack_optspecs_spack_bootstrap_root h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap root' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap list set -g __fish_spack_optspecs_spack_bootstrap_list h/help scope= complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap add @@ -642,7 +646,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap add' -f -a '(__ complete -c spack -n '__fish_spack_using_command_pos 1 bootstrap add' -f -a '(__fish_spack_environments)' complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -f -a trust complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -d 'enable the source immediately upon addition' @@ -820,7 +824,7 @@ complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirro complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -d 'override any configured mirrors with this mirror URL' complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -f -a output_file complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -d 'file where rebuild info should be written' -complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -d 'configuration scope containing mirrors to check' complete -c spack -n '__fish_spack_using_command buildcache check' -s s -l spec -r -f -a spec complete -c spack -n '__fish_spack_using_command buildcache check' -s s -l spec -r -d 'check single spec instead of release specs file' @@ -1067,7 +1071,7 @@ complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolcha complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolchain -d 'Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -d 'Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1081,7 +1085,7 @@ complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchai complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchain -d 'Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -d 'Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1093,7 +1097,7 @@ complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help - complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -d 'configuration scope to modify' # spack compiler rm @@ -1103,14 +1107,14 @@ complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -d 'configuration scope to modify' # spack compiler list set -g __fish_spack_optspecs_spack_compiler_list h/help scope= complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -d 'configuration scope to read from' # spack compiler info @@ -1118,14 +1122,14 @@ set -g __fish_spack_optspecs_spack_compiler_info h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 compiler info' -f -a '(__fish_spack_installed_compilers)' complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -d 'configuration scope to read from' # spack compilers set -g __fish_spack_optspecs_spack_compilers h/help scope= complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -d 'configuration scope to read/modify' # spack concretize @@ -1185,7 +1189,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a update -d ' complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a revert -d 'revert configuration files to their state before update' complete -c spack -n '__fish_spack_using_command config' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify' # spack config get @@ -1704,7 +1708,7 @@ complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -f complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -d 'packages to exclude from search' complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -f -a path complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -d 'one or more alternative search paths for finding externals' -complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command external find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command external find' -l all -f -a all complete -c spack -n '__fish_spack_using_command external find' -l all -d 'search for all packages that Spack knows about' @@ -2277,7 +2281,7 @@ set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsig complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use `--type binary --type source` (default)' @@ -2307,7 +2311,7 @@ set -g __fish_spack_optspecs_spack_mirror_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror remove' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -d 'configuration scope to modify' # spack mirror rm @@ -2315,7 +2319,7 @@ set -g __fish_spack_optspecs_spack_mirror_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror rm' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -d 'configuration scope to modify' # spack mirror set-url @@ -2327,7 +2331,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -f -a p complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -d 'set only the URL used for uploading' complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -f -a fetch complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -d 'set only the URL used for downloading' -complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2365,7 +2369,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a s complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache' complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed complete -c spack -n '__fish_spack_using_command mirror set' -l signed -d 'require signing and signature verification when pushing and installing from this build cache' -complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2386,7 +2390,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password -r set -g __fish_spack_optspecs_spack_mirror_list h/help scope= complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'configuration scope to read from' # spack module @@ -2693,7 +2697,7 @@ complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirecto set -g __fish_spack_optspecs_spack_repo_list h/help scope= complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'configuration scope to read from' # spack repo add @@ -2701,7 +2705,7 @@ set -g __fish_spack_optspecs_spack_repo_add h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo add' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -d 'configuration scope to modify' # spack repo remove @@ -2709,7 +2713,7 @@ set -g __fish_spack_optspecs_spack_repo_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo remove' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -d 'configuration scope to modify' # spack repo rm @@ -2717,7 +2721,7 @@ set -g __fish_spack_optspecs_spack_repo_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo rm' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify' # spack resource From ccf66055fd62ed68edaec87885628b80803cf58a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 6 Jan 2025 16:27:10 -0800 Subject: [PATCH 044/506] file modification audit was careful to omit itself from the trace, but also needed to omit python lib logic --- lib/spack/spack/install_scheme.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/install_scheme.py index 54aa3a312e4944..add64bb7912a84 100644 --- a/lib/spack/spack/install_scheme.py +++ b/lib/spack/spack/install_scheme.py @@ -102,9 +102,10 @@ def _most_recent_internal_call(): stack = inspect.stack() this_file = str(pathlib.Path(__file__).resolve()) + spack_prefix = pathlib.Path(paths.prefix).resolve() for frame in stack: frame_loc = pathlib.Path(frame.filename).resolve() - if str(frame_loc) != this_file: + if str(frame_loc) != this_file and spack_prefix in frame_loc.parents: return frame_loc, frame.lineno return None, None From b03d6792e0d7363579a9cc11bfd1340e97ef9290 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 6 Jan 2025 16:40:49 -0800 Subject: [PATCH 045/506] fix license for new file --- lib/spack/spack/install_scheme.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/install_scheme.py index add64bb7912a84..78845e197c95c1 100644 --- a/lib/spack/spack/install_scheme.py +++ b/lib/spack/spack/install_scheme.py @@ -1,5 +1,4 @@ -# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. +# Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) From cae757bc5977baf714790713a9ad0fb40504787f Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 6 Jan 2025 19:16:40 -0800 Subject: [PATCH 046/506] internal write warnings are now enabled with an option to spack --- lib/spack/spack/main.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index aef5663805fe74..8dfda64e280546 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -512,6 +512,11 @@ def make_argument_parser(**kwargs): action="store_true", help="Disable system config scope for end users", ) + parser.add_argument( + "--guard-writes-into-spack", + action="store_true", + help="Warn when Spack tries to write into its own prefix", + ) return parser @@ -1000,7 +1005,8 @@ def _main(argv=None): cmd_name = args.command[0] cmd_name, args.command = resolve_alias(cmd_name, args.command) - spack.install_scheme.guard_writes_into_spack() + if args.guard_writes_into_spack: + spack.install_scheme.guard_writes_into_spack() # set up a bootstrap context, if asked. # bootstrap context needs to include parsing the command, b/c things From 1f9b03a5bb058ff1cdbb48d24bdbc117e0d41e21 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 6 Jan 2025 20:07:15 -0800 Subject: [PATCH 047/506] make install tree scope higher precendence than user scope --- lib/spack/spack/config.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index e8f6c1e9fd0cae..564d0f32d59b0a 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -471,10 +471,6 @@ def writable_scopes(self) -> Generator[ConfigScope, None, None]: def highest_precedence_scope(self) -> ConfigScope: """Writable scope with highest precedence.""" - if "install" in self.scopes: - # Preference is to write into config scope associated with - # the installation tree, if it exists for this Configuration - return self.scopes["install"] return next(s for s in reversed(self.scopes.values()) if s.writable) # type: ignore def highest_precedence_non_platform_scope(self) -> ConfigScope: @@ -846,13 +842,13 @@ def create() -> Configuration: # Python package's can register configuration scopes via entry_points configuration_paths.extend(config_paths_from_entry_points()) - configuration_paths.append(("install", spack.install_scheme.config())) - # User configuration can override both spack defaults and site config # This is disabled if user asks for no local configuration. if not disable_local_config: configuration_paths.append(("user", spack.paths.user_config_path)) + configuration_paths.append(("install", spack.install_scheme.config())) + # add each scope and its platform-specific directory for name, path in configuration_paths: cfg.push_scope(DirectoryConfigScope(name, path)) From 0e8a15392c04ba9becaf5755123eedf8fe9e94e1 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 6 Jan 2025 20:41:52 -0800 Subject: [PATCH 048/506] simple command that tells spack to start installing outside of the spack prefix --- lib/spack/spack/cmd/read_only_core.py | 20 ++++++++++++++++++++ lib/spack/spack/install_scheme.py | 11 +++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 lib/spack/spack/cmd/read_only_core.py diff --git a/lib/spack/spack/cmd/read_only_core.py b/lib/spack/spack/cmd/read_only_core.py new file mode 100644 index 00000000000000..657b212d63652e --- /dev/null +++ b/lib/spack/spack/cmd/read_only_core.py @@ -0,0 +1,20 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from llnl.util.filesystem import mkdirp + +import spack.install_scheme + +description = "Update Spack to stop writing into its own prefix" +section = "build" +level = "long" + + +def setup_parser(subparser): + pass + + +def read_only_core(parser, args): + config_dir = spack.install_scheme.config(for_alias="out-of-spack") + + mkdirp(config_dir) diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/install_scheme.py index 78845e197c95c1..127d1ee36a84a6 100644 --- a/lib/spack/spack/install_scheme.py +++ b/lib/spack/spack/install_scheme.py @@ -65,7 +65,7 @@ def _determine_scheme(): return _scheme -def default_install_location(): +def default_install_location(for_alias=None): """ Selecting install-tree @@ -74,7 +74,10 @@ def default_install_location(): - otherwise, we are managing installs inside of Spack - ... unless the Spack prefix is not writable by us """ - chosen = scheme() + if for_alias: + chosen = aliases.get(for_alias) + else: + chosen = scheme() if chosen == InstallScheme.OUT_OF_SPACK: return _in_user elif chosen == InstallScheme.IN_SPACK: @@ -83,8 +86,8 @@ def default_install_location(): raise InstallSchemeError(f"Unset or unexpected scheme: {chosen}") -def config(): - root = default_install_location() +def config(for_alias=None): + root = default_install_location(for_alias) cfgs = os.path.join(root, "configs") return cfgs From 6f5024b979c3156d00abf0cdd7cf742495aa3883 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 6 Jan 2025 20:42:27 -0800 Subject: [PATCH 049/506] update completion --- share/spack/spack-completion.bash | 8 +++-- share/spack/spack-completion.fish | 60 +++++++++++++++++-------------- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 9ee1090cba71bb..334a1319810848 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -398,9 +398,9 @@ SPACK_ALIASES="concretise:concretize;containerise:containerize;rm:remove" _spack() { if $list_options then - SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -b --bootstrap -p --profile --sorted-profile --lines -v --verbose --stacktrace -t --backtrace -V --version --print-shell-vars --install-scheme --disable-end-user-config" + SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -b --bootstrap -p --profile --sorted-profile --lines -v --verbose --stacktrace -t --backtrace -V --version --print-shell-vars --install-scheme --disable-end-user-config --guard-writes-into-spack" else - SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean clone commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse logs maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" + SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean clone commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse logs maintainers make-installer mark mirror module patch pkg providers pydoc python read-only-core reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" fi } @@ -1739,6 +1739,10 @@ _spack_python() { fi } +_spack_read_only_core() { + SPACK_COMPREPLY="-h --help" +} + _spack_reindex() { SPACK_COMPREPLY="-h --help" } diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index ef827c9b0eaaf3..0e88e6e6db865c 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -346,7 +346,7 @@ complete -c spack --erase # Everything below here is auto-generated. # spack -set -g __fish_spack_optspecs_spack h/help H/all-help color= c/config= C/config-scope= d/debug timestamp pdb e/env= D/env-dir= E/no-env use-env-repo k/insecure l/enable-locks L/disable-locks m/mock b/bootstrap p/profile sorted-profile= lines= v/verbose stacktrace t/backtrace V/version print-shell-vars= install-scheme= disable-end-user-config +set -g __fish_spack_optspecs_spack h/help H/all-help color= c/config= C/config-scope= d/debug timestamp pdb e/env= D/env-dir= E/no-env use-env-repo k/insecure l/enable-locks L/disable-locks m/mock b/bootstrap p/profile sorted-profile= lines= v/verbose stacktrace t/backtrace V/version print-shell-vars= install-scheme= disable-end-user-config guard-writes-into-spack complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a add -d 'add a spec to an environment' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a arch -d 'print architecture information about this machine' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a audit -d 'audit configuration files, packages, etc.' @@ -406,6 +406,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a pkg -d 'query pac complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a providers -d 'list packages that provide a particular virtual package' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a pydoc -d 'run pydoc from within spack' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a python -d 'launch an interpreter as spack would launch a command' +complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a read-only-core -d 'Update Spack to stop writing into its own prefix' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a reindex -d 'rebuild Spack'"'"'s package database' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a remove -d 'remove specs from an environment' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a rm -d 'remove specs from an environment' @@ -482,6 +483,8 @@ complete -c spack -n '__fish_spack_using_command ' -l install-scheme -r -f -a in complete -c spack -n '__fish_spack_using_command ' -l install-scheme -r -d 'specify install scheme' complete -c spack -n '__fish_spack_using_command ' -l disable-end-user-config -f -a disable_end_user_config complete -c spack -n '__fish_spack_using_command ' -l disable-end-user-config -d 'Disable system config scope for end users' +complete -c spack -n '__fish_spack_using_command ' -l guard-writes-into-spack -f -a guard_writes_into_spack +complete -c spack -n '__fish_spack_using_command ' -l guard-writes-into-spack -d 'Warn when Spack tries to write into its own prefix' # spack add set -g __fish_spack_optspecs_spack_add h/help l/list-name= @@ -606,7 +609,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_enable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap enable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap disable @@ -614,7 +617,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_disable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap disable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap reset @@ -629,14 +632,14 @@ set -g __fish_spack_optspecs_spack_bootstrap_root h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap root' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap list set -g __fish_spack_optspecs_spack_bootstrap_list h/help scope= complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap add @@ -645,7 +648,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap add' -f -a '(__ complete -c spack -n '__fish_spack_using_command_pos 1 bootstrap add' -f -a '(__fish_spack_environments)' complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -f -a trust complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -d 'enable the source immediately upon addition' @@ -823,7 +826,7 @@ complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirro complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -d 'override any configured mirrors with this mirror URL' complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -f -a output_file complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -d 'file where rebuild info should be written' -complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -d 'configuration scope containing mirrors to check' complete -c spack -n '__fish_spack_using_command buildcache check' -s s -l spec -r -f -a spec complete -c spack -n '__fish_spack_using_command buildcache check' -s s -l spec -r -d 'check single spec instead of release specs file' @@ -1082,7 +1085,7 @@ complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolcha complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolchain -d 'Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -d 'Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1096,7 +1099,7 @@ complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchai complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchain -d 'Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -d 'Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1108,7 +1111,7 @@ complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help - complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -d 'configuration scope to modify' # spack compiler rm @@ -1118,14 +1121,14 @@ complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -d 'configuration scope to modify' # spack compiler list set -g __fish_spack_optspecs_spack_compiler_list h/help scope= complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -d 'configuration scope to read from' # spack compiler info @@ -1133,14 +1136,14 @@ set -g __fish_spack_optspecs_spack_compiler_info h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 compiler info' -f -a '(__fish_spack_installed_compilers)' complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -d 'configuration scope to read from' # spack compilers set -g __fish_spack_optspecs_spack_compilers h/help scope= complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -d 'configuration scope to read/modify' # spack concretize @@ -1200,7 +1203,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a update -d ' complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a revert -d 'revert configuration files to their state before update' complete -c spack -n '__fish_spack_using_command config' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify' # spack config get @@ -1741,7 +1744,7 @@ complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -f complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -d 'packages to exclude from search' complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -f -a path complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -d 'one or more alternative search paths for finding externals' -complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command external find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command external find' -l all -f -a all complete -c spack -n '__fish_spack_using_command external find' -l all -d 'search for all packages that Spack knows about' @@ -2308,7 +2311,7 @@ set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsig complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use `--type binary --type source` (default)' @@ -2348,7 +2351,7 @@ set -g __fish_spack_optspecs_spack_mirror_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror remove' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -d 'configuration scope to modify' # spack mirror rm @@ -2356,7 +2359,7 @@ set -g __fish_spack_optspecs_spack_mirror_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror rm' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -d 'configuration scope to modify' # spack mirror set-url @@ -2368,7 +2371,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -f -a p complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -d 'set only the URL used for uploading' complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -f -a fetch complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -d 'set only the URL used for downloading' -complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2416,7 +2419,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a s complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache' complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed complete -c spack -n '__fish_spack_using_command mirror set' -l signed -d 'require signing and signature verification when pushing and installing from this build cache' -complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2447,7 +2450,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password-var set -g __fish_spack_optspecs_spack_mirror_list h/help scope= complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'configuration scope to read from' # spack module @@ -2703,6 +2706,11 @@ complete -c spack -n '__fish_spack_using_command python' -s m -r -d 'run library complete -c spack -n '__fish_spack_using_command python' -l path -f -a show_path complete -c spack -n '__fish_spack_using_command python' -l path -d 'show path to python interpreter that spack uses' +# spack read-only-core +set -g __fish_spack_optspecs_spack_read_only_core h/help +complete -c spack -n '__fish_spack_using_command read-only-core' -s h -l help -f -a help +complete -c spack -n '__fish_spack_using_command read-only-core' -s h -l help -d 'show this help message and exit' + # spack reindex set -g __fish_spack_optspecs_spack_reindex h/help complete -c spack -n '__fish_spack_using_command reindex' -s h -l help -f -a help @@ -2754,7 +2762,7 @@ complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirecto set -g __fish_spack_optspecs_spack_repo_list h/help scope= complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'configuration scope to read from' # spack repo add @@ -2762,7 +2770,7 @@ set -g __fish_spack_optspecs_spack_repo_add h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo add' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -d 'configuration scope to modify' # spack repo remove @@ -2770,7 +2778,7 @@ set -g __fish_spack_optspecs_spack_repo_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo remove' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -d 'configuration scope to modify' # spack repo rm @@ -2778,7 +2786,7 @@ set -g __fish_spack_optspecs_spack_repo_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo rm' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system admin-customer site install user command_line' +complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify' # spack resource From b65ba57b1c4497a0e3772e1189eff9192263e12e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 6 Jan 2025 21:16:57 -0800 Subject: [PATCH 050/506] move default config scope into per_user_spack, not install-tree-based; removes need for install scheme --- lib/spack/spack/cmd/read_only_core.py | 20 ------- lib/spack/spack/config.py | 4 +- lib/spack/spack/install_scheme.py | 76 --------------------------- lib/spack/spack/main.py | 10 ---- lib/spack/spack/paths.py | 3 ++ lib/spack/spack/store.py | 4 +- 6 files changed, 7 insertions(+), 110 deletions(-) delete mode 100644 lib/spack/spack/cmd/read_only_core.py diff --git a/lib/spack/spack/cmd/read_only_core.py b/lib/spack/spack/cmd/read_only_core.py deleted file mode 100644 index 657b212d63652e..00000000000000 --- a/lib/spack/spack/cmd/read_only_core.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright Spack Project Developers. See COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) -from llnl.util.filesystem import mkdirp - -import spack.install_scheme - -description = "Update Spack to stop writing into its own prefix" -section = "build" -level = "long" - - -def setup_parser(subparser): - pass - - -def read_only_core(parser, args): - config_dir = spack.install_scheme.config(for_alias="out-of-spack") - - mkdirp(config_dir) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 564d0f32d59b0a..a26a8cb30a0c5b 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -39,7 +39,6 @@ from llnl.util import filesystem, lang, tty import spack.error -import spack.install_scheme import spack.paths import spack.platforms import spack.schema @@ -847,7 +846,8 @@ def create() -> Configuration: if not disable_local_config: configuration_paths.append(("user", spack.paths.user_config_path)) - configuration_paths.append(("install", spack.install_scheme.config())) + per_spack_cfg = os.path.join(spack.paths.per_spack_user_root, "config") + configuration_paths.append(("per-spack-user", per_spack_cfg)) # add each scope and its platform-specific directory for name, path in configuration_paths: diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/install_scheme.py index 127d1ee36a84a6..b55eff81c0a6b2 100644 --- a/lib/spack/spack/install_scheme.py +++ b/lib/spack/spack/install_scheme.py @@ -19,82 +19,6 @@ import spack.error import spack.paths as paths -_scheme = None - -_in_spack = os.path.join(paths.prefix, "opt", "spack") - -_in_user = os.path.join(paths.per_spack_user_root, "installs") - - -class InstallScheme: - IN_SPACK = 1 - OUT_OF_SPACK = 2 - - -aliases = {"in-spack": InstallScheme.IN_SPACK, "out-of-spack": InstallScheme.OUT_OF_SPACK} - - -def set_scheme(alias): - global _scheme - if alias in aliases: - _scheme = aliases[alias] - else: - raise InstallSchemeError(f"Unknown install scheme: {alias}") - - -def scheme(): - return _determine_scheme() - - -def _determine_scheme(): - """Automatically choose an install scheme if it has not been set.""" - global _scheme - if _scheme: - pass - elif paths.dir_is_occupied(_in_user): - _scheme = InstallScheme.OUT_OF_SPACK - elif not os.access(paths.prefix, os.W_OK): - # If we can't write into the Spack prefix we use ~ - _scheme = InstallScheme.OUT_OF_SPACK - else: - # In the future, we may only choose the in-spack - # scheme if that install tree is populated, but for - # now even new instances of Spack should install - # into the Spack prefix - _scheme = InstallScheme.IN_SPACK - return _scheme - - -def default_install_location(for_alias=None): - """ - Selecting install-tree - - - if user specifies --install-scheme, we use that - - if there is config or installs in $per_spack user, use that - - otherwise, we are managing installs inside of Spack - - ... unless the Spack prefix is not writable by us - """ - if for_alias: - chosen = aliases.get(for_alias) - else: - chosen = scheme() - if chosen == InstallScheme.OUT_OF_SPACK: - return _in_user - elif chosen == InstallScheme.IN_SPACK: - return _in_spack - else: - raise InstallSchemeError(f"Unset or unexpected scheme: {chosen}") - - -def config(for_alias=None): - root = default_install_location(for_alias) - cfgs = os.path.join(root, "configs") - return cfgs - - -class InstallSchemeError(spack.error.SpackError): - pass - def _most_recent_internal_call(): """If called within an audit for a Python library function, finds diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 8dfda64e280546..16d1c173aa6669 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -500,13 +500,6 @@ def make_argument_parser(**kwargs): parser.add_argument( "--print-shell-vars", action="store", help="print info needed by setup-env.*sh" ) - parser.add_argument( - "--install-scheme", - dest="install_scheme", - action="store", - default=None, - help="specify install scheme", - ) parser.add_argument( "--disable-end-user-config", action="store_true", @@ -587,9 +580,6 @@ def setup_main_options(args): if args.color is not None: color.set_color_when(args.color) - if args.install_scheme: - spack.install_scheme.set_scheme(args.install_scheme) - if args.disable_end_user_config: spack.config.end_user_system_scope = False diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index e2def50cd67812..6b3eb1a5431c7b 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -81,6 +81,9 @@ def dir_is_occupied(x, except_for=None): var_path = os.path.join(prefix, "var", "spack") +internal_install_tree_root = os.path.join(prefix, "opt", "spack") + + def user_root(): """Default install tree and config scope. diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 1cb55dbf3259aa..fb8c7bcbf837e9 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -30,7 +30,7 @@ import spack.database import spack.directory_layout import spack.error -import spack.install_scheme +import spack.paths import spack.spec import spack.util.path @@ -78,7 +78,7 @@ def parse_install_tree(config_dict): projections = {"all": all_projection} else: - unpadded_root = install_tree.get("root", spack.install_scheme.default_install_location()) + unpadded_root = install_tree.get("root", spack.paths.internal_install_tree_root) unpadded_root = spack.util.path.canonicalize_path(unpadded_root) padded_length = install_tree.get("padded_length", False) From cc62ec405a1be310617c51081a6160aea86ba4ca Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 6 Jan 2025 21:20:27 -0800 Subject: [PATCH 051/506] style and command completion --- lib/spack/spack/install_scheme.py | 11 ------ share/spack/spack-completion.bash | 8 ++--- share/spack/spack-completion.fish | 60 ++++++++++++++----------------- 3 files changed, 28 insertions(+), 51 deletions(-) diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/install_scheme.py index b55eff81c0a6b2..3db273cebe0305 100644 --- a/lib/spack/spack/install_scheme.py +++ b/lib/spack/spack/install_scheme.py @@ -1,22 +1,11 @@ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - -""" -This module tracks whether we are installing packages into the Spack -prefix or outside of it. If we are installing outside of Spack, that -is controlled by ``$per_spack_user/installs/config``. By default, -when installing outside of Spack, packages are installed into -``$per_spack_user``; you can change that with -``config:install_tree:root``. -""" - import inspect import os import pathlib import warnings -import spack.error import spack.paths as paths diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 334a1319810848..370e33487a8e43 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -398,9 +398,9 @@ SPACK_ALIASES="concretise:concretize;containerise:containerize;rm:remove" _spack() { if $list_options then - SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -b --bootstrap -p --profile --sorted-profile --lines -v --verbose --stacktrace -t --backtrace -V --version --print-shell-vars --install-scheme --disable-end-user-config --guard-writes-into-spack" + SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -b --bootstrap -p --profile --sorted-profile --lines -v --verbose --stacktrace -t --backtrace -V --version --print-shell-vars --disable-end-user-config --guard-writes-into-spack" else - SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean clone commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse logs maintainers make-installer mark mirror module patch pkg providers pydoc python read-only-core reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" + SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean clone commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse logs maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" fi } @@ -1739,10 +1739,6 @@ _spack_python() { fi } -_spack_read_only_core() { - SPACK_COMPREPLY="-h --help" -} - _spack_reindex() { SPACK_COMPREPLY="-h --help" } diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 0e88e6e6db865c..85d1a7ff91587a 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -346,7 +346,7 @@ complete -c spack --erase # Everything below here is auto-generated. # spack -set -g __fish_spack_optspecs_spack h/help H/all-help color= c/config= C/config-scope= d/debug timestamp pdb e/env= D/env-dir= E/no-env use-env-repo k/insecure l/enable-locks L/disable-locks m/mock b/bootstrap p/profile sorted-profile= lines= v/verbose stacktrace t/backtrace V/version print-shell-vars= install-scheme= disable-end-user-config guard-writes-into-spack +set -g __fish_spack_optspecs_spack h/help H/all-help color= c/config= C/config-scope= d/debug timestamp pdb e/env= D/env-dir= E/no-env use-env-repo k/insecure l/enable-locks L/disable-locks m/mock b/bootstrap p/profile sorted-profile= lines= v/verbose stacktrace t/backtrace V/version print-shell-vars= disable-end-user-config guard-writes-into-spack complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a add -d 'add a spec to an environment' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a arch -d 'print architecture information about this machine' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a audit -d 'audit configuration files, packages, etc.' @@ -406,7 +406,6 @@ complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a pkg -d 'query pac complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a providers -d 'list packages that provide a particular virtual package' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a pydoc -d 'run pydoc from within spack' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a python -d 'launch an interpreter as spack would launch a command' -complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a read-only-core -d 'Update Spack to stop writing into its own prefix' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a reindex -d 'rebuild Spack'"'"'s package database' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a remove -d 'remove specs from an environment' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a rm -d 'remove specs from an environment' @@ -479,8 +478,6 @@ complete -c spack -n '__fish_spack_using_command ' -s V -l version -f -a version complete -c spack -n '__fish_spack_using_command ' -s V -l version -d 'show version number and exit' complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -f -a print_shell_vars complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -d 'print info needed by setup-env.*sh' -complete -c spack -n '__fish_spack_using_command ' -l install-scheme -r -f -a install_scheme -complete -c spack -n '__fish_spack_using_command ' -l install-scheme -r -d 'specify install scheme' complete -c spack -n '__fish_spack_using_command ' -l disable-end-user-config -f -a disable_end_user_config complete -c spack -n '__fish_spack_using_command ' -l disable-end-user-config -d 'Disable system config scope for end users' complete -c spack -n '__fish_spack_using_command ' -l guard-writes-into-spack -f -a guard_writes_into_spack @@ -609,7 +606,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_enable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap enable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap disable @@ -617,7 +614,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_disable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap disable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap reset @@ -632,14 +629,14 @@ set -g __fish_spack_optspecs_spack_bootstrap_root h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap root' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap list set -g __fish_spack_optspecs_spack_bootstrap_list h/help scope= complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap add @@ -648,7 +645,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap add' -f -a '(__ complete -c spack -n '__fish_spack_using_command_pos 1 bootstrap add' -f -a '(__fish_spack_environments)' complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -f -a trust complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -d 'enable the source immediately upon addition' @@ -826,7 +823,7 @@ complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirro complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -d 'override any configured mirrors with this mirror URL' complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -f -a output_file complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -d 'file where rebuild info should be written' -complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -d 'configuration scope containing mirrors to check' complete -c spack -n '__fish_spack_using_command buildcache check' -s s -l spec -r -f -a spec complete -c spack -n '__fish_spack_using_command buildcache check' -s s -l spec -r -d 'check single spec instead of release specs file' @@ -1085,7 +1082,7 @@ complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolcha complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolchain -d 'Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -d 'Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1099,7 +1096,7 @@ complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchai complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchain -d 'Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -d 'Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1111,7 +1108,7 @@ complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help - complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -d 'configuration scope to modify' # spack compiler rm @@ -1121,14 +1118,14 @@ complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -d 'configuration scope to modify' # spack compiler list set -g __fish_spack_optspecs_spack_compiler_list h/help scope= complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -d 'configuration scope to read from' # spack compiler info @@ -1136,14 +1133,14 @@ set -g __fish_spack_optspecs_spack_compiler_info h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 compiler info' -f -a '(__fish_spack_installed_compilers)' complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -d 'configuration scope to read from' # spack compilers set -g __fish_spack_optspecs_spack_compilers h/help scope= complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -d 'configuration scope to read/modify' # spack concretize @@ -1203,7 +1200,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a update -d ' complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a revert -d 'revert configuration files to their state before update' complete -c spack -n '__fish_spack_using_command config' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify' # spack config get @@ -1744,7 +1741,7 @@ complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -f complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -d 'packages to exclude from search' complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -f -a path complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -d 'one or more alternative search paths for finding externals' -complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command external find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command external find' -l all -f -a all complete -c spack -n '__fish_spack_using_command external find' -l all -d 'search for all packages that Spack knows about' @@ -2311,7 +2308,7 @@ set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsig complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use `--type binary --type source` (default)' @@ -2351,7 +2348,7 @@ set -g __fish_spack_optspecs_spack_mirror_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror remove' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -d 'configuration scope to modify' # spack mirror rm @@ -2359,7 +2356,7 @@ set -g __fish_spack_optspecs_spack_mirror_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror rm' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -d 'configuration scope to modify' # spack mirror set-url @@ -2371,7 +2368,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -f -a p complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -d 'set only the URL used for uploading' complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -f -a fetch complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -d 'set only the URL used for downloading' -complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2419,7 +2416,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a s complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache' complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed complete -c spack -n '__fish_spack_using_command mirror set' -l signed -d 'require signing and signature verification when pushing and installing from this build cache' -complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2450,7 +2447,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password-var set -g __fish_spack_optspecs_spack_mirror_list h/help scope= complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'configuration scope to read from' # spack module @@ -2706,11 +2703,6 @@ complete -c spack -n '__fish_spack_using_command python' -s m -r -d 'run library complete -c spack -n '__fish_spack_using_command python' -l path -f -a show_path complete -c spack -n '__fish_spack_using_command python' -l path -d 'show path to python interpreter that spack uses' -# spack read-only-core -set -g __fish_spack_optspecs_spack_read_only_core h/help -complete -c spack -n '__fish_spack_using_command read-only-core' -s h -l help -f -a help -complete -c spack -n '__fish_spack_using_command read-only-core' -s h -l help -d 'show this help message and exit' - # spack reindex set -g __fish_spack_optspecs_spack_reindex h/help complete -c spack -n '__fish_spack_using_command reindex' -s h -l help -f -a help @@ -2762,7 +2754,7 @@ complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirecto set -g __fish_spack_optspecs_spack_repo_list h/help scope= complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'configuration scope to read from' # spack repo add @@ -2770,7 +2762,7 @@ set -g __fish_spack_optspecs_spack_repo_add h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo add' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -d 'configuration scope to modify' # spack repo remove @@ -2778,7 +2770,7 @@ set -g __fish_spack_optspecs_spack_repo_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo remove' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -d 'configuration scope to modify' # spack repo rm @@ -2786,7 +2778,7 @@ set -g __fish_spack_optspecs_spack_repo_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo rm' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system admin-customer site user install command_line' +complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify' # spack resource From d962c893215cef2132772dac2edbc7c695ffd346 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 6 Jan 2025 21:24:26 -0800 Subject: [PATCH 052/506] default install location determined by store --- lib/spack/spack/store.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index fb8c7bcbf837e9..958c433ccc7d14 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -78,7 +78,7 @@ def parse_install_tree(config_dict): projections = {"all": all_projection} else: - unpadded_root = install_tree.get("root", spack.paths.internal_install_tree_root) + unpadded_root = install_tree.get("root", _default_install_location()) unpadded_root = spack.util.path.canonicalize_path(unpadded_root) padded_length = install_tree.get("padded_length", False) @@ -125,6 +125,13 @@ def parse_install_tree(config_dict): return root, unpadded_root, projections +def _default_install_location(): + if os.access(spack.paths.prefix, os.W_OK): + return spack.paths.internal_install_tree_root + else: + return os.path.join(spack.paths.per_spack_user_root, "installs") + + class Store: """A store is a path full of installed Spack packages. From 4ed360fcf3578221291e8101dacf4e25353b3744 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Jan 2025 10:39:55 -0800 Subject: [PATCH 053/506] change name of end-user scope for consistency --- lib/spack/spack/config.py | 2 +- lib/spack/spack/paths.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index a26a8cb30a0c5b..47dbb191a5a86d 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -832,7 +832,7 @@ def create() -> Configuration: if not disable_local_config: configuration_paths.append(("system", spack.paths.system_config_path)) if end_user_system_scope: - configuration_paths.append(("admin-customer", spack.paths.admin_customer_cfg)) + configuration_paths.append(("end-user", spack.paths.end_user_cfg_path)) # Site configuration is per spack instance, for sites or projects # No site-level configs should be checked into spack by default. diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 6b3eb1a5431c7b..759a5272823b16 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -48,7 +48,7 @@ def dir_is_occupied(x, except_for=None): #: When Spack is provided by an admin to a user, the admin can #: provide a config that only applies for the end-users -admin_customer_cfg = os.path.join(system_config_path, "admin-customer") +end_user_cfg_path = os.path.join(system_config_path, "end-user") #: synonym for prefix spack_root = prefix From d5cc39b72d50ecac27db3fe7b6994453dd83bfdc Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Jan 2025 11:46:46 -0800 Subject: [PATCH 054/506] site admin scope --- lib/spack/spack/config.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 47dbb191a5a86d..9dfe8f1787d580 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -836,7 +836,23 @@ def create() -> Configuration: # Site configuration is per spack instance, for sites or projects # No site-level configs should be checked into spack by default. - configuration_paths.append(("site", os.path.join(spack.paths.etc_path))) + configuration_paths.append(("site", spack.paths.etc_path)) + + # Site admin scope has two uses: (a) admins can share config with one + # another, but not with end users (b) pip/apt-installed spack can + # change the default install root + site_admin_path = os.path.join(spack.paths.etc_path, "site-admin") + site_admin_accessible = False + try: + if os.path.isdir(site_admin_path): + os.listdir(site_admin_path) + site_admin_accessible = True + except PermissionError: + pass + if site_admin_accessible: + # TODO: this does not need to be platform specific, and it should + # not be writable + configuration_paths.append(("site-admin", site_admin_path)) # Python package's can register configuration scopes via entry_points configuration_paths.extend(config_paths_from_entry_points()) From 674a90690963676806013f0b23232b2350a592ae Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Jan 2025 12:30:38 -0800 Subject: [PATCH 055/506] update command completion --- share/spack/spack-completion.fish | 50 +++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 85d1a7ff91587a..7c941e2d8ac046 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -606,7 +606,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_enable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap enable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap disable @@ -614,7 +614,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_disable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap disable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap reset @@ -629,14 +629,14 @@ set -g __fish_spack_optspecs_spack_bootstrap_root h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap root' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap list set -g __fish_spack_optspecs_spack_bootstrap_list h/help scope= complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap add @@ -645,7 +645,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap add' -f -a '(__ complete -c spack -n '__fish_spack_using_command_pos 1 bootstrap add' -f -a '(__fish_spack_environments)' complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -f -a trust complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -d 'enable the source immediately upon addition' @@ -823,7 +823,7 @@ complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirro complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -d 'override any configured mirrors with this mirror URL' complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -f -a output_file complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -d 'file where rebuild info should be written' -complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -d 'configuration scope containing mirrors to check' complete -c spack -n '__fish_spack_using_command buildcache check' -s s -l spec -r -f -a spec complete -c spack -n '__fish_spack_using_command buildcache check' -s s -l spec -r -d 'check single spec instead of release specs file' @@ -1082,7 +1082,7 @@ complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolcha complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolchain -d 'Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -d 'Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1096,7 +1096,7 @@ complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchai complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchain -d 'Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -d 'Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1108,7 +1108,7 @@ complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help - complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -d 'configuration scope to modify' # spack compiler rm @@ -1118,14 +1118,14 @@ complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -d 'configuration scope to modify' # spack compiler list set -g __fish_spack_optspecs_spack_compiler_list h/help scope= complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -d 'configuration scope to read from' # spack compiler info @@ -1133,14 +1133,14 @@ set -g __fish_spack_optspecs_spack_compiler_info h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 compiler info' -f -a '(__fish_spack_installed_compilers)' complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -d 'configuration scope to read from' # spack compilers set -g __fish_spack_optspecs_spack_compilers h/help scope= complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -d 'configuration scope to read/modify' # spack concretize @@ -1200,7 +1200,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a update -d ' complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a revert -d 'revert configuration files to their state before update' complete -c spack -n '__fish_spack_using_command config' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify' # spack config get @@ -1741,7 +1741,7 @@ complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -f complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -d 'packages to exclude from search' complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -f -a path complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -d 'one or more alternative search paths for finding externals' -complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command external find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command external find' -l all -f -a all complete -c spack -n '__fish_spack_using_command external find' -l all -d 'search for all packages that Spack knows about' @@ -2308,7 +2308,7 @@ set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsig complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use `--type binary --type source` (default)' @@ -2348,7 +2348,7 @@ set -g __fish_spack_optspecs_spack_mirror_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror remove' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -d 'configuration scope to modify' # spack mirror rm @@ -2356,7 +2356,7 @@ set -g __fish_spack_optspecs_spack_mirror_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror rm' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -d 'configuration scope to modify' # spack mirror set-url @@ -2368,7 +2368,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -f -a p complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -d 'set only the URL used for uploading' complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -f -a fetch complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -d 'set only the URL used for downloading' -complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2416,7 +2416,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a s complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache' complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed complete -c spack -n '__fish_spack_using_command mirror set' -l signed -d 'require signing and signature verification when pushing and installing from this build cache' -complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2447,7 +2447,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password-var set -g __fish_spack_optspecs_spack_mirror_list h/help scope= complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'configuration scope to read from' # spack module @@ -2754,7 +2754,7 @@ complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirecto set -g __fish_spack_optspecs_spack_repo_list h/help scope= complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'configuration scope to read from' # spack repo add @@ -2762,7 +2762,7 @@ set -g __fish_spack_optspecs_spack_repo_add h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo add' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -d 'configuration scope to modify' # spack repo remove @@ -2770,7 +2770,7 @@ set -g __fish_spack_optspecs_spack_repo_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo remove' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -d 'configuration scope to modify' # spack repo rm @@ -2778,7 +2778,7 @@ set -g __fish_spack_optspecs_spack_repo_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo rm' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system admin-customer site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify' # spack resource From d4dd578c63879a67071c7e68bffa9294c8c1878c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Jan 2025 14:06:50 -0800 Subject: [PATCH 056/506] ci is insisting I havent run command completion, I want to see where it diffs --- lib/spack/spack/test/cmd/commands.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index 2518c9b81fc72b..294a62905efe92 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -273,4 +273,12 @@ def test_updated_completion_scripts(shell, tmpdir): commands("--aliases", "--format", shell, "--header", header, "--update", new_script) - assert filecmp.cmp(old_script, new_script), msg + if not filecmp.cmp(old_script, new_script): + import difflib + + with open(old_script, "r") as f1, open(new_script, "r") as f2: + l1 = f1.readlines() + l2 = f2.readlines() + diff = difflib.unified_diff(l1, l2, fromfile=old_script, tofile=new_script) + msg = "Diff failure:\n\n" + "".join(diff) + assert False, msg From dbaaaf0fa4742a8a60f72787976fc20633b2bf26 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Jan 2025 14:18:37 -0800 Subject: [PATCH 057/506] style precheck was stopping my unit test debugging --- lib/spack/spack/test/cmd/commands.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index 294a62905efe92..83ce59db435728 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -276,7 +276,9 @@ def test_updated_completion_scripts(shell, tmpdir): if not filecmp.cmp(old_script, new_script): import difflib - with open(old_script, "r") as f1, open(new_script, "r") as f2: + with open(old_script, "r", encoding="utf-8") as f1, open( + new_script, "r", encoding="utf-8" + ) as f2: l1 = f1.readlines() l2 = f2.readlines() diff = difflib.unified_diff(l1, l2, fromfile=old_script, tofile=new_script) From 8efb24ef8ef1d9502c27b943798c205fdeec6c2a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Jan 2025 14:45:55 -0800 Subject: [PATCH 058/506] conditional prefix-internal scope is confusing command completion --- share/spack/spack-completion.fish | 50 +++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 7c941e2d8ac046..1b74b8a329d849 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -606,7 +606,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_enable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap enable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap disable @@ -614,7 +614,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_disable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap disable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap reset @@ -629,14 +629,14 @@ set -g __fish_spack_optspecs_spack_bootstrap_root h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap root' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap list set -g __fish_spack_optspecs_spack_bootstrap_list h/help scope= complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap add @@ -645,7 +645,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap add' -f -a '(__ complete -c spack -n '__fish_spack_using_command_pos 1 bootstrap add' -f -a '(__fish_spack_environments)' complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -f -a trust complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -d 'enable the source immediately upon addition' @@ -823,7 +823,7 @@ complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirro complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -d 'override any configured mirrors with this mirror URL' complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -f -a output_file complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -d 'file where rebuild info should be written' -complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -d 'configuration scope containing mirrors to check' complete -c spack -n '__fish_spack_using_command buildcache check' -s s -l spec -r -f -a spec complete -c spack -n '__fish_spack_using_command buildcache check' -s s -l spec -r -d 'check single spec instead of release specs file' @@ -1082,7 +1082,7 @@ complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolcha complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolchain -d 'Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -d 'Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1096,7 +1096,7 @@ complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchai complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchain -d 'Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -d 'Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1108,7 +1108,7 @@ complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help - complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -d 'configuration scope to modify' # spack compiler rm @@ -1118,14 +1118,14 @@ complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -d 'configuration scope to modify' # spack compiler list set -g __fish_spack_optspecs_spack_compiler_list h/help scope= complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -d 'configuration scope to read from' # spack compiler info @@ -1133,14 +1133,14 @@ set -g __fish_spack_optspecs_spack_compiler_info h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 compiler info' -f -a '(__fish_spack_installed_compilers)' complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -d 'configuration scope to read from' # spack compilers set -g __fish_spack_optspecs_spack_compilers h/help scope= complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -d 'configuration scope to read/modify' # spack concretize @@ -1200,7 +1200,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a update -d ' complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a revert -d 'revert configuration files to their state before update' complete -c spack -n '__fish_spack_using_command config' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify' # spack config get @@ -1741,7 +1741,7 @@ complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -f complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -d 'packages to exclude from search' complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -f -a path complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -d 'one or more alternative search paths for finding externals' -complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command external find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command external find' -l all -f -a all complete -c spack -n '__fish_spack_using_command external find' -l all -d 'search for all packages that Spack knows about' @@ -2308,7 +2308,7 @@ set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsig complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use `--type binary --type source` (default)' @@ -2348,7 +2348,7 @@ set -g __fish_spack_optspecs_spack_mirror_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror remove' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -d 'configuration scope to modify' # spack mirror rm @@ -2356,7 +2356,7 @@ set -g __fish_spack_optspecs_spack_mirror_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror rm' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -d 'configuration scope to modify' # spack mirror set-url @@ -2368,7 +2368,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -f -a p complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -d 'set only the URL used for uploading' complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -f -a fetch complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -d 'set only the URL used for downloading' -complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2416,7 +2416,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a s complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache' complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed complete -c spack -n '__fish_spack_using_command mirror set' -l signed -d 'require signing and signature verification when pushing and installing from this build cache' -complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2447,7 +2447,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password-var set -g __fish_spack_optspecs_spack_mirror_list h/help scope= complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'configuration scope to read from' # spack module @@ -2754,7 +2754,7 @@ complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirecto set -g __fish_spack_optspecs_spack_repo_list h/help scope= complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'configuration scope to read from' # spack repo add @@ -2762,7 +2762,7 @@ set -g __fish_spack_optspecs_spack_repo_add h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo add' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -d 'configuration scope to modify' # spack repo remove @@ -2770,7 +2770,7 @@ set -g __fish_spack_optspecs_spack_repo_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo remove' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -d 'configuration scope to modify' # spack repo rm @@ -2778,7 +2778,7 @@ set -g __fish_spack_optspecs_spack_repo_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo rm' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system end-user site site-admin user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify' # spack resource From 372073033d59cba7d4472d7274f8ad32656d2bee Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Jan 2025 14:48:21 -0800 Subject: [PATCH 059/506] add explanatory comment --- lib/spack/spack/test/cmd/commands.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index 83ce59db435728..cec2885171933a 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -273,6 +273,10 @@ def test_updated_completion_scripts(shell, tmpdir): commands("--aliases", "--format", shell, "--header", header, "--update", new_script) + # TODO: changes in this PR add a site-admin config scope. If one runs + # `spack commands --update-completion` when this scope is available, it + # will add the site-admin scope to the autocompletion scripts, and cause + # a discrepancy with the runner if not filecmp.cmp(old_script, new_script): import difflib From a6fa89db1387290ef72410ca9b0b390b7568a592 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Jan 2025 17:32:21 -0800 Subject: [PATCH 060/506] rename module given its reduced scope --- lib/spack/spack/main.py | 4 ++-- lib/spack/spack/{install_scheme.py => trace.py} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename lib/spack/spack/{install_scheme.py => trace.py} (100%) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 16d1c173aa6669..83380a0797d967 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -38,13 +38,13 @@ import spack.config import spack.environment as ev import spack.error -import spack.install_scheme import spack.modules import spack.paths import spack.platforms import spack.repo import spack.spec import spack.store +import spack.trace import spack.util.debug import spack.util.environment import spack.util.lock @@ -996,7 +996,7 @@ def _main(argv=None): cmd_name, args.command = resolve_alias(cmd_name, args.command) if args.guard_writes_into_spack: - spack.install_scheme.guard_writes_into_spack() + spack.trace.guard_writes_into_spack() # set up a bootstrap context, if asked. # bootstrap context needs to include parsing the command, b/c things diff --git a/lib/spack/spack/install_scheme.py b/lib/spack/spack/trace.py similarity index 100% rename from lib/spack/spack/install_scheme.py rename to lib/spack/spack/trace.py From 39d994c5ab05b2e51c9c962f0526597d08f2cb08 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Jan 2025 17:36:54 -0800 Subject: [PATCH 061/506] update option name --- lib/spack/spack/main.py | 6 +++--- lib/spack/spack/trace.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 83380a0797d967..a7219824128cce 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -506,7 +506,7 @@ def make_argument_parser(**kwargs): help="Disable system config scope for end users", ) parser.add_argument( - "--guard-writes-into-spack", + "--warn-writes-into-spack", action="store_true", help="Warn when Spack tries to write into its own prefix", ) @@ -995,8 +995,8 @@ def _main(argv=None): cmd_name = args.command[0] cmd_name, args.command = resolve_alias(cmd_name, args.command) - if args.guard_writes_into_spack: - spack.trace.guard_writes_into_spack() + if args.warn_writes_into_spack: + spack.trace.warn_writes_into_spack() # set up a bootstrap context, if asked. # bootstrap context needs to include parsing the command, b/c things diff --git a/lib/spack/spack/trace.py b/lib/spack/spack/trace.py index 3db273cebe0305..18acadc227cfcb 100644 --- a/lib/spack/spack/trace.py +++ b/lib/spack/spack/trace.py @@ -72,7 +72,7 @@ def _guard_writes(event, args): _attempted_modify_internal(f"mkdir {abs_path}") -def guard_writes_into_spack(): +def warn_writes_into_spack(): import sys if sys.version_info[:2] >= (3, 8): From 9b44e2e3e642eec953e2a890e713e48179cc0241 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Jan 2025 17:43:09 -0800 Subject: [PATCH 062/506] avoid silently doing nothing --- lib/spack/spack/trace.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/spack/spack/trace.py b/lib/spack/spack/trace.py index 18acadc227cfcb..d78b3425916652 100644 --- a/lib/spack/spack/trace.py +++ b/lib/spack/spack/trace.py @@ -77,3 +77,5 @@ def warn_writes_into_spack(): if sys.version_info[:2] >= (3, 8): sys.addaudithook(_guard_writes) # novermin + else: + raise ValueError(f"Cannot detect writes for Python {sys.version_info[:2]}") From 0c5775f3ff06ac2c84eacdfe8f2bd2a38d020605 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Jan 2025 18:14:19 -0800 Subject: [PATCH 063/506] update command completion --- share/spack/spack-completion.bash | 2 +- share/spack/spack-completion.fish | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 370e33487a8e43..944f0b49109cc4 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -398,7 +398,7 @@ SPACK_ALIASES="concretise:concretize;containerise:containerize;rm:remove" _spack() { if $list_options then - SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -b --bootstrap -p --profile --sorted-profile --lines -v --verbose --stacktrace -t --backtrace -V --version --print-shell-vars --disable-end-user-config --guard-writes-into-spack" + SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -b --bootstrap -p --profile --sorted-profile --lines -v --verbose --stacktrace -t --backtrace -V --version --print-shell-vars --disable-end-user-config --warn-writes-into-spack" else SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean clone commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse logs maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" fi diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 1b74b8a329d849..4dc2aa6f65399d 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -346,7 +346,7 @@ complete -c spack --erase # Everything below here is auto-generated. # spack -set -g __fish_spack_optspecs_spack h/help H/all-help color= c/config= C/config-scope= d/debug timestamp pdb e/env= D/env-dir= E/no-env use-env-repo k/insecure l/enable-locks L/disable-locks m/mock b/bootstrap p/profile sorted-profile= lines= v/verbose stacktrace t/backtrace V/version print-shell-vars= disable-end-user-config guard-writes-into-spack +set -g __fish_spack_optspecs_spack h/help H/all-help color= c/config= C/config-scope= d/debug timestamp pdb e/env= D/env-dir= E/no-env use-env-repo k/insecure l/enable-locks L/disable-locks m/mock b/bootstrap p/profile sorted-profile= lines= v/verbose stacktrace t/backtrace V/version print-shell-vars= disable-end-user-config warn-writes-into-spack complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a add -d 'add a spec to an environment' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a arch -d 'print architecture information about this machine' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a audit -d 'audit configuration files, packages, etc.' @@ -480,8 +480,8 @@ complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -f -a complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -d 'print info needed by setup-env.*sh' complete -c spack -n '__fish_spack_using_command ' -l disable-end-user-config -f -a disable_end_user_config complete -c spack -n '__fish_spack_using_command ' -l disable-end-user-config -d 'Disable system config scope for end users' -complete -c spack -n '__fish_spack_using_command ' -l guard-writes-into-spack -f -a guard_writes_into_spack -complete -c spack -n '__fish_spack_using_command ' -l guard-writes-into-spack -d 'Warn when Spack tries to write into its own prefix' +complete -c spack -n '__fish_spack_using_command ' -l warn-writes-into-spack -f -a warn_writes_into_spack +complete -c spack -n '__fish_spack_using_command ' -l warn-writes-into-spack -d 'Warn when Spack tries to write into its own prefix' # spack add set -g __fish_spack_optspecs_spack_add h/help l/list-name= From ebd5b40aafc864a4b8cc2b58026b5d0b72c1729f Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 18 Mar 2025 17:25:15 -0700 Subject: [PATCH 064/506] forgot to add attribute in merge --- lib/spack/spack/paths.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 759a5272823b16..75c37dfeeb7fa9 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -101,6 +101,9 @@ def user_root(): #: placed in per-spack-instance user root default_misc_cache_path = os.path.join(per_spack_user_root, "cache") +#: concretization cache for Spack concretizations +default_conc_cache_path = os.path.join(default_misc_cache_path, "concretization") + modules_base = None for module_dir in ["lmod", "modules"]: if dir_is_occupied(os.path.join(share_path, module_dir)): From e18303981fe069060ae787374a6f771679719831 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 28 Mar 2025 13:16:44 -0700 Subject: [PATCH 065/506] shell auto-completion --- share/spack/spack-completion.fish | 56 +++++++++++++++++-------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index b4d4f7932b77c7..87c992137b2f5d 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -346,7 +346,7 @@ complete -c spack --erase # Everything below here is auto-generated. # spack -set -g __fish_spack_optspecs_spack h/help H/all-help color= c/config= C/config-scope= d/debug timestamp pdb e/env= D/env-dir= E/no-env use-env-repo k/insecure l/enable-locks L/disable-locks m/mock b/bootstrap p/profile sorted-profile= lines= v/verbose stacktrace t/backtrace V/version print-shell-vars= +set -g __fish_spack_optspecs_spack h/help H/all-help color= c/config= C/config-scope= d/debug timestamp pdb e/env= D/env-dir= E/no-env use-env-repo k/insecure l/enable-locks L/disable-locks m/mock b/bootstrap p/profile sorted-profile= lines= v/verbose stacktrace t/backtrace V/version print-shell-vars= disable-end-user-config warn-writes-into-spack complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a add -d 'add a spec to an environment' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a arch -d 'print architecture information about this machine' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a audit -d 'audit configuration files, packages, etc.' @@ -478,6 +478,10 @@ complete -c spack -n '__fish_spack_using_command ' -s V -l version -f -a version complete -c spack -n '__fish_spack_using_command ' -s V -l version -d 'show version number and exit' complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -f -a print_shell_vars complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -d 'print info needed by setup-env.*sh' +complete -c spack -n '__fish_spack_using_command ' -l disable-end-user-config -f -a disable_end_user_config +complete -c spack -n '__fish_spack_using_command ' -l disable-end-user-config -d 'Disable system config scope for end users' +complete -c spack -n '__fish_spack_using_command ' -l warn-writes-into-spack -f -a warn_writes_into_spack +complete -c spack -n '__fish_spack_using_command ' -l warn-writes-into-spack -d 'Warn when Spack tries to write into its own prefix' # spack add set -g __fish_spack_optspecs_spack_add h/help l/list-name= @@ -602,7 +606,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_enable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap enable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap disable @@ -610,7 +614,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_disable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap disable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap reset @@ -625,14 +629,14 @@ set -g __fish_spack_optspecs_spack_bootstrap_root h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap root' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap list set -g __fish_spack_optspecs_spack_bootstrap_list h/help scope= complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap add @@ -641,7 +645,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap add' -f -a '(__ complete -c spack -n '__fish_spack_using_command_pos 1 bootstrap add' -f -a '(__fish_spack_environments)' complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -f -a trust complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -d 'enable the source immediately upon addition' @@ -819,7 +823,7 @@ complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirro complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -d 'override any configured mirrors with this mirror URL' complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -f -a output_file complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -d 'file where rebuild info should be written' -complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -d 'configuration scope containing mirrors to check' complete -c spack -n '__fish_spack_using_command buildcache check' -s s -l spec -r -f -a spec complete -c spack -n '__fish_spack_using_command buildcache check' -s s -l spec -r -d 'check single spec instead of release specs file' @@ -1087,7 +1091,7 @@ complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolcha complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolchain -d '(DEPRECATED) Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -d '(DEPRECATED) Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1101,7 +1105,7 @@ complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchai complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchain -d '(DEPRECATED) Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -d '(DEPRECATED) Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1113,7 +1117,7 @@ complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help - complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -d 'configuration scope to modify' # spack compiler rm @@ -1123,14 +1127,14 @@ complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -d 'configuration scope to modify' # spack compiler list set -g __fish_spack_optspecs_spack_compiler_list h/help scope= complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -d 'configuration scope to read from' # spack compiler info @@ -1138,14 +1142,14 @@ set -g __fish_spack_optspecs_spack_compiler_info h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 compiler info' -f -a '(__fish_spack_installed_compilers)' complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -d 'configuration scope to read from' # spack compilers set -g __fish_spack_optspecs_spack_compilers h/help scope= complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -d 'configuration scope to read/modify' # spack concretize @@ -1205,7 +1209,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a update -d ' complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a revert -d 'revert configuration files to their state before update' complete -c spack -n '__fish_spack_using_command config' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify' # spack config get @@ -1742,7 +1746,7 @@ complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -f complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -d 'packages to exclude from search' complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -f -a path complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -d 'one or more alternative search paths for finding externals' -complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command external find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command external find' -l all -f -a all complete -c spack -n '__fish_spack_using_command external find' -l all -d 'search for all packages that Spack knows about' @@ -2309,7 +2313,7 @@ set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsig complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use `--type binary --type source` (default)' @@ -2349,7 +2353,7 @@ set -g __fish_spack_optspecs_spack_mirror_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror remove' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -d 'configuration scope to modify' # spack mirror rm @@ -2357,7 +2361,7 @@ set -g __fish_spack_optspecs_spack_mirror_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror rm' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -d 'configuration scope to modify' # spack mirror set-url @@ -2369,7 +2373,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -f -a p complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -d 'set only the URL used for uploading' complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -f -a fetch complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -d 'set only the URL used for downloading' -complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2417,7 +2421,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a s complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache' complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed complete -c spack -n '__fish_spack_using_command mirror set' -l signed -d 'require signing and signature verification when pushing and installing from this build cache' -complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2448,7 +2452,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password-var set -g __fish_spack_optspecs_spack_mirror_list h/help scope= complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'configuration scope to read from' # spack module @@ -2755,7 +2759,7 @@ complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirecto set -g __fish_spack_optspecs_spack_repo_list h/help scope= complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'configuration scope to read from' # spack repo add @@ -2763,7 +2767,7 @@ set -g __fish_spack_optspecs_spack_repo_add h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo add' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -d 'configuration scope to modify' # spack repo remove @@ -2771,7 +2775,7 @@ set -g __fish_spack_optspecs_spack_repo_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo remove' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -d 'configuration scope to modify' # spack repo rm @@ -2779,7 +2783,7 @@ set -g __fish_spack_optspecs_spack_repo_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo rm' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify' # spack resource From 8036c9ececfc7b5c4ec6be4a93e73cf69e6e9b73 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 14 May 2025 15:42:04 -0700 Subject: [PATCH 066/506] partial xdg handling --- lib/spack/spack/config.py | 4 ++-- lib/spack/spack/paths.py | 39 ++++++++++++++++++++++----------------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index cc7f0b2337a767..0536890c89aaa9 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -985,8 +985,8 @@ def create_incremental() -> Generator[Configuration, None, None]: if not disable_local_config: configuration_paths.append(("user", spack.paths.user_config_path)) - per_spack_cfg = os.path.join(spack.paths.per_spack_user_root, "config") - configuration_paths.append(("per-spack-user", per_spack_cfg)) + per_spack_cfg = os.path.join(spack.paths.user_config_path, spack.paths.spack_instance_id) + configuration_paths.append(("this-spack-user", per_spack_cfg)) # add each scope and its platform-specific directory for name, path in configuration_paths: diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 432831724cbe27..6021c33a34d2b6 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -19,11 +19,28 @@ #: This file lives in $prefix/lib/spack/spack/__file__ prefix = str(PurePath(llnl.util.filesystem.ancestor(__file__, 4))) +xdg_config_home = "XDG_CONFIG_HOME" +xdg_state_home = "XDG_STATE_HOME" -# User configuration and caches in $HOME/.spack -# Override w/ `SPACK_USER_CONFIG_PATH` +if xdg_state_home in os.environ: + spack_xdg_state_home = os.path.join(os.environ[xdg_state_home], "spack") +else: + spack_xdg_state_home = os.path.join("~", ".local", "state", "spack") +spack_xdg_state_home = os.path.expanduser(spack_xdg_state_home) + +if xdg_config_home in os.environ: + spack_xdg_config_home = os.path.join(os.environ[xdg_config_home], "spack") +else: + spack_xdg_config_home = os.path.join("~", ".config", "spack") +spack_xdg_config_home = os.path.expanduser(spack_xdg_config_home) + +# xdg_data_home -- $HOME/.local/share + +# User configuration def _get_user_config_path(): - return os.path.expanduser(os.getenv("SPACK_USER_CONFIG_PATH") or "~%s.spack" % os.sep) + return os.path.expanduser( + os.getenv("SPACK_USER_CONFIG_PATH") or spack_xdg_config_home + ) # Configuration in /etc/spack on the system @@ -83,22 +100,10 @@ def dir_is_occupied(x, except_for=None): internal_install_tree_root = os.path.join(prefix, "opt", "spack") -def user_root(): - """Default install tree and config scope. - - Applies when $spack/opt is not an install tree. - - ~// - """ - spack_prefix = prefix - return pathlib.Path(user_config_path, hash.b32_hash(spack_prefix)[:7]) - - -per_spack_user_root = str(user_root()) +spack_instance_id = hash.b32_hash(prefix)[:7] #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) -#: placed in per-spack-instance user root -default_misc_cache_path = os.path.join(per_spack_user_root, "cache") +default_misc_cache_path = os.path.join(spack_xdg_state_home, spack_instance_id, "cache") #: concretization cache for Spack concretizations default_conc_cache_path = os.path.join(default_misc_cache_path, "concretization") From 29b9cc7e1807f4c7052a6cf9645827defbd0bec2 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 14 May 2025 16:38:29 -0700 Subject: [PATCH 067/506] envs move back into spack prefix (unless config:environments_root is set); downloads cache moves to XDG_DATA_HOME --- lib/spack/spack/paths.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 6021c33a34d2b6..2dd4f55428f393 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -21,20 +21,22 @@ xdg_config_home = "XDG_CONFIG_HOME" xdg_state_home = "XDG_STATE_HOME" +xdg_data_home = "XDG_DATA_HOME" -if xdg_state_home in os.environ: - spack_xdg_state_home = os.path.join(os.environ[xdg_state_home], "spack") -else: - spack_xdg_state_home = os.path.join("~", ".local", "state", "spack") -spack_xdg_state_home = os.path.expanduser(spack_xdg_state_home) -if xdg_config_home in os.environ: - spack_xdg_config_home = os.path.join(os.environ[xdg_config_home], "spack") -else: - spack_xdg_config_home = os.path.join("~", ".config", "spack") -spack_xdg_config_home = os.path.expanduser(spack_xdg_config_home) +def _define_xdg_or_backup(xdg_var, backup): + if xdg_var in os.environ: + spack_xdg_defined = os.path.join(xdg_var, "spack") + else: + spack_xdg_defined = os.path.join(backup, "spack") + return os.path.expanduser(spack_xdg_defined) + + +#: Resolved XDG_ counterparts, with additional "spack" subdirectory +spack_xdg_state_home = _define_xdg_or_backup(xdg_state_home, os.path.join("~", ".local", "state")) +spack_xdg_config_home = _define_xdg_or_backup(xdg_config_home, os.path.join("~", "config")) +spack_xdg_data_home = _define_xdg_or_backup(xdg_data_home, os.path.join("~", ".local", "share")) -# xdg_data_home -- $HOME/.local/share # User configuration def _get_user_config_path(): @@ -113,13 +115,11 @@ def dir_is_occupied(x, except_for=None): if dir_is_occupied(os.path.join(share_path, module_dir)): modules_base = share_path if not modules_base: - modules_base = os.path.join(per_spack_user_root, "modules") + modules_base = os.path.join(spack_xdg_data_home, "modules") -old_envs_path = os.path.join(var_path, "environments") -if dir_is_occupied(old_envs_path): - envs_path = old_envs_path -else: - envs_path = os.path.join(per_spack_user_root, "environments") +# Environments can store views and `develop` packages +# TODO: maybe store views/develop packages in a separate location? +envs_path = os.path.join(var_path, "environments") # TODO: we could shutil.mv resources from old paths to new paths @@ -144,7 +144,7 @@ def dir_is_occupied(x, except_for=None): # setting `SPACK_USER_CACHE_PATH`. Otherwise it defaults to ~/.spack. # def _get_user_cache_path(): - return os.path.expanduser(os.getenv("SPACK_USER_CACHE_PATH") or "~%s.spack" % os.sep) + return os.path.expanduser(os.getenv("SPACK_USER_CACHE_PATH") or spack_xdg_data_home) user_cache_path = str(PurePath(_get_user_cache_path())) From 883ce21d5d7d7d6f43a55b29b521ec4912dc5599 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 14 May 2025 16:57:38 -0700 Subject: [PATCH 068/506] put envs in XDG_DATA_HOME, if it's set --- lib/spack/spack/paths.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 2dd4f55428f393..e272f7a59e31c7 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -37,6 +37,11 @@ def _define_xdg_or_backup(xdg_var, backup): spack_xdg_config_home = _define_xdg_or_backup(xdg_config_home, os.path.join("~", "config")) spack_xdg_data_home = _define_xdg_or_backup(xdg_data_home, os.path.join("~", ".local", "share")) +if xdg_data_home in os.environ: + spack_xdg_data_home_nodefault = os.path.expanduser(os.path.join(os.environ[xdg_data_home], "spack")) +else: + spack_xdg_data_home_nodefault = None + # User configuration def _get_user_config_path(): @@ -117,9 +122,17 @@ def dir_is_occupied(x, except_for=None): if not modules_base: modules_base = os.path.join(spack_xdg_data_home, "modules") -# Environments can store views and `develop` packages -# TODO: maybe store views/develop packages in a separate location? -envs_path = os.path.join(var_path, "environments") +old_envs_path = os.path.join(var_path, "environments") +if dir_is_occupied(old_envs_path): + envs_path = old_envs_path +elif spack_xdg_data_home_nodefault: + envs_path = os.path.join(spack_xdg_data_home_nodefault, "environments") +else: + # Environments can store views and `develop` packages, which + # take up too much space for us to place them in ~ unless the + # user explicitly requests it + # TODO: maybe store views/develop packages in a separate location? + envs_path = old_envs_path # TODO: we could shutil.mv resources from old paths to new paths From 5ad0ae53f294d9936945dd5c9204d520511ccad4 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 14 May 2025 17:49:58 -0700 Subject: [PATCH 069/506] build stages go into XDG_CACHE_HOME as a backup; dont use default XDG_DATA_HOME for installs --- etc/spack/defaults/config.yaml | 2 +- lib/spack/spack/paths.py | 2 ++ lib/spack/spack/store.py | 5 +---- lib/spack/spack/util/path.py | 32 ++++++++++++++++---------------- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index b71ff12137b53b..d8514f92078ba8 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -67,7 +67,7 @@ config: # identifies Spack staging to avoid accidentally wiping out non-Spack work. build_stage: - $tempdir/$user/spack-stage - - $user_cache_path/stage + - $spack_xdg_cache_home/stage # - $spack/var/spack/stage # Directory in which to run tests and store test results. diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index e272f7a59e31c7..12010ae0cf17c9 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -22,6 +22,7 @@ xdg_config_home = "XDG_CONFIG_HOME" xdg_state_home = "XDG_STATE_HOME" xdg_data_home = "XDG_DATA_HOME" +xdg_cache_home = "XDG_CACHE_HOME" def _define_xdg_or_backup(xdg_var, backup): @@ -42,6 +43,7 @@ def _define_xdg_or_backup(xdg_var, backup): else: spack_xdg_data_home_nodefault = None +spack_xdg_cache_home = _define_xdg_or_backup(xdg_cache_home, os.path.join("~", ".cache")) # User configuration def _get_user_config_path(): diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 958c433ccc7d14..591b90f5d30846 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -126,10 +126,7 @@ def parse_install_tree(config_dict): def _default_install_location(): - if os.access(spack.paths.prefix, os.W_OK): - return spack.paths.internal_install_tree_root - else: - return os.path.join(spack.paths.per_spack_user_root, "installs") + return spack.paths.internal_install_tree_root class Store: diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index a13ea318a1678b..cc8318753899c9 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -67,7 +67,7 @@ def replacements(): "user": lambda: get_user(), "tempdir": lambda: tempfile.gettempdir(), "user_cache_path": lambda: spack.paths.user_cache_path, - "per_spack_user": lambda: spack.paths.per_spack_user_root, + "spack_xdg_cache_home": lambda: spack.paths.spack_xdg_cache_home, "architecture": lambda: arch, "arch": lambda: arch, "platform": lambda: arch.platform, @@ -158,21 +158,21 @@ def substitute_config_variables(path): Spack allows paths in configs to have some placeholders, as follows: - - $env The active Spack environment. - - $spack The Spack instance's prefix - - $tempdir Default temporary directory returned by tempfile.gettempdir() - - $user The current user's username - - $user_cache_path The user cache directory (~/.spack, unless overridden) - - $per_spack_user Each Spack install has its own dir in ~ - - $architecture The spack architecture triple for the current system - - $arch The spack architecture triple for the current system - - $platform The spack platform for the current system - - $os The OS of the current system - - $operating_system The OS of the current system - - $target The ISA target detected for the system - - $target_family The family of the target detected for the system - - $date The current date (YYYY-MM-DD) - - $spack_short_version The spack short version + - $env The active Spack environment. + - $spack The Spack instance's prefix + - $tempdir Default temporary directory returned by tempfile.gettempdir() + - $user The current user's username + - $user_cache_path The user cache directory ($XDG_DATA_HOME, unless overridden) + - $spack_xdg_cache_home Backup location for temporary data + - $architecture The spack architecture triple for the current system + - $arch The spack architecture triple for the current system + - $platform The spack platform for the current system + - $os The OS of the current system + - $operating_system The OS of the current system + - $target The ISA target detected for the system + - $target_family The family of the target detected for the system + - $date The current date (YYYY-MM-DD) + - $spack_short_version The spack short version These are substituted case-insensitively into the path, and users can use either ``$var`` or ``${var}`` syntax for the variables. $env is only From 8e219e8405bda874da4b1c78247348eeed9a4070 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 14 May 2025 18:03:33 -0700 Subject: [PATCH 070/506] auto style fix --- lib/spack/spack/paths.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 12010ae0cf17c9..6d48fbd92dd833 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -11,6 +11,7 @@ import os import pathlib from pathlib import PurePath +from typing import Optional import llnl.util.filesystem @@ -38,18 +39,20 @@ def _define_xdg_or_backup(xdg_var, backup): spack_xdg_config_home = _define_xdg_or_backup(xdg_config_home, os.path.join("~", "config")) spack_xdg_data_home = _define_xdg_or_backup(xdg_data_home, os.path.join("~", ".local", "share")) +spack_xdg_data_home_nodefault: Optional[str] if xdg_data_home in os.environ: - spack_xdg_data_home_nodefault = os.path.expanduser(os.path.join(os.environ[xdg_data_home], "spack")) + spack_xdg_data_home_nodefault = os.path.expanduser( + os.path.join(os.environ[xdg_data_home], "spack") + ) else: spack_xdg_data_home_nodefault = None spack_xdg_cache_home = _define_xdg_or_backup(xdg_cache_home, os.path.join("~", ".cache")) + # User configuration def _get_user_config_path(): - return os.path.expanduser( - os.getenv("SPACK_USER_CONFIG_PATH") or spack_xdg_config_home - ) + return os.path.expanduser(os.getenv("SPACK_USER_CONFIG_PATH") or spack_xdg_config_home) # Configuration in /etc/spack on the system From 0811ae211858532cd4919d95f77299191b177069 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 10:58:09 -0700 Subject: [PATCH 071/506] fix default for misc cache in yaml --- etc/spack/defaults/config.yaml | 2 +- lib/spack/spack/util/path.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index d8514f92078ba8..10b56c5e1a650f 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -82,7 +82,7 @@ config: # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` - misc_cache: $per_spack_user/cache + misc_cache: $spack_xdg_state_home/$spack_instance_id/cache # Abort downloads after this many seconds if not data is received. diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index cc8318753899c9..491b8966f93531 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -68,6 +68,8 @@ def replacements(): "tempdir": lambda: tempfile.gettempdir(), "user_cache_path": lambda: spack.paths.user_cache_path, "spack_xdg_cache_home": lambda: spack.paths.spack_xdg_cache_home, + "spack_xdg_state_home": lambda: spack.paths.spack_xdg_state_home, + "spack_instance_id": lambda: spack.paths.spack_instance_id, "architecture": lambda: arch, "arch": lambda: arch, "platform": lambda: arch.platform, @@ -164,6 +166,8 @@ def substitute_config_variables(path): - $user The current user's username - $user_cache_path The user cache directory ($XDG_DATA_HOME, unless overridden) - $spack_xdg_cache_home Backup location for temporary data + - $spack_xdg_state_home Long-lived but not-essential cache + - $spack_instance_id Hash that distinguishes Spack instances on the filesystem - $architecture The spack architecture triple for the current system - $arch The spack architecture triple for the current system - $platform The spack platform for the current system From 840a7b91317bfba230f2bc3e410d57afd521ac6f Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 12:26:24 -0700 Subject: [PATCH 072/506] fix default xdg_config_home and update test --- lib/spack/spack/paths.py | 2 +- lib/spack/spack/test/config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 6d48fbd92dd833..63571f213ebb08 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -36,7 +36,7 @@ def _define_xdg_or_backup(xdg_var, backup): #: Resolved XDG_ counterparts, with additional "spack" subdirectory spack_xdg_state_home = _define_xdg_or_backup(xdg_state_home, os.path.join("~", ".local", "state")) -spack_xdg_config_home = _define_xdg_or_backup(xdg_config_home, os.path.join("~", "config")) +spack_xdg_config_home = _define_xdg_or_backup(xdg_config_home, os.path.join("~", ".config")) spack_xdg_data_home = _define_xdg_or_backup(xdg_data_home, os.path.join("~", ".local", "share")) spack_xdg_data_home_nodefault: Optional[str] diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 72b1ecda8ac51a..4c3ab6510ca0a4 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1216,7 +1216,7 @@ def test_user_config_path_is_overridable(working_env): def test_user_config_path_is_default_when_env_var_is_empty(working_env): os.environ["SPACK_USER_CONFIG_PATH"] = "" - assert os.path.expanduser("~%s.spack" % os.sep) == spack.paths._get_user_config_path() + assert os.path.expanduser(os.path.join("~", ".config", "spack")) == spack.paths._get_user_config_path() def test_default_install_tree(monkeypatch, default_config): From a61ce1c600fc479c142a09d0a644f69ef0157f18 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 12:29:45 -0700 Subject: [PATCH 073/506] fix test for user_cache_path default --- lib/spack/spack/test/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 4c3ab6510ca0a4..fb92d4f7fa31c8 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1257,7 +1257,7 @@ def test_user_cache_path_is_overridable(working_env): def test_user_cache_path_is_default_when_env_var_is_empty(working_env): os.environ["SPACK_USER_CACHE_PATH"] = "" - assert os.path.expanduser("~%s.spack" % os.sep) == spack.paths._get_user_cache_path() + assert os.path.expanduser(os.path.join("~", ".local", "share", "spack")) == spack.paths._get_user_cache_path() def test_config_file_dir_failure(tmpdir, mutable_empty_config): From 7ff3df5afa7a5ba70f4f6b53513d3747c70d3bd1 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 12:31:47 -0700 Subject: [PATCH 074/506] auto style fix --- lib/spack/spack/test/config.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index fb92d4f7fa31c8..bf5a708e815119 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1216,7 +1216,10 @@ def test_user_config_path_is_overridable(working_env): def test_user_config_path_is_default_when_env_var_is_empty(working_env): os.environ["SPACK_USER_CONFIG_PATH"] = "" - assert os.path.expanduser(os.path.join("~", ".config", "spack")) == spack.paths._get_user_config_path() + assert ( + os.path.expanduser(os.path.join("~", ".config", "spack")) + == spack.paths._get_user_config_path() + ) def test_default_install_tree(monkeypatch, default_config): @@ -1257,7 +1260,10 @@ def test_user_cache_path_is_overridable(working_env): def test_user_cache_path_is_default_when_env_var_is_empty(working_env): os.environ["SPACK_USER_CACHE_PATH"] = "" - assert os.path.expanduser(os.path.join("~", ".local", "share", "spack")) == spack.paths._get_user_cache_path() + assert ( + os.path.expanduser(os.path.join("~", ".local", "share", "spack")) + == spack.paths._get_user_cache_path() + ) def test_config_file_dir_failure(tmpdir, mutable_empty_config): From 84a77c7b194608cb4da3237e322632ad0692bd08 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 12:47:17 -0700 Subject: [PATCH 075/506] default install location uses XDG_DATA_HOME if it is explicitly set --- lib/spack/spack/paths.py | 12 +++++++++--- lib/spack/spack/store.py | 6 +----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 63571f213ebb08..54e725cd782367 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -109,9 +109,6 @@ def dir_is_occupied(x, except_for=None): var_path = os.path.join(prefix, "var", "spack") -internal_install_tree_root = os.path.join(prefix, "opt", "spack") - - spack_instance_id = hash.b32_hash(prefix)[:7] #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) @@ -139,6 +136,15 @@ def dir_is_occupied(x, except_for=None): # TODO: maybe store views/develop packages in a separate location? envs_path = old_envs_path +old_install_path = os.path.join(prefix, "opt", "spack") +if dir_is_occupied(old_install_path): + default_install_location = old_install_path +elif spack_xdg_data_home_nodefault: + default_install_location = os.path.join(spack_xdg_data_home_nodefault, "installs") +else: + default_install_location = old_install_path + + # TODO: we could shutil.mv resources from old paths to new paths # $spack/var/spack is generally read-only. Older instances may diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 591b90f5d30846..50890a02f72e08 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -78,7 +78,7 @@ def parse_install_tree(config_dict): projections = {"all": all_projection} else: - unpadded_root = install_tree.get("root", _default_install_location()) + unpadded_root = install_tree.get("root", spack.paths.default_install_location) unpadded_root = spack.util.path.canonicalize_path(unpadded_root) padded_length = install_tree.get("padded_length", False) @@ -125,10 +125,6 @@ def parse_install_tree(config_dict): return root, unpadded_root, projections -def _default_install_location(): - return spack.paths.internal_install_tree_root - - class Store: """A store is a path full of installed Spack packages. From 02a438e879ab5e127862a64abf8081c0419d4457 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 13:00:33 -0700 Subject: [PATCH 076/506] downloads, envs, and installs all go into spack_data_home, which points to spack prefix if XDG_DATA_HOME and SPACK_DATA_HOME are not set --- lib/spack/spack/paths.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 54e725cd782367..f12161929835e3 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -23,6 +23,7 @@ xdg_config_home = "XDG_CONFIG_HOME" xdg_state_home = "XDG_STATE_HOME" xdg_data_home = "XDG_DATA_HOME" +spack_data_home_varname = "SPACK_DATA_HOME" xdg_cache_home = "XDG_CACHE_HOME" @@ -49,6 +50,12 @@ def _define_xdg_or_backup(xdg_var, backup): spack_xdg_cache_home = _define_xdg_or_backup(xdg_cache_home, os.path.join("~", ".cache")) +if spack_data_home_varname in os.environ: + spack_data_home = os.environ[spack_data_home_varname] +elif spack_xdg_data_home_nodefault: + spack_data_home = spack_xdg_data_home_nodefault +else: + spack_data_home = os.path.join(prefix, "opt", "data") # User configuration def _get_user_config_path(): @@ -127,22 +134,18 @@ def dir_is_occupied(x, except_for=None): old_envs_path = os.path.join(var_path, "environments") if dir_is_occupied(old_envs_path): envs_path = old_envs_path -elif spack_xdg_data_home_nodefault: - envs_path = os.path.join(spack_xdg_data_home_nodefault, "environments") else: # Environments can store views and `develop` packages, which # take up too much space for us to place them in ~ unless the # user explicitly requests it # TODO: maybe store views/develop packages in a separate location? - envs_path = old_envs_path + envs_path = os.path.join(spack_data_home, "environments") old_install_path = os.path.join(prefix, "opt", "spack") if dir_is_occupied(old_install_path): default_install_location = old_install_path -elif spack_xdg_data_home_nodefault: - default_install_location = os.path.join(spack_xdg_data_home_nodefault, "installs") else: - default_install_location = old_install_path + default_install_location = os.path.join(spack_data_home, "installs") # TODO: we could shutil.mv resources from old paths to new paths @@ -173,7 +176,7 @@ def _get_user_cache_path(): user_cache_path = str(PurePath(_get_user_cache_path())) -default_fetch_cache_path = os.path.join(user_cache_path, "downloads") +default_fetch_cache_path = os.path.join(spack_data_home, "downloads") #: junit, cdash, etc. reports about builds reports_path = os.path.join(user_cache_path, "reports") From 7b373f365668a349b652e2266356903528aca376 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 13:31:29 -0700 Subject: [PATCH 077/506] precedence for using old install path vs. XDG_DATA_HOME --- lib/spack/spack/paths.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index f12161929835e3..7425ba3dd25ec8 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -141,6 +141,12 @@ def dir_is_occupied(x, except_for=None): # TODO: maybe store views/develop packages in a separate location? envs_path = os.path.join(spack_data_home, "environments") +# TODO: proposed precedence +# 1. config:install_tree:root +# 2. explicitly defined SPACK_DATA_HOME +# 3. occupied old install path (inside spack prefix) +# 4. explicitly defined XDG_DATA_HOME +# 5. inside spack prefix (slightly different compared to old install path) old_install_path = os.path.join(prefix, "opt", "spack") if dir_is_occupied(old_install_path): default_install_location = old_install_path From 0a4323f1d000fce04dc2d285ed1e22989fabce6d Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 13:43:40 -0700 Subject: [PATCH 078/506] match proposed precedence --- lib/spack/spack/paths.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 7425ba3dd25ec8..bc55e020f854cf 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -132,7 +132,9 @@ def dir_is_occupied(x, except_for=None): modules_base = os.path.join(spack_xdg_data_home, "modules") old_envs_path = os.path.join(var_path, "environments") -if dir_is_occupied(old_envs_path): +if spack_data_home_varname in os.environ: + envs_path = os.path.join(spack_data_home, "environments") +elif dir_is_occupied(old_envs_path): envs_path = old_envs_path else: # Environments can store views and `develop` packages, which @@ -148,7 +150,9 @@ def dir_is_occupied(x, except_for=None): # 4. explicitly defined XDG_DATA_HOME # 5. inside spack prefix (slightly different compared to old install path) old_install_path = os.path.join(prefix, "opt", "spack") -if dir_is_occupied(old_install_path): +if spack_data_home_varname in os.environ: + default_install_location = os.path.join(spack_data_home, "installs") +elif dir_is_occupied(old_install_path): default_install_location = old_install_path else: default_install_location = os.path.join(spack_data_home, "installs") From b596211f27f5fb0f24179b0bf57a59219f813781 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 13:44:19 -0700 Subject: [PATCH 079/506] move install logic first (no logic change) --- lib/spack/spack/paths.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index bc55e020f854cf..60f2b66f9e24f7 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -131,18 +131,6 @@ def dir_is_occupied(x, except_for=None): if not modules_base: modules_base = os.path.join(spack_xdg_data_home, "modules") -old_envs_path = os.path.join(var_path, "environments") -if spack_data_home_varname in os.environ: - envs_path = os.path.join(spack_data_home, "environments") -elif dir_is_occupied(old_envs_path): - envs_path = old_envs_path -else: - # Environments can store views and `develop` packages, which - # take up too much space for us to place them in ~ unless the - # user explicitly requests it - # TODO: maybe store views/develop packages in a separate location? - envs_path = os.path.join(spack_data_home, "environments") - # TODO: proposed precedence # 1. config:install_tree:root # 2. explicitly defined SPACK_DATA_HOME @@ -157,6 +145,17 @@ def dir_is_occupied(x, except_for=None): else: default_install_location = os.path.join(spack_data_home, "installs") +old_envs_path = os.path.join(var_path, "environments") +if spack_data_home_varname in os.environ: + envs_path = os.path.join(spack_data_home, "environments") +elif dir_is_occupied(old_envs_path): + envs_path = old_envs_path +else: + # Environments can store views and `develop` packages, which + # take up too much space for us to place them in ~ unless the + # user explicitly requests it + # TODO: maybe store views/develop packages in a separate location? + envs_path = os.path.join(spack_data_home, "environments") # TODO: we could shutil.mv resources from old paths to new paths From f7b4dd0aea756cff4dac3064db5fbe75d984b524 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 13:45:29 -0700 Subject: [PATCH 080/506] auto style fix --- lib/spack/spack/paths.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 60f2b66f9e24f7..116cb95530dffa 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -57,6 +57,7 @@ def _define_xdg_or_backup(xdg_var, backup): else: spack_data_home = os.path.join(prefix, "opt", "data") + # User configuration def _get_user_config_path(): return os.path.expanduser(os.getenv("SPACK_USER_CONFIG_PATH") or spack_xdg_config_home) From f9d4fcb71675d7cc19f22d837c04224a07b9111a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 13:46:03 -0700 Subject: [PATCH 081/506] edit comment --- lib/spack/spack/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 116cb95530dffa..b11b84d0b2898f 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -132,7 +132,7 @@ def dir_is_occupied(x, except_for=None): if not modules_base: modules_base = os.path.join(spack_xdg_data_home, "modules") -# TODO: proposed precedence +# Precedence for installs: # 1. config:install_tree:root # 2. explicitly defined SPACK_DATA_HOME # 3. occupied old install path (inside spack prefix) From d3313251621e473ce882db88ecbbf5324bb77446 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 13:56:10 -0700 Subject: [PATCH 082/506] more comments --- lib/spack/spack/paths.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index b11b84d0b2898f..6fc76deae51a0b 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -50,6 +50,13 @@ def _define_xdg_or_backup(xdg_var, backup): spack_xdg_cache_home = _define_xdg_or_backup(xdg_cache_home, os.path.join("~", ".cache")) + +# spack_data_home is where we know we can put large amounts of data: +# users can set SPACK_DATA_HOME to tell spack explicitly about such +# a location. If XDG_DATA_HOME is set, we assume we can use that. +# If neither are set, we assume the spack prefix is the only place +# available to us (we do not use ~ and in particular the default for +# XDG_DATA_HOME). if spack_data_home_varname in os.environ: spack_data_home = os.environ[spack_data_home_varname] elif spack_xdg_data_home_nodefault: @@ -146,16 +153,14 @@ def dir_is_occupied(x, except_for=None): else: default_install_location = os.path.join(spack_data_home, "installs") +# Environments follow the same precedence rules as installs +# (the view and dev_path packages can take up significant space) old_envs_path = os.path.join(var_path, "environments") if spack_data_home_varname in os.environ: envs_path = os.path.join(spack_data_home, "environments") elif dir_is_occupied(old_envs_path): envs_path = old_envs_path else: - # Environments can store views and `develop` packages, which - # take up too much space for us to place them in ~ unless the - # user explicitly requests it - # TODO: maybe store views/develop packages in a separate location? envs_path = os.path.join(spack_data_home, "environments") # TODO: we could shutil.mv resources from old paths to new paths From 60eeb8336a56e701fc725d8adfbb47c0cdf6b263 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 14:32:52 -0700 Subject: [PATCH 083/506] update completion with new scope name --- lib/spack/spack/config.py | 2 +- share/spack/spack-completion.fish | 52 +++++++++++++++---------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 0536890c89aaa9..2e6082856e6c15 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -986,7 +986,7 @@ def create_incremental() -> Generator[Configuration, None, None]: configuration_paths.append(("user", spack.paths.user_config_path)) per_spack_cfg = os.path.join(spack.paths.user_config_path, spack.paths.spack_instance_id) - configuration_paths.append(("this-spack-user", per_spack_cfg)) + configuration_paths.append(("this-spack", per_spack_cfg)) # add each scope and its platform-specific directory for name, path in configuration_paths: diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index d74a3096580b1d..ac6ce59cd57788 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -606,7 +606,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_enable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap enable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap disable @@ -614,7 +614,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_disable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap disable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap reset @@ -629,14 +629,14 @@ set -g __fish_spack_optspecs_spack_bootstrap_root h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap root' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap list set -g __fish_spack_optspecs_spack_bootstrap_list h/help scope= complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap add @@ -645,7 +645,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap add' -f -a '(__ complete -c spack -n '__fish_spack_using_command_pos 1 bootstrap add' -f -a '(__fish_spack_environments)' complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -f -a trust complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -d 'enable the source immediately upon addition' @@ -819,7 +819,7 @@ complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirro complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -d 'override any configured mirrors with this mirror URL' complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -f -a output_file complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -d 'file where rebuild info should be written' -complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -d 'configuration scope containing mirrors to check' # spack buildcache download @@ -1085,7 +1085,7 @@ complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolcha complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolchain -d '(DEPRECATED) Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -d '(DEPRECATED) Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1099,7 +1099,7 @@ complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchai complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchain -d '(DEPRECATED) Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -d '(DEPRECATED) Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1111,7 +1111,7 @@ complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help - complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -d 'configuration scope to modify' # spack compiler rm @@ -1121,21 +1121,21 @@ complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -d 'configuration scope to modify' # spack compiler list set -g __fish_spack_optspecs_spack_compiler_list h/help scope= complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -d 'configuration scope to read from' # spack compiler ls set -g __fish_spack_optspecs_spack_compiler_ls h/help scope= complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -d 'configuration scope to read from' # spack compiler info @@ -1143,14 +1143,14 @@ set -g __fish_spack_optspecs_spack_compiler_info h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 compiler info' -f -a '(__fish_spack_installed_compilers)' complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -d 'configuration scope to read from' # spack compilers set -g __fish_spack_optspecs_spack_compilers h/help scope= complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -d 'configuration scope to read/modify' # spack concretize @@ -1210,7 +1210,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a update -d ' complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a revert -d 'revert configuration files to their state before update' complete -c spack -n '__fish_spack_using_command config' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify' # spack config get @@ -1748,7 +1748,7 @@ complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -f complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -d 'packages to exclude from search' complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -f -a path complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -d 'one or more alternative search paths for finding externals' -complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command external find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command external find' -l all -f -a all complete -c spack -n '__fish_spack_using_command external find' -l all -d 'search for all packages that Spack knows about' @@ -2322,7 +2322,7 @@ set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsig complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use `--type binary --type source` (default)' @@ -2362,7 +2362,7 @@ set -g __fish_spack_optspecs_spack_mirror_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror remove' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -d 'configuration scope to modify' # spack mirror rm @@ -2370,7 +2370,7 @@ set -g __fish_spack_optspecs_spack_mirror_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror rm' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -d 'configuration scope to modify' # spack mirror set-url @@ -2382,7 +2382,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -f -a p complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -d 'set only the URL used for uploading' complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -f -a fetch complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -d 'set only the URL used for downloading' -complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2430,7 +2430,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a s complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache' complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed complete -c spack -n '__fish_spack_using_command mirror set' -l signed -d 'require signing and signature verification when pushing and installing from this build cache' -complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2461,7 +2461,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password-var set -g __fish_spack_optspecs_spack_mirror_list h/help scope= complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'configuration scope to read from' # spack module @@ -2768,7 +2768,7 @@ complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirecto set -g __fish_spack_optspecs_spack_repo_list h/help scope= complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'configuration scope to read from' # spack repo add @@ -2776,7 +2776,7 @@ set -g __fish_spack_optspecs_spack_repo_add h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo add' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -d 'configuration scope to modify' # spack repo remove @@ -2784,7 +2784,7 @@ set -g __fish_spack_optspecs_spack_repo_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo remove' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -d 'configuration scope to modify' # spack repo rm @@ -2792,7 +2792,7 @@ set -g __fish_spack_optspecs_spack_repo_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo rm' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system end-user site user per-spack-user command_line' +complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify' # spack resource From d62868ae597856a12ec580f33fc86a3d5936fd5f Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 14:43:38 -0700 Subject: [PATCH 084/506] fixed issue with actually resolving xdg vars; Ill need to update all the spack.paths xdg variables to methods vs. properties to make them testable --- lib/spack/spack/paths.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 6fc76deae51a0b..5d1c24a1c32bd6 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -27,9 +27,16 @@ xdg_cache_home = "XDG_CACHE_HOME" +# This is for tests that want to clean the environment of XDG_ variables that +# affect spack behavior +def _unset_xdg_vars(env): + for xdg_var in [xdg_config_home, xdg_state_home, xdg_data_home, xdg_cache_home]: + env.pop(xdg_var, None) + + def _define_xdg_or_backup(xdg_var, backup): if xdg_var in os.environ: - spack_xdg_defined = os.path.join(xdg_var, "spack") + spack_xdg_defined = os.path.join(os.environ[xdg_var], "spack") else: spack_xdg_defined = os.path.join(backup, "spack") return os.path.expanduser(spack_xdg_defined) From a5c34518feb2bc6d6a827b4ab609ce4d215f4baa Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 14:56:56 -0700 Subject: [PATCH 085/506] turn spack.paths properties into functions, so we can test behavior by modifying env vars (this work is incomplete) --- lib/spack/spack/paths.py | 78 +++++++++++++++++--------------- lib/spack/spack/store.py | 2 +- lib/spack/spack/test/conftest.py | 6 +++ lib/spack/spack/util/path.py | 6 +-- 4 files changed, 52 insertions(+), 40 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 5d1c24a1c32bd6..b49324bbebe8e2 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -30,8 +30,11 @@ # This is for tests that want to clean the environment of XDG_ variables that # affect spack behavior def _unset_xdg_vars(env): + saved = {} for xdg_var in [xdg_config_home, xdg_state_home, xdg_data_home, xdg_cache_home]: - env.pop(xdg_var, None) + if xdg_var in env: + saved[xdg_var] = env.pop(xdg_var) + return saved def _define_xdg_or_backup(xdg_var, backup): @@ -43,9 +46,9 @@ def _define_xdg_or_backup(xdg_var, backup): #: Resolved XDG_ counterparts, with additional "spack" subdirectory -spack_xdg_state_home = _define_xdg_or_backup(xdg_state_home, os.path.join("~", ".local", "state")) -spack_xdg_config_home = _define_xdg_or_backup(xdg_config_home, os.path.join("~", ".config")) -spack_xdg_data_home = _define_xdg_or_backup(xdg_data_home, os.path.join("~", ".local", "share")) +spack_xdg_state_home = lambda: _define_xdg_or_backup(xdg_state_home, os.path.join("~", ".local", "state")) +spack_xdg_config_home = lambda: _define_xdg_or_backup(xdg_config_home, os.path.join("~", ".config")) +spack_xdg_data_home = lambda: _define_xdg_or_backup(xdg_data_home, os.path.join("~", ".local", "share")) spack_xdg_data_home_nodefault: Optional[str] if xdg_data_home in os.environ: @@ -55,21 +58,22 @@ def _define_xdg_or_backup(xdg_var, backup): else: spack_xdg_data_home_nodefault = None -spack_xdg_cache_home = _define_xdg_or_backup(xdg_cache_home, os.path.join("~", ".cache")) +spack_xdg_cache_home = lambda: _define_xdg_or_backup(xdg_cache_home, os.path.join("~", ".cache")) -# spack_data_home is where we know we can put large amounts of data: -# users can set SPACK_DATA_HOME to tell spack explicitly about such -# a location. If XDG_DATA_HOME is set, we assume we can use that. -# If neither are set, we assume the spack prefix is the only place -# available to us (we do not use ~ and in particular the default for -# XDG_DATA_HOME). -if spack_data_home_varname in os.environ: - spack_data_home = os.environ[spack_data_home_varname] -elif spack_xdg_data_home_nodefault: - spack_data_home = spack_xdg_data_home_nodefault -else: - spack_data_home = os.path.join(prefix, "opt", "data") +def spack_data_home(): + # spack_data_home is where we know we can put large amounts of data: + # users can set SPACK_DATA_HOME to tell spack explicitly about such + # a location. If XDG_DATA_HOME is set, we assume we can use that. + # If neither are set, we assume the spack prefix is the only place + # available to us (we do not use ~ and in particular the default for + # XDG_DATA_HOME). + if spack_data_home_varname in os.environ: + return os.environ[spack_data_home_varname] + elif spack_xdg_data_home_nodefault: + return spack_xdg_data_home_nodefault + else: + return os.path.join(prefix, "opt", "data") # User configuration @@ -131,10 +135,10 @@ def dir_is_occupied(x, except_for=None): var_path = os.path.join(prefix, "var", "spack") -spack_instance_id = hash.b32_hash(prefix)[:7] +spack_instance_id = lambda: hash.b32_hash(prefix)[:7] #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) -default_misc_cache_path = os.path.join(spack_xdg_state_home, spack_instance_id, "cache") +default_misc_cache_path = os.path.join(spack_xdg_state_home(), spack_instance_id(), "cache") #: concretization cache for Spack concretizations default_conc_cache_path = os.path.join(default_misc_cache_path, "concretization") @@ -144,31 +148,33 @@ def dir_is_occupied(x, except_for=None): if dir_is_occupied(os.path.join(share_path, module_dir)): modules_base = share_path if not modules_base: - modules_base = os.path.join(spack_xdg_data_home, "modules") - -# Precedence for installs: -# 1. config:install_tree:root -# 2. explicitly defined SPACK_DATA_HOME -# 3. occupied old install path (inside spack prefix) -# 4. explicitly defined XDG_DATA_HOME -# 5. inside spack prefix (slightly different compared to old install path) -old_install_path = os.path.join(prefix, "opt", "spack") -if spack_data_home_varname in os.environ: - default_install_location = os.path.join(spack_data_home, "installs") -elif dir_is_occupied(old_install_path): - default_install_location = old_install_path -else: - default_install_location = os.path.join(spack_data_home, "installs") + modules_base = os.path.join(spack_xdg_data_home(), "modules") + + +def default_install_location(): + # Precedence for installs: + # 1. config:install_tree:root + # 2. explicitly defined SPACK_DATA_HOME + # 3. occupied old install path (inside spack prefix) + # 4. explicitly defined XDG_DATA_HOME + # 5. inside spack prefix (slightly different compared to old install path) + old_install_path = os.path.join(prefix, "opt", "spack") + if spack_data_home_varname in os.environ: + return os.path.join(spack_data_home(), "installs") + elif dir_is_occupied(old_install_path): + return old_install_path + else: + return os.path.join(spack_data_home(), "installs") # Environments follow the same precedence rules as installs # (the view and dev_path packages can take up significant space) old_envs_path = os.path.join(var_path, "environments") if spack_data_home_varname in os.environ: - envs_path = os.path.join(spack_data_home, "environments") + envs_path = os.path.join(spack_data_home(), "environments") elif dir_is_occupied(old_envs_path): envs_path = old_envs_path else: - envs_path = os.path.join(spack_data_home, "environments") + envs_path = os.path.join(spack_data_home(), "environments") # TODO: we could shutil.mv resources from old paths to new paths diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 50890a02f72e08..27bdfade56f35f 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -78,7 +78,7 @@ def parse_install_tree(config_dict): projections = {"all": all_projection} else: - unpadded_root = install_tree.get("root", spack.paths.default_install_location) + unpadded_root = install_tree.get("root", spack.paths.default_install_location()) unpadded_root = spack.util.path.canonicalize_path(unpadded_root) padded_length = install_tree.get("padded_length", False) diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index b5b8e4d9825637..96f2f0dfa33fa4 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -360,6 +360,12 @@ def clean_user_environment(): os.environ[ev.spack_env_var] = spack_env_value +@pytest.fixture(scope="session", autouse=True) +def clear_xdg_vars(): + saved = spack.paths._unset_xdg_vars(os.environ) + yield + os.environ.update(saved) + # # Make sure global state of active env does not leak between tests. # diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 491b8966f93531..dda7844a5bf0f1 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -67,9 +67,9 @@ def replacements(): "user": lambda: get_user(), "tempdir": lambda: tempfile.gettempdir(), "user_cache_path": lambda: spack.paths.user_cache_path, - "spack_xdg_cache_home": lambda: spack.paths.spack_xdg_cache_home, - "spack_xdg_state_home": lambda: spack.paths.spack_xdg_state_home, - "spack_instance_id": lambda: spack.paths.spack_instance_id, + "spack_xdg_cache_home": lambda: spack.paths.spack_xdg_cache_home(), + "spack_xdg_state_home": lambda: spack.paths.spack_xdg_state_home(), + "spack_instance_id": lambda: spack.paths.spack_instance_id(), "architecture": lambda: arch, "arch": lambda: arch, "platform": lambda: arch.platform, From 4a4ec4f26f3ade26313a52408b6193872d41f46d Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 15:02:33 -0700 Subject: [PATCH 086/506] lambda-fy the spack_xdg variables --- lib/spack/spack/config.py | 2 +- lib/spack/spack/paths.py | 19 +++++++++++++------ lib/spack/spack/test/conftest.py | 1 + 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 2e6082856e6c15..ae1b1292e5ac5d 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -985,7 +985,7 @@ def create_incremental() -> Generator[Configuration, None, None]: if not disable_local_config: configuration_paths.append(("user", spack.paths.user_config_path)) - per_spack_cfg = os.path.join(spack.paths.user_config_path, spack.paths.spack_instance_id) + per_spack_cfg = os.path.join(spack.paths.user_config_path, spack.paths.spack_instance_id()) configuration_paths.append(("this-spack", per_spack_cfg)) # add each scope and its platform-specific directory diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index b49324bbebe8e2..d91231e5c1e9c7 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -46,9 +46,15 @@ def _define_xdg_or_backup(xdg_var, backup): #: Resolved XDG_ counterparts, with additional "spack" subdirectory -spack_xdg_state_home = lambda: _define_xdg_or_backup(xdg_state_home, os.path.join("~", ".local", "state")) -spack_xdg_config_home = lambda: _define_xdg_or_backup(xdg_config_home, os.path.join("~", ".config")) -spack_xdg_data_home = lambda: _define_xdg_or_backup(xdg_data_home, os.path.join("~", ".local", "share")) +spack_xdg_state_home = lambda: _define_xdg_or_backup( + xdg_state_home, os.path.join("~", ".local", "state") +) +spack_xdg_config_home = lambda: _define_xdg_or_backup( + xdg_config_home, os.path.join("~", ".config") +) +spack_xdg_data_home = lambda: _define_xdg_or_backup( + xdg_data_home, os.path.join("~", ".local", "share") +) spack_xdg_data_home_nodefault: Optional[str] if xdg_data_home in os.environ: @@ -78,7 +84,7 @@ def spack_data_home(): # User configuration def _get_user_config_path(): - return os.path.expanduser(os.getenv("SPACK_USER_CONFIG_PATH") or spack_xdg_config_home) + return os.path.expanduser(os.getenv("SPACK_USER_CONFIG_PATH") or spack_xdg_config_home()) # Configuration in /etc/spack on the system @@ -166,6 +172,7 @@ def default_install_location(): else: return os.path.join(spack_data_home(), "installs") + # Environments follow the same precedence rules as installs # (the view and dev_path packages can take up significant space) old_envs_path = os.path.join(var_path, "environments") @@ -199,12 +206,12 @@ def default_install_location(): # setting `SPACK_USER_CACHE_PATH`. Otherwise it defaults to ~/.spack. # def _get_user_cache_path(): - return os.path.expanduser(os.getenv("SPACK_USER_CACHE_PATH") or spack_xdg_data_home) + return os.path.expanduser(os.getenv("SPACK_USER_CACHE_PATH") or spack_xdg_data_home()) user_cache_path = str(PurePath(_get_user_cache_path())) -default_fetch_cache_path = os.path.join(spack_data_home, "downloads") +default_fetch_cache_path = os.path.join(spack_data_home(), "downloads") #: junit, cdash, etc. reports about builds reports_path = os.path.join(user_cache_path, "reports") diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 96f2f0dfa33fa4..141b9c2eb9504a 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -366,6 +366,7 @@ def clear_xdg_vars(): yield os.environ.update(saved) + # # Make sure global state of active env does not leak between tests. # From ec4e55aad14444aa481c07885b2664f682f2d461 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 15:57:52 -0700 Subject: [PATCH 087/506] update bootstrap ci checks with xdg locations --- .github/workflows/bootstrap.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index 7a2759033f3b24..f0e7ec1f8bb96d 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -47,7 +47,7 @@ jobs: spack bootstrap disable github-actions-v0.5 spack external find cmake bison spack -d solve zlib - tree ~/.spack/bootstrap/store/ + tree ~/.local/share/spack/bootstrap/store clingo-sources: runs-on: ${{ matrix.runner }} @@ -73,7 +73,7 @@ jobs: spack bootstrap disable github-actions-v0.5 spack external find --not-buildable cmake bison spack -d solve zlib - tree $HOME/.spack/bootstrap/store/ + tree $HOME/.local/share/spack/bootstrap/store gnupg-sources: runs-on: ${{ matrix.runner }} @@ -100,7 +100,7 @@ jobs: spack bootstrap disable github-actions-v0.6 spack bootstrap disable github-actions-v0.5 spack -d gpg list - tree ~/.spack/bootstrap/store/ + tree ~/.local/share/spack/bootstrap/store from-binaries: runs-on: ${{ matrix.runner }} @@ -160,7 +160,7 @@ jobs: run: | source share/spack/setup-env.sh spack -d gpg list - tree $HOME/.spack/bootstrap/store/ + tree $HOME/.local/share/spack/bootstrap/store windows: @@ -185,10 +185,10 @@ jobs: spack external find --not-buildable cmake bison spack -d solve zlib ./share/spack/qa/validate_last_exit.ps1 - tree $env:userprofile/.spack/bootstrap/store/ + tree $env:userprofile/.local/share/spack/bootstrap/store - name: Bootstrap GnuPG run: | ./share/spack/setup-env.ps1 spack -d gpg list ./share/spack/qa/validate_last_exit.ps1 - tree $env:userprofile/.spack/bootstrap/store/ + tree $env:userprofile/.local/share/spack/bootstrap/store From 97aacd1c6113f58cd75b23a98d677170686ed931 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 May 2025 16:44:59 -0700 Subject: [PATCH 088/506] fix docs error --- lib/spack/spack/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index d91231e5c1e9c7..6fe1ee8b020601 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -45,7 +45,7 @@ def _define_xdg_or_backup(xdg_var, backup): return os.path.expanduser(spack_xdg_defined) -#: Resolved XDG_ counterparts, with additional "spack" subdirectory +#: Resolved XDG_STATE_HOME, with additional "spack" subdirectory spack_xdg_state_home = lambda: _define_xdg_or_backup( xdg_state_home, os.path.join("~", ".local", "state") ) From b0e5c54c1bbb5efd0cc44d84031f8ba9a69070d1 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 2 Jun 2025 11:26:15 -0700 Subject: [PATCH 089/506] update command completion --- share/spack/spack-completion.fish | 58 +++++++++++++++++-------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 16e2d2a29e646e..7b644354368c86 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -346,7 +346,7 @@ complete -c spack --erase # Everything below here is auto-generated. # spack -set -g __fish_spack_optspecs_spack h/help H/all-help color= c/config= C/config-scope= d/debug timestamp pdb e/env= D/env-dir= E/no-env use-env-repo k/insecure l/enable-locks L/disable-locks m/mock b/bootstrap p/profile sorted-profile= lines= v/verbose stacktrace t/backtrace V/version print-shell-vars= +set -g __fish_spack_optspecs_spack h/help H/all-help color= c/config= C/config-scope= d/debug timestamp pdb e/env= D/env-dir= E/no-env use-env-repo k/insecure l/enable-locks L/disable-locks m/mock b/bootstrap p/profile sorted-profile= lines= v/verbose stacktrace t/backtrace V/version print-shell-vars= disable-end-user-config warn-writes-into-spack complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a add -d 'add a spec to an environment' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a arch -d 'print architecture information about this machine' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a audit -d 'audit configuration files, packages, etc.' @@ -478,6 +478,10 @@ complete -c spack -n '__fish_spack_using_command ' -s V -l version -f -a version complete -c spack -n '__fish_spack_using_command ' -s V -l version -d 'show version number and exit' complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -f -a print_shell_vars complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -d 'print info needed by setup-env.*sh' +complete -c spack -n '__fish_spack_using_command ' -l disable-end-user-config -f -a disable_end_user_config +complete -c spack -n '__fish_spack_using_command ' -l disable-end-user-config -d 'Disable system config scope for end users' +complete -c spack -n '__fish_spack_using_command ' -l warn-writes-into-spack -f -a warn_writes_into_spack +complete -c spack -n '__fish_spack_using_command ' -l warn-writes-into-spack -d 'Warn when Spack tries to write into its own prefix' # spack add set -g __fish_spack_optspecs_spack_add h/help l/list-name= @@ -602,7 +606,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_enable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap enable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap disable @@ -610,7 +614,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_disable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap disable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap reset @@ -625,14 +629,14 @@ set -g __fish_spack_optspecs_spack_bootstrap_root h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap root' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap list set -g __fish_spack_optspecs_spack_bootstrap_list h/help scope= complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap add @@ -641,7 +645,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap add' -f -a '(__ complete -c spack -n '__fish_spack_using_command_pos 1 bootstrap add' -f -a '(__fish_spack_environments)' complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -f -a trust complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -d 'enable the source immediately upon addition' @@ -815,7 +819,7 @@ complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirro complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -d 'override any configured mirrors with this mirror URL' complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -f -a output_file complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -d 'file where rebuild info should be written' -complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -d 'configuration scope containing mirrors to check' # spack buildcache download @@ -1081,7 +1085,7 @@ complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolcha complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolchain -d '(DEPRECATED) Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -d '(DEPRECATED) Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1095,7 +1099,7 @@ complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchai complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchain -d '(DEPRECATED) Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -d '(DEPRECATED) Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1107,7 +1111,7 @@ complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help - complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -d 'configuration scope to modify' # spack compiler rm @@ -1117,21 +1121,21 @@ complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -d 'configuration scope to modify' # spack compiler list set -g __fish_spack_optspecs_spack_compiler_list h/help scope= complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -d 'configuration scope to read from' # spack compiler ls set -g __fish_spack_optspecs_spack_compiler_ls h/help scope= complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -d 'configuration scope to read from' # spack compiler info @@ -1139,14 +1143,14 @@ set -g __fish_spack_optspecs_spack_compiler_info h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 compiler info' -f -a '(__fish_spack_installed_compilers)' complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -d 'configuration scope to read from' # spack compilers set -g __fish_spack_optspecs_spack_compilers h/help scope= complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -d 'configuration scope to read/modify' # spack concretize @@ -1207,7 +1211,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a update -d ' complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a revert -d 'revert configuration files to their state before update' complete -c spack -n '__fish_spack_using_command config' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify' # spack config get @@ -1755,7 +1759,7 @@ complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -f complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -d 'packages to exclude from search' complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -f -a path complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -d 'one or more alternative search paths for finding externals' -complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command external find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command external find' -l all -f -a all complete -c spack -n '__fish_spack_using_command external find' -l all -d 'search for all packages that Spack knows about' @@ -2331,7 +2335,7 @@ set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsig complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use `--type binary --type source` (default)' @@ -2371,7 +2375,7 @@ set -g __fish_spack_optspecs_spack_mirror_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror remove' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -d 'configuration scope to modify' # spack mirror rm @@ -2379,7 +2383,7 @@ set -g __fish_spack_optspecs_spack_mirror_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror rm' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -d 'configuration scope to modify' # spack mirror set-url @@ -2391,7 +2395,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -f -a p complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -d 'set only the URL used for uploading' complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -f -a fetch complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -d 'set only the URL used for downloading' -complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2439,7 +2443,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a s complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache' complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed complete -c spack -n '__fish_spack_using_command mirror set' -l signed -d 'require signing and signature verification when pushing and installing from this build cache' -complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2470,7 +2474,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password-var set -g __fish_spack_optspecs_spack_mirror_list h/help scope= complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'configuration scope to read from' # spack module @@ -2778,7 +2782,7 @@ complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirecto set -g __fish_spack_optspecs_spack_repo_list h/help scope= complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'configuration scope to read from' # spack repo add @@ -2786,7 +2790,7 @@ set -g __fish_spack_optspecs_spack_repo_add h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo add' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -d 'configuration scope to modify' # spack repo remove @@ -2794,7 +2798,7 @@ set -g __fish_spack_optspecs_spack_repo_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo remove' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -d 'configuration scope to modify' # spack repo rm @@ -2802,7 +2806,7 @@ set -g __fish_spack_optspecs_spack_repo_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo rm' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify' # spack repo migrate From 4cda8a25950c5f015b46e9f5f156e3a7221df237 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 2 Jun 2025 17:44:08 -0700 Subject: [PATCH 090/506] this should fix it, plus some discussion of why failure occurred --- lib/spack/spack/installer.py | 12 +++++++++++- lib/spack/spack/test/data/config/config.yaml | 6 ++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index b39863a0036dc9..3f57dde0affe56 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -1428,6 +1428,16 @@ def complete(self): self.record.fail(e) +if sys.platform == "win32": + # This seems like the real problem: AFAIK we don't have + # locks on Windows (we still skip the entire llnl.util.lock + # test module there) and we shouldn't be doing concurrent + # installs there. + _default_concurrent_procs = 1 +else: + _default_concurrent_procs = 4 + + class PackageInstaller: """ Class for managing the install process for a Spack instance based on a bottom-up DAG approach. @@ -1464,7 +1474,7 @@ def __init__( unsigned: Optional[bool] = None, use_cache: bool = False, verbose: bool = False, - concurrent_packages: int = 4, + concurrent_packages: int = _default_concurrent_procs, ) -> None: """ Arguments: diff --git a/lib/spack/spack/test/data/config/config.yaml b/lib/spack/spack/test/data/config/config.yaml index fc50b4b7c02f0b..2524faba3b208b 100644 --- a/lib/spack/spack/test/data/config/config.yaml +++ b/lib/spack/spack/test/data/config/config.yaml @@ -7,6 +7,12 @@ config: - $spack/lib/spack/spack/test/data/templates_again build_stage: - $tempdir/$user/spack-stage + # You can also unset these next two to "solve" the + # problem: it goes away because in a multi-package + # install, the installations subprocs won't be + # trying to simultaneously generate the cache indices + # at all (for some reason, the test config is not + # transmitted to the subprocesses on windows) source_cache: $user_cache_path/source misc_cache: $user_cache_path/cache verify_ssl: true From 32ad7e7a4c5f0c2d0d738f75b8cc0ddd25bc62c4 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 3 Jun 2025 23:59:12 -0700 Subject: [PATCH 091/506] move default fetch cache path out of user_cache_path section --- lib/spack/spack/paths.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 93994c8be037a4..34bcf2e8ff727d 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -182,6 +182,8 @@ def default_install_location(): else: envs_path = os.path.join(spack_data_home(), "environments") +default_fetch_cache_path = os.path.join(spack_data_home(), "downloads") + # TODO: we could shutil.mv resources from old paths to new paths # $spack/var/spack is generally read-only. Older instances may @@ -194,7 +196,6 @@ def default_install_location(): mock_gpg_data_path = os.path.join(var_path, "gpg.mock", "data") mock_gpg_keys_path = os.path.join(var_path, "gpg.mock", "keys") - # Below paths are where Spack can write information for the user. # Some are caches, some are not exactly caches. # @@ -210,8 +211,6 @@ def _get_user_cache_path(): user_cache_path = str(PurePath(_get_user_cache_path())) -default_fetch_cache_path = os.path.join(spack_data_home(), "downloads") - #: junit, cdash, etc. reports about builds reports_path = os.path.join(user_cache_path, "reports") From 3c7998b8e6150e7f35358484b7a8306e0ca77c71 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 4 Jun 2025 00:04:42 -0700 Subject: [PATCH 092/506] style edit --- lib/spack/spack/paths.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 34bcf2e8ff727d..f2f449e2a31aa5 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -196,6 +196,7 @@ def default_install_location(): mock_gpg_data_path = os.path.join(var_path, "gpg.mock", "data") mock_gpg_keys_path = os.path.join(var_path, "gpg.mock", "keys") + # Below paths are where Spack can write information for the user. # Some are caches, some are not exactly caches. # From ae5537757a71e1f3509010f284b5c882f2e89eb5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 4 Jun 2025 17:22:35 -0700 Subject: [PATCH 093/506] partial work: move attribute paths into class --- lib/spack/spack/paths.py | 92 +++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 43 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index f2f449e2a31aa5..08a14ffb4fdcbf 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -17,8 +17,6 @@ import spack.util.hash as hash -#: This file lives in $prefix/lib/spack/spack/__file__ -prefix = str(PurePath(llnl.util.filesystem.ancestor(__file__, 4))) xdg_config_home = "XDG_CONFIG_HOME" xdg_state_home = "XDG_STATE_HOME" @@ -101,59 +99,67 @@ def dir_is_occupied(x, except_for=None): return x.is_dir() and bool(set(x.iterdir()) - except_for) -#: User configuration location -user_config_path = _get_user_config_path() +class SpackPaths: + def __init__(self, _prefix=None): + #: This file lives in $prefix/lib/spack/spack/__file__ + self.prefix = _prefix or str(PurePath(llnl.util.filesystem.ancestor(__file__, 4))) -#: System configuration location -system_config_path = _get_system_config_path() + #: synonym for prefix + self.spack_root = self.prefix -#: When Spack is provided by an admin to a user, the admin can -#: provide a config that only applies for the end-users -end_user_cfg_path = os.path.join(system_config_path, "end-user") + #: bin directory in the spack prefix + self.bin_path = os.path.join(self.prefix, "bin") -#: synonym for prefix -spack_root = prefix + #: The spack script itself + self.spack_script = os.path.join(self.bin_path, "spack") -#: bin directory in the spack prefix -bin_path = os.path.join(prefix, "bin") + #: The sbang script in the spack installation + self.sbang_script = os.path.join(self.bin_path, "sbang") -#: The spack script itself -spack_script = os.path.join(bin_path, "spack") + # spack directory hierarchy + self.lib_path = os.path.join(self.prefix, "lib", "spack") + self.external_path = os.path.join(self.lib_path, "external") + self.module_path = os.path.join(self.lib_path, "spack") + self.command_path = os.path.join(self.module_path, "cmd") + self.analyzers_path = os.path.join(self.module_path, "analyzers") + self.platform_path = os.path.join(self.module_path, "platforms") + self.compilers_path = os.path.join(self.module_path, "compilers") + self.operating_system_path = os.path.join(self.module_path, "operating_systems") + self.test_path = os.path.join(self.module_path, "test") + self.hooks_path = os.path.join(self.module_path, "hooks") + self.share_path = os.path.join(self.prefix, "share", "spack") + self.etc_path = os.path.join(self.prefix, "etc", "spack") + self.default_license_dir = os.path.join(self.etc_path, "licenses") + self.var_path = os.path.join(self.prefix, "var", "spack") -#: The sbang script in the spack installation -sbang_script = os.path.join(bin_path, "sbang") + #: User configuration location + self.user_config_path = _get_user_config_path() -# spack directory hierarchy -lib_path = os.path.join(prefix, "lib", "spack") -external_path = os.path.join(lib_path, "external") -module_path = os.path.join(lib_path, "spack") -command_path = os.path.join(module_path, "cmd") -analyzers_path = os.path.join(module_path, "analyzers") -platform_path = os.path.join(module_path, "platforms") -compilers_path = os.path.join(module_path, "compilers") -operating_system_path = os.path.join(module_path, "operating_systems") -test_path = os.path.join(module_path, "test") -hooks_path = os.path.join(module_path, "hooks") -share_path = os.path.join(prefix, "share", "spack") -etc_path = os.path.join(prefix, "etc", "spack") -default_license_dir = os.path.join(etc_path, "licenses") -var_path = os.path.join(prefix, "var", "spack") + #: System configuration location + self.system_config_path = _get_system_config_path() + #: When Spack is provided by an admin to a user, the admin can + #: provide a config that only applies for the end-users + self.end_user_cfg_path = os.path.join(self.system_config_path, "end-user") -spack_instance_id = lambda: hash.b32_hash(prefix)[:7] + self.spack_instance_id = lambda: hash.b32_hash(self.prefix)[:7] -#: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) -default_misc_cache_path = os.path.join(spack_xdg_state_home(), spack_instance_id(), "cache") + #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) + self.default_misc_cache_path = os.path.join(spack_xdg_state_home(), self.spack_instance_id(), "cache") -#: concretization cache for Spack concretizations -default_conc_cache_path = os.path.join(default_misc_cache_path, "concretization") + #: concretization cache for Spack concretizations + self.default_conc_cache_path = os.path.join(self.default_misc_cache_path, "concretization") -modules_base = None -for module_dir in ["lmod", "modules"]: - if dir_is_occupied(os.path.join(share_path, module_dir)): - modules_base = share_path -if not modules_base: - modules_base = os.path.join(spack_xdg_data_home(), "modules") + self.modules_base = None + for module_dir in ["lmod", "modules"]: + if dir_is_occupied(os.path.join(self.share_path, module_dir)): + modules_base = self.share_path + if not self.modules_base: + self.modules_base = os.path.join(spack_xdg_data_home(), "modules") + +this_spack = SpackPaths() +for attr, value in vars(this_spack).items(): + globals()[attr] = value def default_install_location(): From 0503cc5d152f74525af5fab3a6832f1ff492048f Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 4 Jun 2025 18:08:16 -0700 Subject: [PATCH 094/506] more intermediate work --- lib/spack/spack/paths.py | 114 ++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 61 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 08a14ffb4fdcbf..d99e2be903514c 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -43,6 +43,12 @@ def _define_xdg_or_backup(xdg_var, backup): return os.path.expanduser(spack_xdg_defined) +def dir_is_occupied(x, except_for=None): + x = pathlib.Path(x) + except_for = except_for or set() + return x.is_dir() and bool(set(x.iterdir()) - except_for) + + #: Resolved XDG_STATE_HOME, with additional "spack" subdirectory spack_xdg_state_home = lambda: _define_xdg_or_backup( xdg_state_home, os.path.join("~", ".local", "state") @@ -54,50 +60,6 @@ def _define_xdg_or_backup(xdg_var, backup): xdg_data_home, os.path.join("~", ".local", "share") ) -spack_xdg_data_home_nodefault: Optional[str] -if xdg_data_home in os.environ: - spack_xdg_data_home_nodefault = os.path.expanduser( - os.path.join(os.environ[xdg_data_home], "spack") - ) -else: - spack_xdg_data_home_nodefault = None - -spack_xdg_cache_home = lambda: _define_xdg_or_backup(xdg_cache_home, os.path.join("~", ".cache")) - - -def spack_data_home(): - # spack_data_home is where we know we can put large amounts of data: - # users can set SPACK_DATA_HOME to tell spack explicitly about such - # a location. If XDG_DATA_HOME is set, we assume we can use that. - # If neither are set, we assume the spack prefix is the only place - # available to us (we do not use ~ and in particular the default for - # XDG_DATA_HOME). - if spack_data_home_varname in os.environ: - return os.environ[spack_data_home_varname] - elif spack_xdg_data_home_nodefault: - return spack_xdg_data_home_nodefault - else: - return os.path.join(prefix, "opt", "data") - - -# User configuration -def _get_user_config_path(): - return os.path.expanduser(os.getenv("SPACK_USER_CONFIG_PATH") or spack_xdg_config_home()) - - -# Configuration in /etc/spack on the system -# Override w/ `SPACK_SYSTEM_CONFIG_PATH` -def _get_system_config_path(): - return os.path.expanduser( - os.getenv("SPACK_SYSTEM_CONFIG_PATH") or os.sep + os.path.join("etc", "spack") - ) - - -def dir_is_occupied(x, except_for=None): - x = pathlib.Path(x) - except_for = except_for or set() - return x.is_dir() and bool(set(x.iterdir()) - except_for) - class SpackPaths: def __init__(self, _prefix=None): @@ -132,11 +94,25 @@ def __init__(self, _prefix=None): self.default_license_dir = os.path.join(self.etc_path, "licenses") self.var_path = os.path.join(self.prefix, "var", "spack") + # $spack/var/spack is generally read-only. Older instances may + # write gpg keys or environments into ...var/ + self.repos_path = os.path.join(self.var_path, "repos") + self.test_repos_path = os.path.join(self.var_path, "test_repos") + self.packages_path = os.path.join(self.repos_path, "spack_repo", "builtin") + self.mock_packages_path = os.path.join(self.test_repos_path, "spack_repo", "builtin_mock") + + self.mock_gpg_data_path = os.path.join(self.var_path, "gpg.mock", "data") + self.mock_gpg_keys_path = os.path.join(self.var_path, "gpg.mock", "keys") + #: User configuration location - self.user_config_path = _get_user_config_path() + self.user_config_path = os.path.expanduser( + os.getenv("SPACK_USER_CONFIG_PATH") or spack_xdg_config_home() + ) #: System configuration location - self.system_config_path = _get_system_config_path() + self.system_config_path = os.path.expanduser( + os.getenv("SPACK_SYSTEM_CONFIG_PATH") or os.sep + os.path.join("etc", "spack") + ) #: When Spack is provided by an admin to a user, the admin can #: provide a config that only applies for the end-users @@ -153,7 +129,7 @@ def __init__(self, _prefix=None): self.modules_base = None for module_dir in ["lmod", "modules"]: if dir_is_occupied(os.path.join(self.share_path, module_dir)): - modules_base = self.share_path + self.modules_base = self.share_path if not self.modules_base: self.modules_base = os.path.join(spack_xdg_data_home(), "modules") @@ -162,12 +138,40 @@ def __init__(self, _prefix=None): globals()[attr] = value + + +spack_xdg_data_home_nodefault: Optional[str] +if xdg_data_home in os.environ: + spack_xdg_data_home_nodefault = os.path.expanduser( + os.path.join(os.environ[xdg_data_home], "spack") + ) +else: + spack_xdg_data_home_nodefault = None + +spack_xdg_cache_home = lambda: _define_xdg_or_backup(xdg_cache_home, os.path.join("~", ".cache")) + + +def spack_data_home(): + # spack_data_home is where we know we can put large amounts of data: + # users can set SPACK_DATA_HOME to tell spack explicitly about such + # a location. If XDG_DATA_HOME is set, we assume we can use that. + # If neither are set, we assume the spack prefix is the only place + # available to us (we do not use ~ and in particular the default for + # XDG_DATA_HOME). + if spack_data_home_varname in os.environ: + return os.environ[spack_data_home_varname] + elif spack_xdg_data_home_nodefault: + return spack_xdg_data_home_nodefault + else: + return os.path.join(prefix, "opt", "data") + + def default_install_location(): # Precedence for installs: # 1. config:install_tree:root # 2. explicitly defined SPACK_DATA_HOME - # 3. occupied old install path (inside spack prefix) - # 4. explicitly defined XDG_DATA_HOME + # 3. explicitly defined XDG_DATA_HOME + # 4. occupied old install path (inside spack prefix) # 5. inside spack prefix (slightly different compared to old install path) old_install_path = os.path.join(prefix, "opt", "spack") if spack_data_home_varname in os.environ: @@ -190,18 +194,6 @@ def default_install_location(): default_fetch_cache_path = os.path.join(spack_data_home(), "downloads") -# TODO: we could shutil.mv resources from old paths to new paths - -# $spack/var/spack is generally read-only. Older instances may -# write gpg keys or environments into ...var/ -repos_path = os.path.join(var_path, "repos") -test_repos_path = os.path.join(var_path, "test_repos") -packages_path = os.path.join(repos_path, "spack_repo", "builtin") -mock_packages_path = os.path.join(test_repos_path, "spack_repo", "builtin_mock") - -mock_gpg_data_path = os.path.join(var_path, "gpg.mock", "data") -mock_gpg_keys_path = os.path.join(var_path, "gpg.mock", "keys") - # Below paths are where Spack can write information for the user. # Some are caches, some are not exactly caches. From f904325eea90cdb13dc079a56ed4d9f391b92e4a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 4 Jun 2025 20:53:52 -0700 Subject: [PATCH 095/506] moving more fields into object: actually implement new reordering of e.g. install prefix --- lib/spack/spack/environment/environment.py | 2 +- lib/spack/spack/paths.py | 115 +++++++++++---------- lib/spack/spack/store.py | 2 +- 3 files changed, 60 insertions(+), 59 deletions(-) diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index 7749e67944201b..467fdb8fef8505 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -61,7 +61,7 @@ #: default path where environments are stored in the spack tree -default_env_path = spack.paths.envs_path +default_env_path = spack.paths.default_envs_path #: Name of the input yaml file for an environment diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index d99e2be903514c..e80c9dd203abbc 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -60,6 +60,16 @@ def dir_is_occupied(x, except_for=None): xdg_data_home, os.path.join("~", ".local", "share") ) +spack_xdg_data_home_nodefault: Optional[str] +if xdg_data_home in os.environ: + spack_xdg_data_home_nodefault = os.path.expanduser( + os.path.join(os.environ[xdg_data_home], "spack") + ) +else: + spack_xdg_data_home_nodefault = None + +spack_xdg_cache_home = lambda: _define_xdg_or_backup(xdg_cache_home, os.path.join("~", ".cache")) + class SpackPaths: def __init__(self, _prefix=None): @@ -133,68 +143,59 @@ def __init__(self, _prefix=None): if not self.modules_base: self.modules_base = os.path.join(spack_xdg_data_home(), "modules") + # ------ + # The next 3 locations can store a lot of data and used to be + # inside of Spack by default. They can be set to any location + # with `config:` settings, and those have priority. They + # can also be redirected by setting the SPACK_DATA_HOME or + # XDG_DATA_HOME environment variables. If none of those are + # set, then they point to inside of the Spack prefix. + # + # Precedence: + # 1. config: setting (code consults the config before this + # module) + # 2. explicitly defined SPACK_DATA_HOME + # 3. explicitly defined XDG_DATA_HOME + # 4. old default path, if occupied (inside spack prefix) + # 5. inside spack prefix (slightly different compared to old + # install path) + old_install_path = os.path.join(self.prefix, "opt", "spack") + self.default_install_location = self.use_spack_data_home_or_old_location("installs", old_install_path) + + old_envs_path = os.path.join(self.var_path, "environments") + self.default_envs_path = self.use_spack_data_home_or_old_location("environments", old_envs_path) + + old_fetch_cache_path = os.path.join(self.var_path, "cache") + self.default_fetch_cache_path = self.use_spack_data_home_or_old_location("downloads", old_fetch_cache_path) + + def use_spack_data_home_or_old_location(self, subdir, old_location): + # spack_data_home is where we know we can put large amounts of data. + # Users can set SPACK_DATA_HOME to tell spack explicitly about such + # a location. If XDG_DATA_HOME is set, we assume we can use that. + # If neither are set, we assume the spack prefix is the only place + # available to us (we do not use ~ and in particular the default for + # XDG_DATA_HOME). + spack_data_home_explicit = None + if spack_data_home_varname in os.environ: + spack_data_home_explicit = os.environ[spack_data_home_varname] + elif spack_xdg_data_home_nodefault: + spack_data_home_explicit = spack_xdg_data_home_nodefault + + spack_data_home_default = os.path.join(self.prefix, "opt", "data") + + if spack_data_home_explicit: + return os.path.join(spack_data_home_explicit, subdir) + elif dir_is_occupied(old_location): + return old_location + else: + return os.path.join(spack_data_home_default, subdir) + + this_spack = SpackPaths() for attr, value in vars(this_spack).items(): globals()[attr] = value - - -spack_xdg_data_home_nodefault: Optional[str] -if xdg_data_home in os.environ: - spack_xdg_data_home_nodefault = os.path.expanduser( - os.path.join(os.environ[xdg_data_home], "spack") - ) -else: - spack_xdg_data_home_nodefault = None - -spack_xdg_cache_home = lambda: _define_xdg_or_backup(xdg_cache_home, os.path.join("~", ".cache")) - - -def spack_data_home(): - # spack_data_home is where we know we can put large amounts of data: - # users can set SPACK_DATA_HOME to tell spack explicitly about such - # a location. If XDG_DATA_HOME is set, we assume we can use that. - # If neither are set, we assume the spack prefix is the only place - # available to us (we do not use ~ and in particular the default for - # XDG_DATA_HOME). - if spack_data_home_varname in os.environ: - return os.environ[spack_data_home_varname] - elif spack_xdg_data_home_nodefault: - return spack_xdg_data_home_nodefault - else: - return os.path.join(prefix, "opt", "data") - - -def default_install_location(): - # Precedence for installs: - # 1. config:install_tree:root - # 2. explicitly defined SPACK_DATA_HOME - # 3. explicitly defined XDG_DATA_HOME - # 4. occupied old install path (inside spack prefix) - # 5. inside spack prefix (slightly different compared to old install path) - old_install_path = os.path.join(prefix, "opt", "spack") - if spack_data_home_varname in os.environ: - return os.path.join(spack_data_home(), "installs") - elif dir_is_occupied(old_install_path): - return old_install_path - else: - return os.path.join(spack_data_home(), "installs") - - -# Environments follow the same precedence rules as installs -# (the view and dev_path packages can take up significant space) -old_envs_path = os.path.join(var_path, "environments") -if spack_data_home_varname in os.environ: - envs_path = os.path.join(spack_data_home(), "environments") -elif dir_is_occupied(old_envs_path): - envs_path = old_envs_path -else: - envs_path = os.path.join(spack_data_home(), "environments") - -default_fetch_cache_path = os.path.join(spack_data_home(), "downloads") - - # Below paths are where Spack can write information for the user. # Some are caches, some are not exactly caches. # diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 27bdfade56f35f..50890a02f72e08 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -78,7 +78,7 @@ def parse_install_tree(config_dict): projections = {"all": all_projection} else: - unpadded_root = install_tree.get("root", spack.paths.default_install_location()) + unpadded_root = install_tree.get("root", spack.paths.default_install_location) unpadded_root = spack.util.path.canonicalize_path(unpadded_root) padded_length = install_tree.get("padded_length", False) From fb00eb90bf5eeadaaeaab9bd3878d8d33c799b13 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 4 Jun 2025 21:13:39 -0700 Subject: [PATCH 096/506] nearly all values relocated into object now --- lib/spack/spack/paths.py | 95 +++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 49 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index e80c9dd203abbc..84bff35ade5f3f 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -56,6 +56,7 @@ def dir_is_occupied(x, except_for=None): spack_xdg_config_home = lambda: _define_xdg_or_backup( xdg_config_home, os.path.join("~", ".config") ) +spack_xdg_cache_home = lambda: _define_xdg_or_backup(xdg_cache_home, os.path.join("~", ".cache")) spack_xdg_data_home = lambda: _define_xdg_or_backup( xdg_data_home, os.path.join("~", ".local", "share") ) @@ -68,8 +69,6 @@ def dir_is_occupied(x, except_for=None): else: spack_xdg_data_home_nodefault = None -spack_xdg_cache_home = lambda: _define_xdg_or_backup(xdg_cache_home, os.path.join("~", ".cache")) - class SpackPaths: def __init__(self, _prefix=None): @@ -143,11 +142,11 @@ def __init__(self, _prefix=None): if not self.modules_base: self.modules_base = os.path.join(spack_xdg_data_home(), "modules") - # ------ - # The next 3 locations can store a lot of data and used to be - # inside of Spack by default. They can be set to any location - # with `config:` settings, and those have priority. They - # can also be redirected by setting the SPACK_DATA_HOME or + # ------ Next section + # Spack can write a lot of data into the next 3 locations, and + # they used to be inside of Spack by default. They can be set + # to any location with `config:` settings, and those have priority. + # They can also be redirected by setting the SPACK_DATA_HOME or # XDG_DATA_HOME environment variables. If none of those are # set, then they point to inside of the Spack prefix. # @@ -168,6 +167,46 @@ def __init__(self, _prefix=None): old_fetch_cache_path = os.path.join(self.var_path, "cache") self.default_fetch_cache_path = self.use_spack_data_home_or_old_location("downloads", old_fetch_cache_path) + # ------ Next section + # Spack can write data into the following locations, but it + # isn't expected to be substantial, so Spack can choose to set + # "~" as a default. They are all organized under a single + # directory that users can refer to in config as $user_cache_path + # + # The options that start with `default_` below are overridable in + # `config.yaml`, but they default to use `user_cache_path/`. + # + # You can override the top-level directory (the user cache path) by + # setting `SPACK_USER_CACHE_PATH`. Otherwise it defaults to ~/.spack. + self.user_cache_path = str(PurePath(os.path.expanduser(os.getenv("SPACK_USER_CACHE_PATH") or spack_xdg_data_home()))) + + #: junit, cdash, etc. reports about builds + reports_path = os.path.join(self.user_cache_path, "reports") + + #: installation test (spack test) output + self.default_test_path = os.path.join(self.user_cache_path, "test") + + #: spack monitor analysis directories + self.default_monitor_path = os.path.join(reports_path, "monitor") + + #: git repositories fetched to compare commits to versions + self.user_repos_cache_path = os.path.join(self.user_cache_path, "git_repos") + + #: bootstrap store for bootstrapping clingo and other tools + self.default_user_bootstrap_path = os.path.join(self.user_cache_path, "bootstrap") + + old_gpg_path = os.path.join("prefix", "opt" "spack", "gpg") + if dir_is_occupied(old_gpg_path): + self.gpg_path = old_gpg_path + else: + self.gpg_path = os.path.join(self.user_cache_path, "gpg") + + old_gpg_keys_path = os.path.join(self.var_path, "gpg") + if dir_is_occupied(old_gpg_keys_path): + self.gpg_keys_path = old_gpg_keys_path + else: + self.gpg_keys_path = os.path.join(self.user_cache_path, "gpg-keys") + def use_spack_data_home_or_old_location(self, subdir, old_location): # spack_data_home is where we know we can put large amounts of data. # Users can set SPACK_DATA_HOME to tell spack explicitly about such @@ -196,48 +235,6 @@ def use_spack_data_home_or_old_location(self, subdir, old_location): globals()[attr] = value -# Below paths are where Spack can write information for the user. -# Some are caches, some are not exactly caches. -# -# The options that start with `default_` below are overridable in -# `config.yaml`, but they default to use `user_cache_path/`. -# -# You can override the top-level directory (the user cache path) by -# setting `SPACK_USER_CACHE_PATH`. Otherwise it defaults to ~/.spack. -# -def _get_user_cache_path(): - return os.path.expanduser(os.getenv("SPACK_USER_CACHE_PATH") or spack_xdg_data_home()) - - -user_cache_path = str(PurePath(_get_user_cache_path())) - -#: junit, cdash, etc. reports about builds -reports_path = os.path.join(user_cache_path, "reports") - -#: installation test (spack test) output -default_test_path = os.path.join(user_cache_path, "test") - -#: spack monitor analysis directories -default_monitor_path = os.path.join(reports_path, "monitor") - -#: git repositories fetched to compare commits to versions -user_repos_cache_path = os.path.join(user_cache_path, "git_repos") - -#: bootstrap store for bootstrapping clingo and other tools -default_user_bootstrap_path = os.path.join(user_cache_path, "bootstrap") - -old_gpg_path = os.path.join("prefix", "opt" "spack", "gpg") -if dir_is_occupied(old_gpg_path): - gpg_path = old_gpg_path -else: - gpg_path = os.path.join(user_cache_path, "gpg") - -old_gpg_keys_path = os.path.join(var_path, "gpg") -if dir_is_occupied(old_gpg_keys_path): - gpg_keys_path = old_gpg_keys_path -else: - gpg_keys_path = os.path.join(user_cache_path, "gpg-keys") - #: Recorded directory where spack command was originally invoked spack_working_dir = None From bba8a2c87268b1abc9e2674d83b7bb5f55eea302 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 4 Jun 2025 21:32:24 -0700 Subject: [PATCH 097/506] define section for some variables --- lib/spack/spack/paths.py | 49 ++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 84bff35ade5f3f..81a38cb5705f94 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -113,28 +113,8 @@ def __init__(self, _prefix=None): self.mock_gpg_data_path = os.path.join(self.var_path, "gpg.mock", "data") self.mock_gpg_keys_path = os.path.join(self.var_path, "gpg.mock", "keys") - #: User configuration location - self.user_config_path = os.path.expanduser( - os.getenv("SPACK_USER_CONFIG_PATH") or spack_xdg_config_home() - ) - - #: System configuration location - self.system_config_path = os.path.expanduser( - os.getenv("SPACK_SYSTEM_CONFIG_PATH") or os.sep + os.path.join("etc", "spack") - ) - - #: When Spack is provided by an admin to a user, the admin can - #: provide a config that only applies for the end-users - self.end_user_cfg_path = os.path.join(self.system_config_path, "end-user") - self.spack_instance_id = lambda: hash.b32_hash(self.prefix)[:7] - #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) - self.default_misc_cache_path = os.path.join(spack_xdg_state_home(), self.spack_instance_id(), "cache") - - #: concretization cache for Spack concretizations - self.default_conc_cache_path = os.path.join(self.default_misc_cache_path, "concretization") - self.modules_base = None for module_dir in ["lmod", "modules"]: if dir_is_occupied(os.path.join(self.share_path, module_dir)): @@ -207,6 +187,35 @@ def __init__(self, _prefix=None): else: self.gpg_keys_path = os.path.join(self.user_cache_path, "gpg-keys") + # ------ Next section + # Spack can also write data into the following locations, and their + # defaults are not controlled by SPACK/XDG_DATA_HOME or + # SPACK_USER_CACHE_PATH. Like the prior section, the data written + # into these locations isn't expected to take up much space, so in + # some cases defaults to "~" (in those cases in compliance with + # XDG defaults). + + #: User configuration location + self.user_config_path = os.path.expanduser( + os.getenv("SPACK_USER_CONFIG_PATH") or spack_xdg_config_home() + ) + + #: System configuration location + self.system_config_path = os.path.expanduser( + os.getenv("SPACK_SYSTEM_CONFIG_PATH") or os.sep + os.path.join("etc", "spack") + ) + + #: When Spack is provided by an admin to a user, the admin can + #: provide a config that only applies for the end-users + self.end_user_cfg_path = os.path.join(self.system_config_path, "end-user") + + #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) + self.default_misc_cache_path = os.path.join(spack_xdg_state_home(), self.spack_instance_id(), "misc-cache") + + #: concretization cache for Spack concretizations + self.default_conc_cache_path = os.path.join(self.default_misc_cache_path, "concretization") + + def use_spack_data_home_or_old_location(self, subdir, old_location): # spack_data_home is where we know we can put large amounts of data. # Users can set SPACK_DATA_HOME to tell spack explicitly about such From cec035b1007511d6af3cc2b51c5872e8880239a7 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 4 Jun 2025 21:39:05 -0700 Subject: [PATCH 098/506] created new subsection for paths with slightly different behavior --- lib/spack/spack/paths.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 81a38cb5705f94..95a35cab8600ff 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -115,13 +115,6 @@ def __init__(self, _prefix=None): self.spack_instance_id = lambda: hash.b32_hash(self.prefix)[:7] - self.modules_base = None - for module_dir in ["lmod", "modules"]: - if dir_is_occupied(os.path.join(self.share_path, module_dir)): - self.modules_base = self.share_path - if not self.modules_base: - self.modules_base = os.path.join(spack_xdg_data_home(), "modules") - # ------ Next section # Spack can write a lot of data into the next 3 locations, and # they used to be inside of Spack by default. They can be set @@ -175,6 +168,14 @@ def __init__(self, _prefix=None): #: bootstrap store for bootstrapping clingo and other tools self.default_user_bootstrap_path = os.path.join(self.user_cache_path, "bootstrap") + # ------ Next section + # The next three locations used to be written inside of the + # Spack prefix, and are now organized under $user_cache_path + # by default, *except* for old installs of Spack that have + # data written into the old locations (in which case, when + # they pull this update, they will continue to use those + # locations) + old_gpg_path = os.path.join("prefix", "opt" "spack", "gpg") if dir_is_occupied(old_gpg_path): self.gpg_path = old_gpg_path @@ -187,6 +188,13 @@ def __init__(self, _prefix=None): else: self.gpg_keys_path = os.path.join(self.user_cache_path, "gpg-keys") + self.modules_base = None + for module_dir in ["lmod", "modules"]: + if dir_is_occupied(os.path.join(self.share_path, module_dir)): + self.modules_base = self.share_path + if not self.modules_base: + self.modules_base = os.path.join(self.user_cache_path, "modules") + # ------ Next section # Spack can also write data into the following locations, and their # defaults are not controlled by SPACK/XDG_DATA_HOME or From 991c256c16b7589c46f904139111201ff6826d2b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 4 Jun 2025 21:42:39 -0700 Subject: [PATCH 099/506] explain spack_instance_id --- lib/spack/spack/paths.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 95a35cab8600ff..05b3847bc976d2 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -113,6 +113,9 @@ def __init__(self, _prefix=None): self.mock_gpg_data_path = os.path.join(self.var_path, "gpg.mock", "data") self.mock_gpg_keys_path = os.path.join(self.var_path, "gpg.mock", "keys") + #: Not a location itself, but used for when Spack instances + #: share the same cache base directory for caches that should + #: not be shared between those instances. self.spack_instance_id = lambda: hash.b32_hash(self.prefix)[:7] # ------ Next section From 0df34f2ea9672ecfc72ca6b1a3c268c08c496221 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 4 Jun 2025 22:26:20 -0700 Subject: [PATCH 100/506] fleshed out section comment --- lib/spack/spack/paths.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 05b3847bc976d2..9cd2ce61d02f53 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -149,11 +149,15 @@ def __init__(self, _prefix=None): # "~" as a default. They are all organized under a single # directory that users can refer to in config as $user_cache_path # - # The options that start with `default_` below are overridable in - # `config.yaml`, but they default to use `user_cache_path/`. - # # You can override the top-level directory (the user cache path) by # setting `SPACK_USER_CACHE_PATH`. Otherwise it defaults to ~/.spack. + # + # Precedence: + # 1. Config setting (not available for all of these) + # 2. SPACK_USER_CACHE_PATH + # 3. explicitly defined SPACK_DATA_HOME + # 4. explicitly defined XDG_DATA_HOME + # 5. default for XDG_DATA_HOME self.user_cache_path = str(PurePath(os.path.expanduser(os.getenv("SPACK_USER_CACHE_PATH") or spack_xdg_data_home()))) #: junit, cdash, etc. reports about builds @@ -169,6 +173,7 @@ def __init__(self, _prefix=None): self.user_repos_cache_path = os.path.join(self.user_cache_path, "git_repos") #: bootstrap store for bootstrapping clingo and other tools + #: overridden by `bootstrap:root` self.default_user_bootstrap_path = os.path.join(self.user_cache_path, "bootstrap") # ------ Next section @@ -221,9 +226,11 @@ def __init__(self, _prefix=None): self.end_user_cfg_path = os.path.join(self.system_config_path, "end-user") #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) + #: overridden by `config:misc_cache` self.default_misc_cache_path = os.path.join(spack_xdg_state_home(), self.spack_instance_id(), "misc-cache") #: concretization cache for Spack concretizations + #: overridden by `config:concretization_cache:url` self.default_conc_cache_path = os.path.join(self.default_misc_cache_path, "concretization") From 7d3c819fc1716f79ae4d87b51a4a628f82b8e2c0 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 4 Jun 2025 23:09:22 -0700 Subject: [PATCH 101/506] took spack_ off of spack_xdg_ variable names to avoid confusion --- etc/spack/defaults/config.yaml | 4 +- lib/spack/spack/paths.py | 86 +++++++++++++++++++--------------- lib/spack/spack/util/path.py | 6 +-- 3 files changed, 53 insertions(+), 43 deletions(-) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 10b56c5e1a650f..ab7dd75a279434 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -67,7 +67,7 @@ config: # identifies Spack staging to avoid accidentally wiping out non-Spack work. build_stage: - $tempdir/$user/spack-stage - - $spack_xdg_cache_home/stage + - $xdg_cache_home/stage # - $spack/var/spack/stage # Directory in which to run tests and store test results. @@ -82,7 +82,7 @@ config: # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` - misc_cache: $spack_xdg_state_home/$spack_instance_id/cache + misc_cache: $xdg_state_home/$spack_instance_id/cache # Abort downloads after this many seconds if not data is received. diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 9cd2ce61d02f53..34b850deaed7bd 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -10,28 +10,33 @@ """ import os import pathlib +from enum import Enum from pathlib import PurePath -from typing import Optional import llnl.util.filesystem import spack.util.hash as hash -xdg_config_home = "XDG_CONFIG_HOME" -xdg_state_home = "XDG_STATE_HOME" -xdg_data_home = "XDG_DATA_HOME" -spack_data_home_varname = "SPACK_DATA_HOME" -xdg_cache_home = "XDG_CACHE_HOME" +class XDG_vars(Enum): + config_home = "XDG_CONFIG_HOME" + state_home = "XDG_STATE_HOME" + data_home = "XDG_DATA_HOME" + cache_home = "XDG_CACHE_HOME" + + +class Location_vars(Enum): + user_cache_path = "USER_CACHE_PATH" + spack_data_home = "SPACK_DATA_HOME" # This is for tests that want to clean the environment of XDG_ variables that # affect spack behavior def _unset_xdg_vars(env): saved = {} - for xdg_var in [xdg_config_home, xdg_state_home, xdg_data_home, xdg_cache_home]: - if xdg_var in env: - saved[xdg_var] = env.pop(xdg_var) + for xdg_var in XDG_vars: + if xdg_var.value in env: + saved[xdg_var.value] = env.pop(xdg_var.value) return saved @@ -49,27 +54,6 @@ def dir_is_occupied(x, except_for=None): return x.is_dir() and bool(set(x.iterdir()) - except_for) -#: Resolved XDG_STATE_HOME, with additional "spack" subdirectory -spack_xdg_state_home = lambda: _define_xdg_or_backup( - xdg_state_home, os.path.join("~", ".local", "state") -) -spack_xdg_config_home = lambda: _define_xdg_or_backup( - xdg_config_home, os.path.join("~", ".config") -) -spack_xdg_cache_home = lambda: _define_xdg_or_backup(xdg_cache_home, os.path.join("~", ".cache")) -spack_xdg_data_home = lambda: _define_xdg_or_backup( - xdg_data_home, os.path.join("~", ".local", "share") -) - -spack_xdg_data_home_nodefault: Optional[str] -if xdg_data_home in os.environ: - spack_xdg_data_home_nodefault = os.path.expanduser( - os.path.join(os.environ[xdg_data_home], "spack") - ) -else: - spack_xdg_data_home_nodefault = None - - class SpackPaths: def __init__(self, _prefix=None): #: This file lives in $prefix/lib/spack/spack/__file__ @@ -116,7 +100,22 @@ def __init__(self, _prefix=None): #: Not a location itself, but used for when Spack instances #: share the same cache base directory for caches that should #: not be shared between those instances. - self.spack_instance_id = lambda: hash.b32_hash(self.prefix)[:7] + self.spack_instance_id = hash.b32_hash(self.prefix)[:7] + + # Resolved XDG_x_HOME variables, with additional "spack" subdirectory. + # Resolves to default value from XDG spec if unset. + self.xdg_state_home = _define_xdg_or_backup( + XDG_vars.state_home.value, os.path.join("~", ".local", "state") + ) + self.xdg_config_home = _define_xdg_or_backup( + XDG_vars.config_home.value, os.path.join("~", ".config") + ) + self.xdg_cache_home = _define_xdg_or_backup( + XDG_vars.cache_home.value, os.path.join("~", ".cache") + ) + self.xdg_data_home = _define_xdg_or_backup( + XDG_vars.data_home.value, os.path.join("~", ".local", "share") + ) # ------ Next section # Spack can write a lot of data into the next 3 locations, and @@ -158,7 +157,7 @@ def __init__(self, _prefix=None): # 3. explicitly defined SPACK_DATA_HOME # 4. explicitly defined XDG_DATA_HOME # 5. default for XDG_DATA_HOME - self.user_cache_path = str(PurePath(os.path.expanduser(os.getenv("SPACK_USER_CACHE_PATH") or spack_xdg_data_home()))) + self.user_cache_path = str(PurePath(os.path.expanduser(os.getenv("SPACK_USER_CACHE_PATH") or self.data_home_for_small_data()))) #: junit, cdash, etc. reports about builds reports_path = os.path.join(self.user_cache_path, "reports") @@ -213,7 +212,7 @@ def __init__(self, _prefix=None): #: User configuration location self.user_config_path = os.path.expanduser( - os.getenv("SPACK_USER_CONFIG_PATH") or spack_xdg_config_home() + os.getenv("SPACK_USER_CONFIG_PATH") or self.xdg_config_home ) #: System configuration location @@ -227,12 +226,17 @@ def __init__(self, _prefix=None): #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) #: overridden by `config:misc_cache` - self.default_misc_cache_path = os.path.join(spack_xdg_state_home(), self.spack_instance_id(), "misc-cache") + self.default_misc_cache_path = os.path.join(self.xdg_state_home, self.spack_instance_id, "misc-cache") #: concretization cache for Spack concretizations #: overridden by `config:concretization_cache:url` self.default_conc_cache_path = os.path.join(self.default_misc_cache_path, "concretization") + def data_home_for_small_data(self): + if Location_vars.spack_data_home.value in os.environ: + return os.environ[Location_vars.spack_data_home.value] + else: + return self.xdg_data_home def use_spack_data_home_or_old_location(self, subdir, old_location): # spack_data_home is where we know we can put large amounts of data. @@ -241,11 +245,17 @@ def use_spack_data_home_or_old_location(self, subdir, old_location): # If neither are set, we assume the spack prefix is the only place # available to us (we do not use ~ and in particular the default for # XDG_DATA_HOME). + xdg_data_home_nodefault = None + if XDG_vars.data_home.value in os.environ: + xdg_data_home_nodefault = os.path.expanduser( + os.path.join(os.environ[XDG_vars.data_home.value], "spack") + ) + spack_data_home_explicit = None - if spack_data_home_varname in os.environ: - spack_data_home_explicit = os.environ[spack_data_home_varname] - elif spack_xdg_data_home_nodefault: - spack_data_home_explicit = spack_xdg_data_home_nodefault + if Location_vars.spack_data_home.value in os.environ: + spack_data_home_explicit = os.environ[Location_vars.spack_data_home.value] + elif xdg_data_home_nodefault: + spack_data_home_explicit = xdg_data_home_nodefault spack_data_home_default = os.path.join(self.prefix, "opt", "data") diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index dda7844a5bf0f1..fe7f46192ff983 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -67,9 +67,9 @@ def replacements(): "user": lambda: get_user(), "tempdir": lambda: tempfile.gettempdir(), "user_cache_path": lambda: spack.paths.user_cache_path, - "spack_xdg_cache_home": lambda: spack.paths.spack_xdg_cache_home(), - "spack_xdg_state_home": lambda: spack.paths.spack_xdg_state_home(), - "spack_instance_id": lambda: spack.paths.spack_instance_id(), + "xdg_cache_home": lambda: spack.paths.xdg_cache_home, + "xdg_state_home": lambda: spack.paths.xdg_state_home, + "spack_instance_id": lambda: spack.paths.spack_instance_id, "architecture": lambda: arch, "arch": lambda: arch, "platform": lambda: arch.platform, From 140e62e2ef421dfe709140185dbc6f2525a71720 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 4 Jun 2025 23:41:06 -0700 Subject: [PATCH 102/506] simplify method --- lib/spack/spack/config.py | 2 +- lib/spack/spack/paths.py | 22 ++++++---------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 33cc839a22ad34..27d8b14737badc 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -962,7 +962,7 @@ def create_incremental() -> Generator[Configuration, None, None]: if not disable_local_config: configuration_paths.append(("user", spack.paths.user_config_path)) - per_spack_cfg = os.path.join(spack.paths.user_config_path, spack.paths.spack_instance_id()) + per_spack_cfg = os.path.join(spack.paths.user_config_path, spack.paths.spack_instance_id) configuration_paths.append(("this-spack", per_spack_cfg)) # add each scope diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 34b850deaed7bd..01de1dbb5aefa9 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -245,26 +245,16 @@ def use_spack_data_home_or_old_location(self, subdir, old_location): # If neither are set, we assume the spack prefix is the only place # available to us (we do not use ~ and in particular the default for # XDG_DATA_HOME). - xdg_data_home_nodefault = None - if XDG_vars.data_home.value in os.environ: - xdg_data_home_nodefault = os.path.expanduser( - os.path.join(os.environ[XDG_vars.data_home.value], "spack") - ) - - spack_data_home_explicit = None if Location_vars.spack_data_home.value in os.environ: - spack_data_home_explicit = os.environ[Location_vars.spack_data_home.value] - elif xdg_data_home_nodefault: - spack_data_home_explicit = xdg_data_home_nodefault - - spack_data_home_default = os.path.join(self.prefix, "opt", "data") - - if spack_data_home_explicit: - return os.path.join(spack_data_home_explicit, subdir) + return os.path.join(os.environ[Location_vars.spack_data_home.value], subdir) + elif XDG_vars.data_home.value in os.environ: + return os.path.expanduser( + os.path.join(os.environ[XDG_vars.data_home.value], "spack", subdir) + ) elif dir_is_occupied(old_location): return old_location else: - return os.path.join(spack_data_home_default, subdir) + return os.path.join(self.prefix, "opt", "data") this_spack = SpackPaths() From 26342a85780b3609883de42e2cae7d364d484d22 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 00:07:18 -0700 Subject: [PATCH 103/506] explicitly define module level variables for style checker (and do style fixes) --- lib/spack/spack/paths.py | 80 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 01de1dbb5aefa9..a499bf2faf0ac8 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -134,13 +134,19 @@ def __init__(self, _prefix=None): # 5. inside spack prefix (slightly different compared to old # install path) old_install_path = os.path.join(self.prefix, "opt", "spack") - self.default_install_location = self.use_spack_data_home_or_old_location("installs", old_install_path) + self.default_install_location = self.use_spack_data_home_or_old_location( + "installs", old_install_path + ) old_envs_path = os.path.join(self.var_path, "environments") - self.default_envs_path = self.use_spack_data_home_or_old_location("environments", old_envs_path) + self.default_envs_path = self.use_spack_data_home_or_old_location( + "environments", old_envs_path + ) old_fetch_cache_path = os.path.join(self.var_path, "cache") - self.default_fetch_cache_path = self.use_spack_data_home_or_old_location("downloads", old_fetch_cache_path) + self.default_fetch_cache_path = self.use_spack_data_home_or_old_location( + "downloads", old_fetch_cache_path + ) # ------ Next section # Spack can write data into the following locations, but it @@ -157,16 +163,22 @@ def __init__(self, _prefix=None): # 3. explicitly defined SPACK_DATA_HOME # 4. explicitly defined XDG_DATA_HOME # 5. default for XDG_DATA_HOME - self.user_cache_path = str(PurePath(os.path.expanduser(os.getenv("SPACK_USER_CACHE_PATH") or self.data_home_for_small_data()))) + self.user_cache_path = str( + PurePath( + os.path.expanduser( + os.getenv("SPACK_USER_CACHE_PATH") or self.data_home_for_small_data() + ) + ) + ) #: junit, cdash, etc. reports about builds - reports_path = os.path.join(self.user_cache_path, "reports") + self.reports_path = os.path.join(self.user_cache_path, "reports") #: installation test (spack test) output self.default_test_path = os.path.join(self.user_cache_path, "test") #: spack monitor analysis directories - self.default_monitor_path = os.path.join(reports_path, "monitor") + self.default_monitor_path = os.path.join(self.reports_path, "monitor") #: git repositories fetched to compare commits to versions self.user_repos_cache_path = os.path.join(self.user_cache_path, "git_repos") @@ -204,7 +216,7 @@ def __init__(self, _prefix=None): # ------ Next section # Spack can also write data into the following locations, and their - # defaults are not controlled by SPACK/XDG_DATA_HOME or + # defaults are not controlled by SPACK/XDG_DATA_HOME or # SPACK_USER_CACHE_PATH. Like the prior section, the data written # into these locations isn't expected to take up much space, so in # some cases defaults to "~" (in those cases in compliance with @@ -226,7 +238,9 @@ def __init__(self, _prefix=None): #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) #: overridden by `config:misc_cache` - self.default_misc_cache_path = os.path.join(self.xdg_state_home, self.spack_instance_id, "misc-cache") + self.default_misc_cache_path = os.path.join( + self.xdg_state_home, self.spack_instance_id, "misc-cache" + ) #: concretization cache for Spack concretizations #: overridden by `config:concretization_cache:url` @@ -258,6 +272,56 @@ def use_spack_data_home_or_old_location(self, subdir, old_location): this_spack = SpackPaths() + +prefix = this_spack.prefix +spack_root = this_spack.spack_root +bin_path = this_spack.bin_path +spack_script = this_spack.spack_script +sbang_script = this_spack.sbang_script +lib_path = this_spack.lib_path +external_path = this_spack.external_path +module_path = this_spack.module_path +command_path = this_spack.command_path +analyzers_path = this_spack.analyzers_path +platform_path = this_spack.platform_path +compilers_path = this_spack.compilers_path +operating_system_path = this_spack.operating_system_path +test_path = this_spack.test_path +hooks_path = this_spack.hooks_path +share_path = this_spack.share_path +etc_path = this_spack.etc_path +default_license_dir = this_spack.default_license_dir +var_path = this_spack.var_path +repos_path = this_spack.repos_path +test_repos_path = this_spack.test_repos_path +packages_path = this_spack.packages_path +mock_packages_path = this_spack.mock_packages_path +mock_gpg_data_path = this_spack.mock_gpg_data_path +mock_gpg_keys_path = this_spack.mock_gpg_keys_path +spack_instance_id = this_spack.spack_instance_id +xdg_state_home = this_spack.xdg_state_home +xdg_config_home = this_spack.xdg_config_home +xdg_cache_home = this_spack.xdg_cache_home +xdg_data_home = this_spack.xdg_data_home +default_install_location = this_spack.default_install_location +default_envs_path = this_spack.default_envs_path +default_fetch_cache_path = this_spack.default_fetch_cache_path +user_cache_path = this_spack.user_cache_path +reports_path = this_spack.reports_path +default_test_path = this_spack.default_test_path +default_monitor_path = this_spack.default_monitor_path +user_repos_cache_path = this_spack.user_repos_cache_path +default_user_bootstrap_path = this_spack.default_user_bootstrap_path +gpg_path = this_spack.gpg_path +gpg_keys_path = this_spack.gpg_keys_path +modules_base = this_spack.modules_base +user_config_path = this_spack.user_config_path +system_config_path = this_spack.system_config_path +end_user_cfg_path = this_spack.end_user_cfg_path +default_misc_cache_path = this_spack.default_misc_cache_path +default_conc_cache_path = this_spack.default_conc_cache_path + + for attr, value in vars(this_spack).items(): globals()[attr] = value From 1107d5b87606e52481cd5b3074f3c560180bad5f Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 00:11:57 -0700 Subject: [PATCH 104/506] calling object this_spack was making style check insert spack import --- lib/spack/spack/paths.py | 102 +++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 53 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index a499bf2faf0ac8..08731851c120db 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -271,59 +271,55 @@ def use_spack_data_home_or_old_location(self, subdir, old_location): return os.path.join(self.prefix, "opt", "data") -this_spack = SpackPaths() - -prefix = this_spack.prefix -spack_root = this_spack.spack_root -bin_path = this_spack.bin_path -spack_script = this_spack.spack_script -sbang_script = this_spack.sbang_script -lib_path = this_spack.lib_path -external_path = this_spack.external_path -module_path = this_spack.module_path -command_path = this_spack.command_path -analyzers_path = this_spack.analyzers_path -platform_path = this_spack.platform_path -compilers_path = this_spack.compilers_path -operating_system_path = this_spack.operating_system_path -test_path = this_spack.test_path -hooks_path = this_spack.hooks_path -share_path = this_spack.share_path -etc_path = this_spack.etc_path -default_license_dir = this_spack.default_license_dir -var_path = this_spack.var_path -repos_path = this_spack.repos_path -test_repos_path = this_spack.test_repos_path -packages_path = this_spack.packages_path -mock_packages_path = this_spack.mock_packages_path -mock_gpg_data_path = this_spack.mock_gpg_data_path -mock_gpg_keys_path = this_spack.mock_gpg_keys_path -spack_instance_id = this_spack.spack_instance_id -xdg_state_home = this_spack.xdg_state_home -xdg_config_home = this_spack.xdg_config_home -xdg_cache_home = this_spack.xdg_cache_home -xdg_data_home = this_spack.xdg_data_home -default_install_location = this_spack.default_install_location -default_envs_path = this_spack.default_envs_path -default_fetch_cache_path = this_spack.default_fetch_cache_path -user_cache_path = this_spack.user_cache_path -reports_path = this_spack.reports_path -default_test_path = this_spack.default_test_path -default_monitor_path = this_spack.default_monitor_path -user_repos_cache_path = this_spack.user_repos_cache_path -default_user_bootstrap_path = this_spack.default_user_bootstrap_path -gpg_path = this_spack.gpg_path -gpg_keys_path = this_spack.gpg_keys_path -modules_base = this_spack.modules_base -user_config_path = this_spack.user_config_path -system_config_path = this_spack.system_config_path -end_user_cfg_path = this_spack.end_user_cfg_path -default_misc_cache_path = this_spack.default_misc_cache_path -default_conc_cache_path = this_spack.default_conc_cache_path - - -for attr, value in vars(this_spack).items(): - globals()[attr] = value +locations = SpackPaths() + +prefix = locations.prefix +spack_root = locations.spack_root +bin_path = locations.bin_path +spack_script = locations.spack_script +sbang_script = locations.sbang_script +lib_path = locations.lib_path +external_path = locations.external_path +module_path = locations.module_path +command_path = locations.command_path +analyzers_path = locations.analyzers_path +platform_path = locations.platform_path +compilers_path = locations.compilers_path +operating_system_path = locations.operating_system_path +test_path = locations.test_path +hooks_path = locations.hooks_path +share_path = locations.share_path +etc_path = locations.etc_path +default_license_dir = locations.default_license_dir +var_path = locations.var_path +repos_path = locations.repos_path +test_repos_path = locations.test_repos_path +packages_path = locations.packages_path +mock_packages_path = locations.mock_packages_path +mock_gpg_data_path = locations.mock_gpg_data_path +mock_gpg_keys_path = locations.mock_gpg_keys_path +spack_instance_id = locations.spack_instance_id +xdg_state_home = locations.xdg_state_home +xdg_config_home = locations.xdg_config_home +xdg_cache_home = locations.xdg_cache_home +xdg_data_home = locations.xdg_data_home +default_install_location = locations.default_install_location +default_envs_path = locations.default_envs_path +default_fetch_cache_path = locations.default_fetch_cache_path +user_cache_path = locations.user_cache_path +reports_path = locations.reports_path +default_test_path = locations.default_test_path +default_monitor_path = locations.default_monitor_path +user_repos_cache_path = locations.user_repos_cache_path +default_user_bootstrap_path = locations.default_user_bootstrap_path +gpg_path = locations.gpg_path +gpg_keys_path = locations.gpg_keys_path +modules_base = locations.modules_base +user_config_path = locations.user_config_path +system_config_path = locations.system_config_path +end_user_cfg_path = locations.end_user_cfg_path +default_misc_cache_path = locations.default_misc_cache_path +default_conc_cache_path = locations.default_conc_cache_path #: Recorded directory where spack command was originally invoked From 4942fd9be65d8e9eac2b89a8e7e45a7c0a497de3 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 00:17:01 -0700 Subject: [PATCH 105/506] rename method --- lib/spack/spack/paths.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 08731851c120db..e70d15046d4a6c 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -134,17 +134,17 @@ def __init__(self, _prefix=None): # 5. inside spack prefix (slightly different compared to old # install path) old_install_path = os.path.join(self.prefix, "opt", "spack") - self.default_install_location = self.use_spack_data_home_or_old_location( + self.default_install_location = self.data_home_for_large_data( "installs", old_install_path ) old_envs_path = os.path.join(self.var_path, "environments") - self.default_envs_path = self.use_spack_data_home_or_old_location( + self.default_envs_path = self.data_home_for_large_data( "environments", old_envs_path ) old_fetch_cache_path = os.path.join(self.var_path, "cache") - self.default_fetch_cache_path = self.use_spack_data_home_or_old_location( + self.default_fetch_cache_path = self.data_home_for_large_data( "downloads", old_fetch_cache_path ) @@ -252,13 +252,7 @@ def data_home_for_small_data(self): else: return self.xdg_data_home - def use_spack_data_home_or_old_location(self, subdir, old_location): - # spack_data_home is where we know we can put large amounts of data. - # Users can set SPACK_DATA_HOME to tell spack explicitly about such - # a location. If XDG_DATA_HOME is set, we assume we can use that. - # If neither are set, we assume the spack prefix is the only place - # available to us (we do not use ~ and in particular the default for - # XDG_DATA_HOME). + def data_home_for_large_data(self, subdir, old_location): if Location_vars.spack_data_home.value in os.environ: return os.path.join(os.environ[Location_vars.spack_data_home.value], subdir) elif XDG_vars.data_home.value in os.environ: From 7c3c0a02e9410edd73007378f4c39e1fc19fc6b8 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 00:18:27 -0700 Subject: [PATCH 106/506] style fix --- lib/spack/spack/paths.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index e70d15046d4a6c..a54b4439d5cebe 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -134,14 +134,10 @@ def __init__(self, _prefix=None): # 5. inside spack prefix (slightly different compared to old # install path) old_install_path = os.path.join(self.prefix, "opt", "spack") - self.default_install_location = self.data_home_for_large_data( - "installs", old_install_path - ) + self.default_install_location = self.data_home_for_large_data("installs", old_install_path) old_envs_path = os.path.join(self.var_path, "environments") - self.default_envs_path = self.data_home_for_large_data( - "environments", old_envs_path - ) + self.default_envs_path = self.data_home_for_large_data("environments", old_envs_path) old_fetch_cache_path = os.path.join(self.var_path, "cache") self.default_fetch_cache_path = self.data_home_for_large_data( From a44004359585891dc7cc571f6539530391ffef79 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 00:32:39 -0700 Subject: [PATCH 107/506] a test --- lib/spack/spack/test/paths.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 lib/spack/spack/test/paths.py diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py new file mode 100644 index 00000000000000..252bf51e9617be --- /dev/null +++ b/lib/spack/spack/test/paths.py @@ -0,0 +1,21 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import spack.paths as paths +import os +import pathlib + + +def test_install_location(working_env, tmpdir): + base_prefix = str(tmpdir.join("prefix").ensure(dir=True)) + xdg_data_home = str(tmpdir.join("xdg_data_home")) + os.environ["XDG_DATA_HOME"] = xdg_data_home + p1 = paths.SpackPaths(base_prefix) + assert p1.default_install_location == str(pathlib.Path(xdg_data_home) / "spack" / "installs") + + # Check that SPACK_DATA_HOME overrides + spack_data_home = str(tmpdir.join("spack_data_home")) + os.environ["SPACK_DATA_HOME"] = spack_data_home + p2 = paths.SpackPaths(base_prefix) + assert p2.default_install_location == str(pathlib.Path(spack_data_home) / "installs") \ No newline at end of file From 0359ed60bf3f6cd85e443db7595bf53e62c1d4e8 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 00:33:39 -0700 Subject: [PATCH 108/506] style fix --- lib/spack/spack/test/paths.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 252bf51e9617be..6ca79e985a0b8b 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -2,10 +2,11 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import spack.paths as paths import os import pathlib +import spack.paths as paths + def test_install_location(working_env, tmpdir): base_prefix = str(tmpdir.join("prefix").ensure(dir=True)) @@ -18,4 +19,4 @@ def test_install_location(working_env, tmpdir): spack_data_home = str(tmpdir.join("spack_data_home")) os.environ["SPACK_DATA_HOME"] = spack_data_home p2 = paths.SpackPaths(base_prefix) - assert p2.default_install_location == str(pathlib.Path(spack_data_home) / "installs") \ No newline at end of file + assert p2.default_install_location == str(pathlib.Path(spack_data_home) / "installs") From 50def049ac720a8db5d9149e0cccc787a71dd04e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 01:14:50 -0700 Subject: [PATCH 109/506] re-add explanation of env vars controlling config --- lib/spack/spack/paths.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 0fc4dd78c295b6..47f43115cd42f7 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -221,6 +221,12 @@ def __init__(self, _prefix=None): # some cases defaults to "~" (in those cases in compliance with # XDG defaults). + # There are three environment variables you can use to isolate spack from + # the host environment: + # - `SPACK_USER_CONFIG_PATH`: override `~/.spack` location (for config and caches) + # - `SPACK_SYSTEM_CONFIG_PATH`: override `/etc/spack` configuration scope. + # - `SPACK_DISABLE_LOCAL_CONFIG`: disable both of these locations. + #: User configuration location self.user_config_path = os.path.expanduser( os.getenv("SPACK_USER_CONFIG_PATH") or self.xdg_config_home From d0ceeb6f1c2b1e21209b15426a320fc9935b8bb1 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 01:29:17 -0700 Subject: [PATCH 110/506] forgot command completion --- share/spack/spack-completion.fish | 58 +++++++++++++++++-------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 73f0d1e1222b3a..0c8a5b643a8a19 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -346,7 +346,7 @@ complete -c spack --erase # Everything below here is auto-generated. # spack -set -g __fish_spack_optspecs_spack h/help H/all-help color= c/config= C/config-scope= d/debug timestamp pdb e/env= D/env-dir= E/no-env use-env-repo k/insecure l/enable-locks L/disable-locks m/mock b/bootstrap p/profile sorted-profile= lines= v/verbose stacktrace t/backtrace V/version print-shell-vars= +set -g __fish_spack_optspecs_spack h/help H/all-help color= c/config= C/config-scope= d/debug timestamp pdb e/env= D/env-dir= E/no-env use-env-repo k/insecure l/enable-locks L/disable-locks m/mock b/bootstrap p/profile sorted-profile= lines= v/verbose stacktrace t/backtrace V/version print-shell-vars= disable-end-user-config warn-writes-into-spack complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a add -d 'add a spec to an environment' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a arch -d 'print architecture information about this machine' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a audit -d 'audit configuration files, packages, etc.' @@ -478,6 +478,10 @@ complete -c spack -n '__fish_spack_using_command ' -s V -l version -f -a version complete -c spack -n '__fish_spack_using_command ' -s V -l version -d 'show version number and exit' complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -f -a print_shell_vars complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -d 'print info needed by setup-env.*sh' +complete -c spack -n '__fish_spack_using_command ' -l disable-end-user-config -f -a disable_end_user_config +complete -c spack -n '__fish_spack_using_command ' -l disable-end-user-config -d 'Disable system config scope for end users' +complete -c spack -n '__fish_spack_using_command ' -l warn-writes-into-spack -f -a warn_writes_into_spack +complete -c spack -n '__fish_spack_using_command ' -l warn-writes-into-spack -d 'Warn when Spack tries to write into its own prefix' # spack add set -g __fish_spack_optspecs_spack_add h/help l/list-name= @@ -602,7 +606,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_enable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap enable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap disable @@ -610,7 +614,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_disable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap disable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap reset @@ -625,14 +629,14 @@ set -g __fish_spack_optspecs_spack_bootstrap_root h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap root' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap list set -g __fish_spack_optspecs_spack_bootstrap_list h/help scope= complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap add @@ -641,7 +645,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap add' -f -a '(__ complete -c spack -n '__fish_spack_using_command_pos 1 bootstrap add' -f -a '(__fish_spack_environments)' complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -f -a trust complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -d 'enable the source immediately upon addition' @@ -815,7 +819,7 @@ complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirro complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -d 'override any configured mirrors with this mirror URL' complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -f -a output_file complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -d 'file where rebuild info should be written' -complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -d 'configuration scope containing mirrors to check' # spack buildcache download @@ -1081,7 +1085,7 @@ complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolcha complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolchain -d '(DEPRECATED) Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -d '(DEPRECATED) Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1095,7 +1099,7 @@ complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchai complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchain -d '(DEPRECATED) Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -d '(DEPRECATED) Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1107,7 +1111,7 @@ complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help - complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -d 'configuration scope to modify' # spack compiler rm @@ -1117,14 +1121,14 @@ complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -d 'configuration scope to modify' # spack compiler list set -g __fish_spack_optspecs_spack_compiler_list h/help scope= remote complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler list' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler list' -l remote -d 'list also compilers from registered buildcaches' @@ -1133,7 +1137,7 @@ complete -c spack -n '__fish_spack_using_command compiler list' -l remote -d 'li set -g __fish_spack_optspecs_spack_compiler_ls h/help scope= remote complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler ls' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler ls' -l remote -d 'list also compilers from registered buildcaches' @@ -1143,14 +1147,14 @@ set -g __fish_spack_optspecs_spack_compiler_info h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 compiler info' -f -a '(__fish_spack_installed_compilers)' complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -d 'configuration scope to read from' # spack compilers set -g __fish_spack_optspecs_spack_compilers h/help scope= complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -d 'configuration scope to read/modify' # spack concretize @@ -1211,7 +1215,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a update -d ' complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a revert -d 'revert configuration files to their state before update' complete -c spack -n '__fish_spack_using_command config' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify' # spack config get @@ -1759,7 +1763,7 @@ complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -f complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -d 'packages to exclude from search' complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -f -a path complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -d 'one or more alternative search paths for finding externals' -complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command external find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command external find' -l all -f -a all complete -c spack -n '__fish_spack_using_command external find' -l all -d 'search for all packages that Spack knows about' @@ -2335,7 +2339,7 @@ set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsig complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use `--type binary --type source` (default)' @@ -2375,7 +2379,7 @@ set -g __fish_spack_optspecs_spack_mirror_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror remove' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -d 'configuration scope to modify' # spack mirror rm @@ -2383,7 +2387,7 @@ set -g __fish_spack_optspecs_spack_mirror_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror rm' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -d 'configuration scope to modify' # spack mirror set-url @@ -2395,7 +2399,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -f -a p complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -d 'set only the URL used for uploading' complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -f -a fetch complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -d 'set only the URL used for downloading' -complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2443,7 +2447,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a s complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache' complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed complete -c spack -n '__fish_spack_using_command mirror set' -l signed -d 'require signing and signature verification when pushing and installing from this build cache' -complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2474,7 +2478,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password-var set -g __fish_spack_optspecs_spack_mirror_list h/help scope= complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'configuration scope to read from' # spack module @@ -2782,7 +2786,7 @@ complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirecto set -g __fish_spack_optspecs_spack_repo_list h/help scope= complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'configuration scope to read from' # spack repo add @@ -2794,7 +2798,7 @@ complete -c spack -n '__fish_spack_using_command repo add' -l name -r -f -a name complete -c spack -n '__fish_spack_using_command repo add' -l name -r -d 'config name for the package repository, defaults to the namespace of the repository' complete -c spack -n '__fish_spack_using_command repo add' -l path -r -f -a path complete -c spack -n '__fish_spack_using_command repo add' -l path -r -d 'relative path to the Spack package repository inside a git repository. Can be repeated to add multiple package repositories in case of a monorepo' -complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -d 'configuration scope to modify' # spack repo remove @@ -2802,7 +2806,7 @@ set -g __fish_spack_optspecs_spack_repo_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo remove' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -d 'configuration scope to modify' # spack repo rm @@ -2810,7 +2814,7 @@ set -g __fish_spack_optspecs_spack_repo_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo rm' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify' # spack repo migrate From 34bf0d66ff0c6a3b922fb8c0d30877621fb8fb40 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 10:08:39 -0700 Subject: [PATCH 111/506] move some tests for paths from config module to paths test module - modify them not to use private members --- lib/spack/spack/test/config.py | 25 ------------------------- lib/spack/spack/test/paths.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index b48dff4c4ac82d..479c99e5824353 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1203,31 +1203,6 @@ def test_internal_config_scope_cache_clearing(): assert internal_scope.sections["config"] == data -def test_system_config_path_is_overridable(working_env): - p = "/some/path" - os.environ["SPACK_SYSTEM_CONFIG_PATH"] = p - assert spack.paths._get_system_config_path() == p - - -def test_system_config_path_is_default_when_env_var_is_empty(working_env): - os.environ["SPACK_SYSTEM_CONFIG_PATH"] = "" - assert os.sep + os.path.join("etc", "spack") == spack.paths._get_system_config_path() - - -def test_user_config_path_is_overridable(working_env): - p = "/some/path" - os.environ["SPACK_USER_CONFIG_PATH"] = p - assert p == spack.paths._get_user_config_path() - - -def test_user_config_path_is_default_when_env_var_is_empty(working_env): - os.environ["SPACK_USER_CONFIG_PATH"] = "" - assert ( - os.path.expanduser(os.path.join("~", ".config", "spack")) - == spack.paths._get_user_config_path() - ) - - def test_default_install_tree(monkeypatch, default_config): s = spack.spec.Spec("nonexistent@x.y.z arch=foo-bar-baz") monkeypatch.setattr(s, "dag_hash", lambda length: "abc123") diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 6ca79e985a0b8b..473438dda4e2f3 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -20,3 +20,32 @@ def test_install_location(working_env, tmpdir): os.environ["SPACK_DATA_HOME"] = spack_data_home p2 = paths.SpackPaths(base_prefix) assert p2.default_install_location == str(pathlib.Path(spack_data_home) / "installs") + + +def test_system_config_path_is_overridable(working_env, tmpdir): + redirect_syscfg_path = "/some/path" + os.environ["SPACK_SYSTEM_CONFIG_PATH"] = redirect_syscfg_path + p1 = paths.SpackPaths(str(tmpdir)) + assert p1.system_config_path == redirect_syscfg_path + + +def test_system_config_path_is_default_when_env_var_is_empty(working_env, tmpdir): + os.environ["SPACK_SYSTEM_CONFIG_PATH"] = "" + p1 = paths.SpackPaths(str(tmpdir)) + assert os.sep + os.path.join("etc", "spack") == p1.system_config_path + + +def test_user_config_path_is_overridable(working_env, tmpdir): + redirect_usrcfg_path = "/some/path" + os.environ["SPACK_USER_CONFIG_PATH"] = redirect_usrcfg_path + p1 = paths.SpackPaths(str(tmpdir)) + assert p1.user_config_path == redirect_usrcfg_path + + +def test_user_config_path_is_default_when_env_var_is_empty(working_env, tmpdir): + os.environ["SPACK_USER_CONFIG_PATH"] = "" + p1 = paths.SpackPaths(str(tmpdir)) + assert ( + os.path.expanduser(os.path.join("~", ".config", "spack")) + == p1.user_config_path + ) \ No newline at end of file From a77520f1525d15083f118b3e80e876b35c761e38 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 10:09:42 -0700 Subject: [PATCH 112/506] style fix --- lib/spack/spack/test/paths.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 473438dda4e2f3..866d1df05b659d 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -45,7 +45,4 @@ def test_user_config_path_is_overridable(working_env, tmpdir): def test_user_config_path_is_default_when_env_var_is_empty(working_env, tmpdir): os.environ["SPACK_USER_CONFIG_PATH"] = "" p1 = paths.SpackPaths(str(tmpdir)) - assert ( - os.path.expanduser(os.path.join("~", ".config", "spack")) - == p1.user_config_path - ) \ No newline at end of file + assert os.path.expanduser(os.path.join("~", ".config", "spack")) == p1.user_config_path From 84f83e0b049acf5dd8ed9a9269e65c2a7d0bbdac Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 10:36:07 -0700 Subject: [PATCH 113/506] incomplete rename --- lib/spack/spack/util/path.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index fe7f46192ff983..4ff87b172abfeb 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -165,8 +165,8 @@ def substitute_config_variables(path): - $tempdir Default temporary directory returned by tempfile.gettempdir() - $user The current user's username - $user_cache_path The user cache directory ($XDG_DATA_HOME, unless overridden) - - $spack_xdg_cache_home Backup location for temporary data - - $spack_xdg_state_home Long-lived but not-essential cache + - $xdg_cache_home XDG location for temporary data + - $xdg_state_home XDG long-lived but not-essential cache - $spack_instance_id Hash that distinguishes Spack instances on the filesystem - $architecture The spack architecture triple for the current system - $arch The spack architecture triple for the current system From 7c8b14e0d96b30c31d8062df1bb973c636d49cd8 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 11:02:06 -0700 Subject: [PATCH 114/506] comment was out of date --- lib/spack/spack/paths.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 7dbe6eb177365a..8ae8e18e322cd6 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -150,7 +150,9 @@ def __init__(self, _prefix=None): # directory that users can refer to in config as $user_cache_path # # You can override the top-level directory (the user cache path) by - # setting `SPACK_USER_CACHE_PATH`. Otherwise it defaults to ~/.spack. + # setting `SPACK_USER_CACHE_PATH`, `SPACK_DATA_HOME`, or + # `XDG_DATA_HOME`; if none of those are set, then the default for + # `XDG_DATA_HOME` is used (~/.local/share). # # Precedence: # 1. Config setting (not available for all of these) From 96254b9e3282cd3d91cc2778ddeeaddfbdb4a14f Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 12:51:52 -0700 Subject: [PATCH 115/506] partial --- etc/spack/defaults/config.yaml | 8 +++++++- etc/spack/defaults/modules.yaml | 4 ++++ lib/spack/spack/paths.py | 25 ++++++++++++++++++++----- lib/spack/spack/util/path.py | 2 ++ 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index ab7dd75a279434..da20468676dbaf 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -17,6 +17,7 @@ config: # This is the path to the root of the Spack install tree. # You can use $spack here to refer to the root of the spack instance. install_tree: + root: $large_data_home/installs projections: all: "{architecture.platform}-{architecture.target}/{name}-{version}-{hash}" # install_tree can include an optional padded length (int or boolean) @@ -76,8 +77,13 @@ config: test_stage: $user_cache_path/test + # Cache directory for already downloaded source tarballs and archived + # repositories. This can be purged with `spack clean --downloads`. + source_cache: $large_data_home/downloads + + ## Directory where spack managed environments are created and stored - # environments_root: $spack/var/spack/environments + # environments_root: $large_data_home/environments # Cache directory for miscellaneous files, like the package index. diff --git a/etc/spack/defaults/modules.yaml b/etc/spack/defaults/modules.yaml index ee89695b227671..b5cb0e83a3f0a9 100644 --- a/etc/spack/defaults/modules.yaml +++ b/etc/spack/defaults/modules.yaml @@ -36,6 +36,10 @@ modules: # These are configurations for the module set named "default" default: + # Where to install modules + roots: + tcl: $modules_base/tcl + lmod: $modules_base/lmod # What type of modules to use ("tcl" and/or "lmod") enable: [] diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 8ae8e18e322cd6..eb6e7108cc5def 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -133,16 +133,21 @@ def __init__(self, _prefix=None): # 5. inside spack prefix (slightly different compared to old # install path) old_install_path = os.path.join(self.prefix, "opt", "spack") - self.default_install_location = self.data_home_for_large_data("installs", old_install_path) + self.default_install_location = self.large_data_component("installs", old_install_path) old_envs_path = os.path.join(self.var_path, "environments") - self.default_envs_path = self.data_home_for_large_data("environments", old_envs_path) + self.default_envs_path = self.large_data_component("environments", old_envs_path) old_fetch_cache_path = os.path.join(self.var_path, "cache") - self.default_fetch_cache_path = self.data_home_for_large_data( + self.default_fetch_cache_path = self.large_data_component( "downloads", old_fetch_cache_path ) + # Not an explicit destination, but rather the bucket that holds + # the above three items, unless they are individually set with + # config: values + self.large_data_home = self.data_home_for_large_data() + # ------ Next section # Spack can write data into the following locations, but it # isn't expected to be substantial, so Spack can choose to set @@ -258,7 +263,7 @@ def data_home_for_small_data(self): else: return self.xdg_data_home - def data_home_for_large_data(self, subdir, old_location): + def large_data_component(self, subdir, old_location): if Location_vars.spack_data_home.value in os.environ: return os.path.join(os.environ[Location_vars.spack_data_home.value], subdir) elif XDG_vars.data_home.value in os.environ: @@ -270,6 +275,16 @@ def data_home_for_large_data(self, subdir, old_location): else: return os.path.join(self.prefix, "opt", "data") + def data_home_for_large_data(self): + if Location_vars.spack_data_home.value in os.environ: + return os.path.join(os.environ[Location_vars.spack_data_home.value], subdir) + elif XDG_vars.data_home.value in os.environ: + return os.path.expanduser( + os.path.join(os.environ[XDG_vars.data_home.value], "spack", subdir) + ) + else: + return os.path.join(self.prefix, "opt", "data") + locations = SpackPaths() @@ -302,7 +317,7 @@ def data_home_for_large_data(self, subdir, old_location): xdg_config_home = locations.xdg_config_home xdg_cache_home = locations.xdg_cache_home xdg_data_home = locations.xdg_data_home -default_install_location = locations.default_install_location +large_data_home = locations.large_data_home default_envs_path = locations.default_envs_path default_fetch_cache_path = locations.default_fetch_cache_path user_cache_path = locations.user_cache_path diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 4ff87b172abfeb..064666dcda8831 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -67,6 +67,7 @@ def replacements(): "user": lambda: get_user(), "tempdir": lambda: tempfile.gettempdir(), "user_cache_path": lambda: spack.paths.user_cache_path, + "large_data_home": lambda: spack.paths.large_data_home, "xdg_cache_home": lambda: spack.paths.xdg_cache_home, "xdg_state_home": lambda: spack.paths.xdg_state_home, "spack_instance_id": lambda: spack.paths.spack_instance_id, @@ -165,6 +166,7 @@ def substitute_config_variables(path): - $tempdir Default temporary directory returned by tempfile.gettempdir() - $user The current user's username - $user_cache_path The user cache directory ($XDG_DATA_HOME, unless overridden) + - $large_data_home Where spack can write large amounts of persistent data - $xdg_cache_home XDG location for temporary data - $xdg_state_home XDG long-lived but not-essential cache - $spack_instance_id Hash that distinguishes Spack instances on the filesystem From 5d2102087e5dbac29d96d16741015a50aed96c6a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 13:18:05 -0700 Subject: [PATCH 116/506] reintroduce default config via new variables --- etc/spack/defaults/config.yaml | 6 ++--- etc/spack/defaults/modules.yaml | 2 +- lib/spack/spack/paths.py | 17 +------------ lib/spack/spack/util/path.py | 42 ++++++++++++++++++--------------- 4 files changed, 28 insertions(+), 39 deletions(-) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index da20468676dbaf..d19f0af685d7bd 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -17,7 +17,7 @@ config: # This is the path to the root of the Spack install tree. # You can use $spack here to refer to the root of the spack instance. install_tree: - root: $large_data_home/installs + root: $default_install_root/installs projections: all: "{architecture.platform}-{architecture.target}/{name}-{version}-{hash}" # install_tree can include an optional padded length (int or boolean) @@ -79,11 +79,11 @@ config: # Cache directory for already downloaded source tarballs and archived # repositories. This can be purged with `spack clean --downloads`. - source_cache: $large_data_home/downloads + source_cache: $default_download_root ## Directory where spack managed environments are created and stored - # environments_root: $large_data_home/environments + # environments_root: $default_envs_root/environments # Cache directory for miscellaneous files, like the package index. diff --git a/etc/spack/defaults/modules.yaml b/etc/spack/defaults/modules.yaml index b5cb0e83a3f0a9..d66a13f26bb6e3 100644 --- a/etc/spack/defaults/modules.yaml +++ b/etc/spack/defaults/modules.yaml @@ -38,7 +38,7 @@ modules: default: # Where to install modules roots: - tcl: $modules_base/tcl + tcl: $modules_base/modules lmod: $modules_base/lmod # What type of modules to use ("tcl" and/or "lmod") enable: [] diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index eb6e7108cc5def..b3ef16dc848e70 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -143,11 +143,6 @@ def __init__(self, _prefix=None): "downloads", old_fetch_cache_path ) - # Not an explicit destination, but rather the bucket that holds - # the above three items, unless they are individually set with - # config: values - self.large_data_home = self.data_home_for_large_data() - # ------ Next section # Spack can write data into the following locations, but it # isn't expected to be substantial, so Spack can choose to set @@ -275,16 +270,6 @@ def large_data_component(self, subdir, old_location): else: return os.path.join(self.prefix, "opt", "data") - def data_home_for_large_data(self): - if Location_vars.spack_data_home.value in os.environ: - return os.path.join(os.environ[Location_vars.spack_data_home.value], subdir) - elif XDG_vars.data_home.value in os.environ: - return os.path.expanduser( - os.path.join(os.environ[XDG_vars.data_home.value], "spack", subdir) - ) - else: - return os.path.join(self.prefix, "opt", "data") - locations = SpackPaths() @@ -317,7 +302,7 @@ def data_home_for_large_data(self): xdg_config_home = locations.xdg_config_home xdg_cache_home = locations.xdg_cache_home xdg_data_home = locations.xdg_data_home -large_data_home = locations.large_data_home +default_install_location = locations.default_install_location default_envs_path = locations.default_envs_path default_fetch_cache_path = locations.default_fetch_cache_path user_cache_path = locations.user_cache_path diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 064666dcda8831..bd64619dae37a1 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -67,7 +67,9 @@ def replacements(): "user": lambda: get_user(), "tempdir": lambda: tempfile.gettempdir(), "user_cache_path": lambda: spack.paths.user_cache_path, - "large_data_home": lambda: spack.paths.large_data_home, + "default_install_root": lambda: spack.paths.default_install_location, + "default_envs_root": lambda: spack.paths.default_envs_path, + "default_download_root": lambda: spack.paths.default_fetch_cache_path, "xdg_cache_home": lambda: spack.paths.xdg_cache_home, "xdg_state_home": lambda: spack.paths.xdg_state_home, "spack_instance_id": lambda: spack.paths.spack_instance_id, @@ -161,24 +163,26 @@ def substitute_config_variables(path): Spack allows paths in configs to have some placeholders, as follows: - - $env The active Spack environment. - - $spack The Spack instance's prefix - - $tempdir Default temporary directory returned by tempfile.gettempdir() - - $user The current user's username - - $user_cache_path The user cache directory ($XDG_DATA_HOME, unless overridden) - - $large_data_home Where spack can write large amounts of persistent data - - $xdg_cache_home XDG location for temporary data - - $xdg_state_home XDG long-lived but not-essential cache - - $spack_instance_id Hash that distinguishes Spack instances on the filesystem - - $architecture The spack architecture triple for the current system - - $arch The spack architecture triple for the current system - - $platform The spack platform for the current system - - $os The OS of the current system - - $operating_system The OS of the current system - - $target The ISA target detected for the system - - $target_family The family of the target detected for the system - - $date The current date (YYYY-MM-DD) - - $spack_short_version The spack short version + - $env The active Spack environment. + - $spack The Spack instance's prefix + - $tempdir Default temporary directory returned by tempfile.gettempdir() + - $user The current user's username + - $user_cache_path The user cache directory ($XDG_DATA_HOME, unless overridden) + - $default_install_root Where installs go by default + - $default_envs_root Where environments are managed by default + - $default_download_root Where downloads go by default + - $xdg_cache_home XDG location for temporary data + - $xdg_state_home XDG long-lived but not-essential cache + - $spack_instance_id Hash that distinguishes Spack instances on the filesystem + - $architecture The spack architecture triple for the current system + - $arch The spack architecture triple for the current system + - $platform The spack platform for the current system + - $os The OS of the current system + - $operating_system The OS of the current system + - $target The ISA target detected for the system + - $target_family The family of the target detected for the system + - $date The current date (YYYY-MM-DD) + - $spack_short_version The spack short version These are substituted case-insensitively into the path, and users can use either ``$var`` or ``${var}`` syntax for the variables. $env is only From e54d8d3fe9b701d748b69f0a0fda7e10cd942048 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 13:20:59 -0700 Subject: [PATCH 117/506] redundant subdir --- etc/spack/defaults/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index d19f0af685d7bd..888643e81b8a00 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -17,7 +17,7 @@ config: # This is the path to the root of the Spack install tree. # You can use $spack here to refer to the root of the spack instance. install_tree: - root: $default_install_root/installs + root: $default_install_root projections: all: "{architecture.platform}-{architecture.target}/{name}-{version}-{hash}" # install_tree can include an optional padded length (int or boolean) From cc170f9d048a15182fea45c0cd7865e69ee96724 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 13:24:33 -0700 Subject: [PATCH 118/506] rm default config pointing a write directory into spack --- etc/spack/defaults/config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 888643e81b8a00..d8f2fa96c0917f 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -69,7 +69,6 @@ config: build_stage: - $tempdir/$user/spack-stage - $xdg_cache_home/stage - # - $spack/var/spack/stage # Directory in which to run tests and store test results. # Tests will be stored in directories named by date/time and package From 8eaaa7ce6f10d0cd82b8649891c702b6e84f29ae Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 14:33:23 -0700 Subject: [PATCH 119/506] mv more paths tests --- lib/spack/spack/test/config.py | 14 -------------- lib/spack/spack/test/paths.py | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 479c99e5824353..da129548dc19e9 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1233,20 +1233,6 @@ def test_local_config_can_be_disabled(working_env): assert "user" in cfg.scopes -def test_user_cache_path_is_overridable(working_env): - p = "/some/path" - os.environ["SPACK_USER_CACHE_PATH"] = p - assert spack.paths._get_user_cache_path() == p - - -def test_user_cache_path_is_default_when_env_var_is_empty(working_env): - os.environ["SPACK_USER_CACHE_PATH"] = "" - assert ( - os.path.expanduser(os.path.join("~", ".local", "share", "spack")) - == spack.paths._get_user_cache_path() - ) - - def test_config_file_dir_failure(tmpdir, mutable_empty_config): with pytest.raises(spack.config.ConfigFileError, match="not a file"): spack.config.read_config_file(tmpdir.strpath) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 866d1df05b659d..4888ab0fea29ed 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -46,3 +46,19 @@ def test_user_config_path_is_default_when_env_var_is_empty(working_env, tmpdir): os.environ["SPACK_USER_CONFIG_PATH"] = "" p1 = paths.SpackPaths(str(tmpdir)) assert os.path.expanduser(os.path.join("~", ".config", "spack")) == p1.user_config_path + + +def test_user_cache_path_is_overridable(working_env, tmpdir): + redirect_usr_cache = "/some/path" + os.environ["SPACK_USER_CACHE_PATH"] = redirect_usr_cache + p1 = paths.SpackPaths(str(tmpdir)) + assert p1.user_cache_path == redirect_usr_cache + + +def test_user_cache_path_is_default_when_env_var_is_empty(working_env, tmpdir): + os.environ["SPACK_USER_CACHE_PATH"] = "" + p1 = paths.SpackPaths(str(tmpdir)) + assert ( + os.path.expanduser(os.path.join("~", ".local", "share", "spack")) + == p1.user_cache_path + ) \ No newline at end of file From 7289ff2a8ff7afeffd9d083f9f84bde9ea9b0bc7 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 15:07:17 -0700 Subject: [PATCH 120/506] style fix --- lib/spack/spack/test/paths.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 4888ab0fea29ed..22b143b219a251 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -58,7 +58,4 @@ def test_user_cache_path_is_overridable(working_env, tmpdir): def test_user_cache_path_is_default_when_env_var_is_empty(working_env, tmpdir): os.environ["SPACK_USER_CACHE_PATH"] = "" p1 = paths.SpackPaths(str(tmpdir)) - assert ( - os.path.expanduser(os.path.join("~", ".local", "share", "spack")) - == p1.user_cache_path - ) \ No newline at end of file + assert os.path.expanduser(os.path.join("~", ".local", "share", "spack")) == p1.user_cache_path From 98aba4a17a2054dbdf15458e04d088b8578c16ca Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 15:09:40 -0700 Subject: [PATCH 121/506] update comment --- lib/spack/spack/test/cmd/commands.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index 43dae061e7cf48..6bc4d9271903da 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -284,10 +284,7 @@ def test_updated_completion_scripts(shell, tmpdir): commands("--aliases", "--format", shell, "--header", header, "--update", new_script) - # TODO: changes in this PR add a site-admin config scope. If one runs - # `spack commands --update-completion` when this scope is available, it - # will add the site-admin scope to the autocompletion scripts, and cause - # a discrepancy with the runner + # If there is a diff, something is wrong: in that case output what the diff is if not filecmp.cmp(old_script, new_script): import difflib From 12b2a6ddc05877c8f597960b4dee99dd3dd4ea5e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 15:13:02 -0700 Subject: [PATCH 122/506] update comment --- lib/spack/spack/test/cmd/commands.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index 6bc4d9271903da..d736dfd39d6388 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -284,7 +284,10 @@ def test_updated_completion_scripts(shell, tmpdir): commands("--aliases", "--format", shell, "--header", header, "--update", new_script) - # If there is a diff, something is wrong: in that case output what the diff is + # If there is a diff, something is wrong: in that case output what the diff is. + # If someone runs `spack commands --update-completion` when the site-admin + # scope is available, it will be added to the autocompletion scripts, and + # cause a discrepancy with the runner if not filecmp.cmp(old_script, new_script): import difflib From cc1968bd9b73d0f458ba4737061fba6619ee5499 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 15:15:09 -0700 Subject: [PATCH 123/506] update comment --- lib/spack/spack/test/cmd/commands.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index d736dfd39d6388..1511d5fc00043b 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -285,9 +285,10 @@ def test_updated_completion_scripts(shell, tmpdir): commands("--aliases", "--format", shell, "--header", header, "--update", new_script) # If there is a diff, something is wrong: in that case output what the diff is. - # If someone runs `spack commands --update-completion` when the site-admin + # TODO: if someone runs `spack commands --update-completion` when the site-admin # scope is available, it will be added to the autocompletion scripts, and - # cause a discrepancy with the runner + # cause a discrepancy with the runner; it might be worth changing the update + # command to omit that. if not filecmp.cmp(old_script, new_script): import difflib From 3629b48c2f7590108cf482606ffe2d9013b7a9cf Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 16:04:28 -0700 Subject: [PATCH 124/506] update completion w/ merge from develop --- share/spack/spack-completion.fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index eb2fa2b82fd1b8..636fcbd3ebe9f5 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -2811,7 +2811,7 @@ complete -c spack -n '__fish_spack_using_command repo set' -l destination -r -f complete -c spack -n '__fish_spack_using_command repo set' -l destination -r -d 'destination to clone git repository into' complete -c spack -n '__fish_spack_using_command repo set' -l path -r -f -a path complete -c spack -n '__fish_spack_using_command repo set' -l path -r -d 'relative path to the Spack package repository inside a git repository. Can be repeated to add multiple package repositories in case of a monorepo' -complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -d 'configuration scope to modify' # spack repo remove From a3748d37ae0c3f422bf2aa494f17aaecf25aec8b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 17:33:56 -0700 Subject: [PATCH 125/506] platform-agnostic paths --- lib/spack/spack/test/paths.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 22b143b219a251..a3187ee60d45f7 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -9,7 +9,7 @@ def test_install_location(working_env, tmpdir): - base_prefix = str(tmpdir.join("prefix").ensure(dir=True)) + base_prefix = str(tmpdir.join("base-prefix").ensure(dir=True)) xdg_data_home = str(tmpdir.join("xdg_data_home")) os.environ["XDG_DATA_HOME"] = xdg_data_home p1 = paths.SpackPaths(base_prefix) @@ -23,9 +23,9 @@ def test_install_location(working_env, tmpdir): def test_system_config_path_is_overridable(working_env, tmpdir): - redirect_syscfg_path = "/some/path" + redirect_syscfg_path = str(pathlib.Path(tmpdir) / "redirected_syscfg") os.environ["SPACK_SYSTEM_CONFIG_PATH"] = redirect_syscfg_path - p1 = paths.SpackPaths(str(tmpdir)) + p1 = paths.SpackPaths(str(tmpdir.join("base-prefix").ensure(dir=True))) assert p1.system_config_path == redirect_syscfg_path @@ -36,9 +36,9 @@ def test_system_config_path_is_default_when_env_var_is_empty(working_env, tmpdir def test_user_config_path_is_overridable(working_env, tmpdir): - redirect_usrcfg_path = "/some/path" + redirect_usrcfg_path = str(pathlib.Path(tmpdir) / "redirected_usrcfg") os.environ["SPACK_USER_CONFIG_PATH"] = redirect_usrcfg_path - p1 = paths.SpackPaths(str(tmpdir)) + p1 = paths.SpackPaths(str(tmpdir.join("base-prefix").ensure(dir=True))) assert p1.user_config_path == redirect_usrcfg_path @@ -49,9 +49,9 @@ def test_user_config_path_is_default_when_env_var_is_empty(working_env, tmpdir): def test_user_cache_path_is_overridable(working_env, tmpdir): - redirect_usr_cache = "/some/path" + redirect_usr_cache = str(pathlib.Path(tmpdir) / "redirected_usr_cache") os.environ["SPACK_USER_CACHE_PATH"] = redirect_usr_cache - p1 = paths.SpackPaths(str(tmpdir)) + p1 = paths.SpackPaths(str(tmpdir.join("base-prefix").ensure(dir=True))) assert p1.user_cache_path == redirect_usr_cache From ab9a252407dacdbec435003fa2ba31ea5fb5e4cf Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 21:49:55 -0700 Subject: [PATCH 126/506] expand install dir test and fix bug with install dir selection logic --- lib/spack/spack/paths.py | 2 +- lib/spack/spack/test/paths.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index b3ef16dc848e70..0a8ff361bc44d5 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -268,7 +268,7 @@ def large_data_component(self, subdir, old_location): elif dir_is_occupied(old_location): return old_location else: - return os.path.join(self.prefix, "opt", "data") + return os.path.join(self.prefix, "opt", "data", subdir) locations = SpackPaths() diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index a3187ee60d45f7..f6aa05bf60e134 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -9,13 +9,27 @@ def test_install_location(working_env, tmpdir): + # With no direction from env vars, a fresh clone of Spack + # should default to using the Spack prefix. It was moved from + # where it used to be base_prefix = str(tmpdir.join("base-prefix").ensure(dir=True)) + p1 = paths.SpackPaths(base_prefix) + assert p1.default_install_location == str(pathlib.Path(base_prefix) / "opt" / "data" / "installs") + + # If XDG_DATA_HOME and SPACK_DATA_HOME aren't set, and + # there are installs in the old prefix, use that + preexisting_install_dir = pathlib.Path(base_prefix) / "opt" / "spack" / ".spack-db" + (preexisting_install_dir).mkdir(parents=True) + p1 = paths.SpackPaths(base_prefix) + assert p1.default_install_location == str(preexisting_install_dir.parent) + + # XDG_DATA_HOME overrides all the above xdg_data_home = str(tmpdir.join("xdg_data_home")) os.environ["XDG_DATA_HOME"] = xdg_data_home p1 = paths.SpackPaths(base_prefix) assert p1.default_install_location == str(pathlib.Path(xdg_data_home) / "spack" / "installs") - # Check that SPACK_DATA_HOME overrides + # Check that SPACK_DATA_HOME overrides all the above spack_data_home = str(tmpdir.join("spack_data_home")) os.environ["SPACK_DATA_HOME"] = spack_data_home p2 = paths.SpackPaths(base_prefix) From 0c44c60fb4bbacb81143a10170c68d92edadde67 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 21:52:38 -0700 Subject: [PATCH 127/506] style fix --- lib/spack/spack/test/paths.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index f6aa05bf60e134..1cff1bfd6abbb3 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -14,7 +14,9 @@ def test_install_location(working_env, tmpdir): # where it used to be base_prefix = str(tmpdir.join("base-prefix").ensure(dir=True)) p1 = paths.SpackPaths(base_prefix) - assert p1.default_install_location == str(pathlib.Path(base_prefix) / "opt" / "data" / "installs") + assert p1.default_install_location == str( + pathlib.Path(base_prefix) / "opt" / "data" / "installs" + ) # If XDG_DATA_HOME and SPACK_DATA_HOME aren't set, and # there are installs in the old prefix, use that From 41dd6f9e87bfedcf9a8637a3fe78b448e54c0503 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 22:06:58 -0700 Subject: [PATCH 128/506] more testing --- lib/spack/spack/paths.py | 2 +- lib/spack/spack/test/paths.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 0a8ff361bc44d5..d8981b057fd1db 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -195,7 +195,7 @@ def __init__(self, _prefix=None): # they pull this update, they will continue to use those # locations) - old_gpg_path = os.path.join("prefix", "opt" "spack", "gpg") + old_gpg_path = os.path.join(self.prefix, "opt" "spack", "gpg") if dir_is_occupied(old_gpg_path): self.gpg_path = old_gpg_path else: diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 1cff1bfd6abbb3..a013d4c290b844 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -70,6 +70,28 @@ def test_user_cache_path_is_overridable(working_env, tmpdir): p1 = paths.SpackPaths(str(tmpdir.join("base-prefix").ensure(dir=True))) assert p1.user_cache_path == redirect_usr_cache + # Check that things that are supposed to be bundled inside of + # $user_cache_path are also relocated + assert p1.package_repos_path == str(pathlib.Path(redirect_usr_cache) / "package_repos") + +def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmpdir): + user_cache_path = str(tmpdir.join("user_cache")) + base_prefix = str(tmpdir.join("base-prefix").ensure(dir=True)) + os.environ["SPACK_USER_CACHE_PATH"] = user_cache_path + + p1 = paths.SpackPaths(base_prefix) + assert p1.gpg_path == str(pathlib.Path(user_cache_path) / "gpg") + assert p1.gpg_keys_path == str(pathlib.Path(user_cache_path) / "gpg-keys") + + old_gpg_dir = pathlib.Path(base_prefix) / "opt" / "spack" / "gpg" + (old_gpg_dir).mkdir(parents=True) + p1 = paths.SpackPaths(base_prefix) + # Old dir exists, but is empty, so it should not be used + assert p1.gpg_path == str(pathlib.Path(user_cache_path) / "gpg") + (old_gpg_dir / "something").touch() + p1 = paths.SpackPaths(base_prefix) + assert p1.gpg_path == str(old_gpg_dir) + def test_user_cache_path_is_default_when_env_var_is_empty(working_env, tmpdir): os.environ["SPACK_USER_CACHE_PATH"] = "" From 8c5a5db34996ec07a3d399c021a5a87cb3fd0a26 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 22:13:23 -0700 Subject: [PATCH 129/506] fix test --- lib/spack/spack/paths.py | 2 +- lib/spack/spack/test/paths.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index d8981b057fd1db..dc60aeb9208c17 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -195,7 +195,7 @@ def __init__(self, _prefix=None): # they pull this update, they will continue to use those # locations) - old_gpg_path = os.path.join(self.prefix, "opt" "spack", "gpg") + old_gpg_path = os.path.join(self.prefix, "opt", "spack", "gpg") if dir_is_occupied(old_gpg_path): self.gpg_path = old_gpg_path else: diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index a013d4c290b844..c8a09c0f5887e2 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -74,6 +74,7 @@ def test_user_cache_path_is_overridable(working_env, tmpdir): # $user_cache_path are also relocated assert p1.package_repos_path == str(pathlib.Path(redirect_usr_cache) / "package_repos") + def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmpdir): user_cache_path = str(tmpdir.join("user_cache")) base_prefix = str(tmpdir.join("base-prefix").ensure(dir=True)) From 0059b10f5f60a27a827250d7d297fad5d00d3e1d Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 22:17:48 -0700 Subject: [PATCH 130/506] expand test for gpg paths redirection logic --- lib/spack/spack/test/paths.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index c8a09c0f5887e2..a8a82ecb1a1ee0 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -91,7 +91,16 @@ def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmpdir): assert p1.gpg_path == str(pathlib.Path(user_cache_path) / "gpg") (old_gpg_dir / "something").touch() p1 = paths.SpackPaths(base_prefix) + # Now it should redirect assert p1.gpg_path == str(old_gpg_dir) + # But the keys are handled separately and should use the new path + assert p1.gpg_keys_path == str(pathlib.Path(user_cache_path) / "gpg-keys") + + old_gpg_keys_dir = pathlib.Path(base_prefix) / "var" / "spack" / "gpg" + old_gpg_keys_dir.mkdir(parents=True) + (old_gpg_keys_dir / "something").touch() + p1 = paths.SpackPaths(base_prefix) + assert p1.gpg_keys_path == str(old_gpg_keys_dir) def test_user_cache_path_is_default_when_env_var_is_empty(working_env, tmpdir): From 3be2f74116e5acbae34c17cb8ea549f0768182b2 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 22:19:41 -0700 Subject: [PATCH 131/506] style fix --- lib/spack/spack/test/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index a8a82ecb1a1ee0..5a615f4b557211 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -88,7 +88,7 @@ def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmpdir): (old_gpg_dir).mkdir(parents=True) p1 = paths.SpackPaths(base_prefix) # Old dir exists, but is empty, so it should not be used - assert p1.gpg_path == str(pathlib.Path(user_cache_path) / "gpg") + assert p1.gpg_path == str(pathlib.Path(user_cache_path) / "gpg") (old_gpg_dir / "something").touch() p1 = paths.SpackPaths(base_prefix) # Now it should redirect From 27aa74437af0996490c51c90e4466762959e1c7b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 22:44:10 -0700 Subject: [PATCH 132/506] attempt an explanation of default_install_root that would be sensible for a user --- etc/spack/defaults/config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index d8f2fa96c0917f..6f1b0a8308b8bb 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -17,6 +17,10 @@ config: # This is the path to the root of the Spack install tree. # You can use $spack here to refer to the root of the spack instance. install_tree: + # If you set this to anything else, that setting has precedence. This + # value tells spack to check several possibilities by order of + # preference: SPACK_DATA_HOME, XDG_DATA_HOME, and then lastly + # choosing a location inside of Spack if neither are defined. root: $default_install_root projections: all: "{architecture.platform}-{architecture.target}/{name}-{version}-{hash}" From 56cc2c9ebbdda4213c432701a8a4c78e8d1bd36d Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 23:33:22 -0700 Subject: [PATCH 133/506] expanded comments for default_ variables in user config --- etc/spack/defaults/config.yaml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 6f1b0a8308b8bb..6486ae91350b7d 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -17,10 +17,12 @@ config: # This is the path to the root of the Spack install tree. # You can use $spack here to refer to the root of the spack instance. install_tree: - # If you set this to anything else, that setting has precedence. This - # value tells spack to check several possibilities by order of - # preference: SPACK_DATA_HOME, XDG_DATA_HOME, and then lastly - # choosing a location inside of Spack if neither are defined. + # If set to a non-default value, this setting has precedence. + # This default value tells spack to check (in order of precedence): + # $SPACK_DATA_HOME/installs + # $XDG_DATA_HOME/spack/installs + # $spack/opt/spack if previously used by an older version of spack + # $spack/opt/data/installs root: $default_install_root projections: all: "{architecture.platform}-{architecture.target}/{name}-{version}-{hash}" @@ -82,11 +84,23 @@ config: # Cache directory for already downloaded source tarballs and archived # repositories. This can be purged with `spack clean --downloads`. + # If set to a non-default value, this setting has precedence. + # This default value tells spack to check (in order of precedence): + # $SPACK_DATA_HOME/downloads + # $XDG_DATA_HOME/spack/downloads + # $spack/var/spack/cache if previously used by an older version of spack + # $spack/opt/data/downloads source_cache: $default_download_root ## Directory where spack managed environments are created and stored - # environments_root: $default_envs_root/environments + # If set to a non-default value, this setting has precedence. + # This default value tells spack to check (in order of precedence): + # $SPACK_DATA_HOME/environments + # $XDG_DATA_HOME/spack/environments + # $spack/var/spack/environments if previously used by an older version of spack + # $spack/opt/data/environments + # environments_root: $default_envs_root # Cache directory for miscellaneous files, like the package index. From d81e0c6c65ec7ee748dd5790a211123dc898a776 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Jun 2025 23:38:36 -0700 Subject: [PATCH 134/506] add similar explanation for modules_base --- etc/spack/defaults/modules.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/spack/defaults/modules.yaml b/etc/spack/defaults/modules.yaml index d66a13f26bb6e3..b152e967a66b5d 100644 --- a/etc/spack/defaults/modules.yaml +++ b/etc/spack/defaults/modules.yaml @@ -37,6 +37,10 @@ modules: # These are configurations for the module set named "default" default: # Where to install modules + # If set to a non-default value, this setting has precedence. + # This default value tells spack to check (in order of precedence): + # $spack/share/spack/{modules/lmod} if previously used by an older version of spack + # $user_cache_path/modules/{modules/lmod} otherwise roots: tcl: $modules_base/modules lmod: $modules_base/lmod From 85cca7bc75a9f8d649a86c44e51cbd136b384b21 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 00:23:26 -0700 Subject: [PATCH 135/506] implement override mechanism where all XDG_ vars have a SPACK_ override (and expose these as config vars) --- lib/spack/spack/paths.py | 69 ++++++++++++++++++++---------------- lib/spack/spack/util/path.py | 12 ++++--- 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index dc60aeb9208c17..0e5c4fc80abce2 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -10,6 +10,7 @@ """ import os import pathlib +from collections import namedtuple from enum import Enum from pathlib import PurePath @@ -25,9 +26,25 @@ class XDG_vars(Enum): cache_home = "XDG_CACHE_HOME" +class XDG_overrides(Enum): + config_home = "SPACK_CONFIG_HOME" + state_home = "SPACK_STATE_HOME" + data_home = "SPACK_DATA_HOME" + cache_home = "SPACK_CACHE_HOME" + + +xdg_mapping = namedtuple("xdg_mapping", ["spack", "xdg", "xdg_default"]) + + +class XDG_mappings(Enum): + config_home = xdg_mapping(spack=XDG_overrides.config_home.value, xdg=XDG_vars.config_home.value, xdg_default=os.path.join("~", ".config")) + state_home = xdg_mapping(spack=XDG_overrides.state_home.value, xdg=XDG_vars.state_home.value, xdg_default=os.path.join("~", ".local", "state")) + data_home = xdg_mapping(spack=XDG_overrides.data_home.value, xdg=XDG_vars.data_home.value, xdg_default=os.path.join("~", ".local", "share")) + cache_home = xdg_mapping(spack=XDG_overrides.cache_home.value, xdg=XDG_vars.cache_home.value, xdg_default=os.path.join("~", ".cache")) + + class Location_vars(Enum): user_cache_path = "USER_CACHE_PATH" - spack_data_home = "SPACK_DATA_HOME" # This is for tests that want to clean the environment of XDG_ variables that @@ -40,12 +57,15 @@ def _unset_xdg_vars(env): return saved -def _define_xdg_or_backup(xdg_var, backup): - if xdg_var in os.environ: - spack_xdg_defined = os.path.join(os.environ[xdg_var], "spack") +def _spack_xdg_or_backup(xdg_mapping): + if xdg_mapping.spack in os.environ: + val = os.environ[xdg_mapping.spack] + elif xdg_mapping.xdg in os.environ: + val = os.path.join(os.environ[xdg_mapping.xdg], "spack") else: - spack_xdg_defined = os.path.join(backup, "spack") - return os.path.expanduser(spack_xdg_defined) + val = os.path.join(xdg_mapping.xdg_default, "spack") + + return os.path.expanduser(val) def dir_is_occupied(x, except_for=None): @@ -103,18 +123,10 @@ def __init__(self, _prefix=None): # Resolved XDG_x_HOME variables, with additional "spack" subdirectory. # Resolves to default value from XDG spec if unset. - self.xdg_state_home = _define_xdg_or_backup( - XDG_vars.state_home.value, os.path.join("~", ".local", "state") - ) - self.xdg_config_home = _define_xdg_or_backup( - XDG_vars.config_home.value, os.path.join("~", ".config") - ) - self.xdg_cache_home = _define_xdg_or_backup( - XDG_vars.cache_home.value, os.path.join("~", ".cache") - ) - self.xdg_data_home = _define_xdg_or_backup( - XDG_vars.data_home.value, os.path.join("~", ".local", "share") - ) + self.spack_state_home = _spack_xdg_or_backup(XDG_mappings.state_home.value) + self.spack_config_home = _spack_xdg_or_backup(XDG_mappings.config_home.value) + self.spack_cache_home = _spack_xdg_or_backup(XDG_mappings.cache_home.value) + self.spack_data_home = _spack_xdg_or_backup(XDG_mappings.data_home.value) # ------ Next section # Spack can write a lot of data into the next 3 locations, and @@ -230,7 +242,7 @@ def __init__(self, _prefix=None): #: User configuration location self.user_config_path = os.path.expanduser( - os.getenv("SPACK_USER_CONFIG_PATH") or self.xdg_config_home + os.getenv("SPACK_USER_CONFIG_PATH") or self.spack_config_home ) #: System configuration location @@ -245,7 +257,7 @@ def __init__(self, _prefix=None): #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) #: overridden by `config:misc_cache` self.default_misc_cache_path = os.path.join( - self.xdg_state_home, self.spack_instance_id, "misc-cache" + self.spack_state_home, self.spack_instance_id, "misc-cache" ) #: concretization cache for Spack concretizations @@ -253,14 +265,11 @@ def __init__(self, _prefix=None): self.default_conc_cache_path = os.path.join(self.default_misc_cache_path, "concretization") def data_home_for_small_data(self): - if Location_vars.spack_data_home.value in os.environ: - return os.environ[Location_vars.spack_data_home.value] - else: - return self.xdg_data_home + return self.spack_data_home def large_data_component(self, subdir, old_location): - if Location_vars.spack_data_home.value in os.environ: - return os.path.join(os.environ[Location_vars.spack_data_home.value], subdir) + if XDG_overrides.data_home.value in os.environ: + return os.path.join(os.environ[XDG_overrides.data_home.value], subdir) elif XDG_vars.data_home.value in os.environ: return os.path.expanduser( os.path.join(os.environ[XDG_vars.data_home.value], "spack", subdir) @@ -298,10 +307,10 @@ def large_data_component(self, subdir, old_location): mock_gpg_data_path = locations.mock_gpg_data_path mock_gpg_keys_path = locations.mock_gpg_keys_path spack_instance_id = locations.spack_instance_id -xdg_state_home = locations.xdg_state_home -xdg_config_home = locations.xdg_config_home -xdg_cache_home = locations.xdg_cache_home -xdg_data_home = locations.xdg_data_home +spack_state_home = locations.spack_state_home +spack_config_home = locations.spack_config_home +spack_cache_home = locations.spack_cache_home +spack_data_home = locations.spack_data_home default_install_location = locations.default_install_location default_envs_path = locations.default_envs_path default_fetch_cache_path = locations.default_fetch_cache_path diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index bd64619dae37a1..07808017e9504b 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -70,8 +70,10 @@ def replacements(): "default_install_root": lambda: spack.paths.default_install_location, "default_envs_root": lambda: spack.paths.default_envs_path, "default_download_root": lambda: spack.paths.default_fetch_cache_path, - "xdg_cache_home": lambda: spack.paths.xdg_cache_home, - "xdg_state_home": lambda: spack.paths.xdg_state_home, + "spack_data_home": lambda: spack.paths.spack_data_home, + "spack_cache_home": lambda: spack.paths.spack_cache_home, + "spack_state_home": lambda: spack.paths.spack_state_home, + "spack_config_home": lambda: spack.paths.spack_config_home, "spack_instance_id": lambda: spack.paths.spack_instance_id, "architecture": lambda: arch, "arch": lambda: arch, @@ -171,8 +173,10 @@ def substitute_config_variables(path): - $default_install_root Where installs go by default - $default_envs_root Where environments are managed by default - $default_download_root Where downloads go by default - - $xdg_cache_home XDG location for temporary data - - $xdg_state_home XDG long-lived but not-essential cache + - $spack_data_home SPACK_DATA_HOME, XDG_DATA_HOME, or its default + - $spack_cache_home SPACK_CACHE_HOME, XDG_CACHE_HOME, or its default + - $spack_state_home SPACK_STATE_HOME, XDG_STATE_HOME, or its default + - $spack_config_home SPACK_CONFIG_HOME, XDG_CONFIG_HOME, or its default - $spack_instance_id Hash that distinguishes Spack instances on the filesystem - $architecture The spack architecture triple for the current system - $arch The spack architecture triple for the current system From 98896097311836cd2e36d9db6b84851a08437690 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 00:25:43 -0700 Subject: [PATCH 136/506] style fix --- lib/spack/spack/paths.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 0e5c4fc80abce2..e5d49155f48590 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -37,10 +37,26 @@ class XDG_overrides(Enum): class XDG_mappings(Enum): - config_home = xdg_mapping(spack=XDG_overrides.config_home.value, xdg=XDG_vars.config_home.value, xdg_default=os.path.join("~", ".config")) - state_home = xdg_mapping(spack=XDG_overrides.state_home.value, xdg=XDG_vars.state_home.value, xdg_default=os.path.join("~", ".local", "state")) - data_home = xdg_mapping(spack=XDG_overrides.data_home.value, xdg=XDG_vars.data_home.value, xdg_default=os.path.join("~", ".local", "share")) - cache_home = xdg_mapping(spack=XDG_overrides.cache_home.value, xdg=XDG_vars.cache_home.value, xdg_default=os.path.join("~", ".cache")) + config_home = xdg_mapping( + spack=XDG_overrides.config_home.value, + xdg=XDG_vars.config_home.value, + xdg_default=os.path.join("~", ".config"), + ) + state_home = xdg_mapping( + spack=XDG_overrides.state_home.value, + xdg=XDG_vars.state_home.value, + xdg_default=os.path.join("~", ".local", "state"), + ) + data_home = xdg_mapping( + spack=XDG_overrides.data_home.value, + xdg=XDG_vars.data_home.value, + xdg_default=os.path.join("~", ".local", "share"), + ) + cache_home = xdg_mapping( + spack=XDG_overrides.cache_home.value, + xdg=XDG_vars.cache_home.value, + xdg_default=os.path.join("~", ".cache"), + ) class Location_vars(Enum): @@ -61,7 +77,7 @@ def _spack_xdg_or_backup(xdg_mapping): if xdg_mapping.spack in os.environ: val = os.environ[xdg_mapping.spack] elif xdg_mapping.xdg in os.environ: - val = os.path.join(os.environ[xdg_mapping.xdg], "spack") + val = os.path.join(os.environ[xdg_mapping.xdg], "spack") else: val = os.path.join(xdg_mapping.xdg_default, "spack") From 634b8a34152d30017f954ebc668197f44bdbb35f Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 00:27:08 -0700 Subject: [PATCH 137/506] forgot to update config defaults now that there no xdg_ config vars, just spack_ config vars --- etc/spack/defaults/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 6486ae91350b7d..3a123a5f0cde69 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -74,7 +74,7 @@ config: # identifies Spack staging to avoid accidentally wiping out non-Spack work. build_stage: - $tempdir/$user/spack-stage - - $xdg_cache_home/stage + - $spack_cache_home/stage # Directory in which to run tests and store test results. # Tests will be stored in directories named by date/time and package @@ -105,7 +105,7 @@ config: # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` - misc_cache: $xdg_state_home/$spack_instance_id/cache + misc_cache: $spack_state_home/$spack_instance_id/cache # Abort downloads after this many seconds if not data is received. From 4a610d008779fa5123d90c100bfcb943dee9aa59 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 00:33:49 -0700 Subject: [PATCH 138/506] update completion with merge --- share/spack/spack-completion.fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 68b2e8ed39e0fb..f241d5208d8754 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -2795,7 +2795,7 @@ complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'conf set -g __fish_spack_optspecs_spack_repo_ls h/help scope= complete -c spack -n '__fish_spack_using_command repo ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -d 'configuration scope to read from' # spack repo add From aa72813b75a6b32adbf7557e7e30f8a05c95353a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 10:04:40 -0700 Subject: [PATCH 139/506] make system scope higher priority than site-admin --- lib/spack/spack/config.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 187f8ee8631d9c..484c638d7e2024 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -929,17 +929,6 @@ def create_incremental() -> Generator[Configuration, None, None]: disable_local_config = "SPACK_DISABLE_LOCAL_CONFIG" in os.environ - # System configuration is per machine. - # This is disabled if user asks for no local configuration. - if not disable_local_config: - configuration_paths.append(("system", spack.paths.system_config_path)) - if end_user_system_scope: - configuration_paths.append(("end-user", spack.paths.end_user_cfg_path)) - - # Site configuration is per spack instance, for sites or projects - # No site-level configs should be checked into spack by default. - configuration_paths.append(("site", spack.paths.etc_path)) - # Site admin scope has two uses: (a) admins can share config with one # another, but not with end users (b) pip/apt-installed spack can # change the default install root @@ -956,6 +945,17 @@ def create_incremental() -> Generator[Configuration, None, None]: # not be writable configuration_paths.append(("site-admin", site_admin_path)) + # System configuration is per machine. + # This is disabled if user asks for no local configuration. + if not disable_local_config: + configuration_paths.append(("system", spack.paths.system_config_path)) + if end_user_system_scope: + configuration_paths.append(("end-user", spack.paths.end_user_cfg_path)) + + # Site configuration is per spack instance, for sites or projects + # No site-level configs should be checked into spack by default. + configuration_paths.append(("site", spack.paths.etc_path)) + # Python package's can register configuration scopes via entry_points configuration_paths.extend(config_paths_from_entry_points()) From a041de44364eaca08be05fbcc1c4d5288fe9aff5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 10:05:41 -0700 Subject: [PATCH 140/506] update comment --- lib/spack/spack/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 484c638d7e2024..0e0c480baf780d 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -941,8 +941,7 @@ def create_incremental() -> Generator[Configuration, None, None]: except PermissionError: pass if site_admin_accessible: - # TODO: this does not need to be platform specific, and it should - # not be writable + # TODO: list this as an available scope, but not a writable one configuration_paths.append(("site-admin", site_admin_path)) # System configuration is per machine. From c8a85401ee0306e0960b4ffbe9e27d426fbcdd77 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 10:27:57 -0700 Subject: [PATCH 141/506] clear spack_ overrides as well for clean-env context manager --- lib/spack/spack/paths.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index e5d49155f48590..f2a5f6ceab6b5b 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -12,6 +12,7 @@ import pathlib from collections import namedtuple from enum import Enum +import itertools from pathlib import PurePath import llnl.util.filesystem @@ -64,10 +65,10 @@ class Location_vars(Enum): # This is for tests that want to clean the environment of XDG_ variables that -# affect spack behavior +# affect spack behavior (and the corresponding SPACK_ overrides) def _unset_xdg_vars(env): saved = {} - for xdg_var in XDG_vars: + for xdg_var in itertools.chain(XDG_vars, XDG_overrides): if xdg_var.value in env: saved[xdg_var.value] = env.pop(xdg_var.value) return saved From 4ec27c4ecb56e4fba262d7240296bd410f74a580 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 10:33:30 -0700 Subject: [PATCH 142/506] update comment --- lib/spack/spack/paths.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index f2a5f6ceab6b5b..4ce338bb5f108c 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -65,7 +65,9 @@ class Location_vars(Enum): # This is for tests that want to clean the environment of XDG_ variables that -# affect spack behavior (and the corresponding SPACK_ overrides) +# affect spack behavior (and the corresponding SPACK_ overrides). Note that +# these vars will affect .default_test_path for the running instance, but +# the unit tests will not see the env vars def _unset_xdg_vars(env): saved = {} for xdg_var in itertools.chain(XDG_vars, XDG_overrides): From 9ea5972b1998ed8f2584522482cee065e10c5890 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 10:34:19 -0700 Subject: [PATCH 143/506] style fix --- lib/spack/spack/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 4ce338bb5f108c..d4f9cb44673589 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -8,11 +8,11 @@ throughout Spack and should bring in a minimal number of external dependencies. """ +import itertools import os import pathlib from collections import namedtuple from enum import Enum -import itertools from pathlib import PurePath import llnl.util.filesystem From e5715cd710b101807f614455340c2685ded5e496 Mon Sep 17 00:00:00 2001 From: "Matthew L. Curry" Date: Fri, 6 Jun 2025 04:53:22 -0400 Subject: [PATCH 144/506] Docs update part 1 --- lib/spack/docs/config_yaml.rst | 21 ++-- lib/spack/docs/configuration.rst | 181 ++++++++++++++++++++--------- lib/spack/docs/getting_started.rst | 14 ++- 3 files changed, 149 insertions(+), 67 deletions(-) diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index 2c06877e55cb01..20f63e23d0b433 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -15,15 +15,17 @@ see the default settings by looking at .. literalinclude:: _spack_root/etc/spack/defaults/config.yaml :language: yaml -These settings can be overridden in ``etc/spack/config.yaml`` or -``~/.spack/config.yaml``. See :ref:`configuration-scopes` for details. +These settings can be overridden in ``etc/spack/config.yaml``, or +``~/.config/spack/config.yaml``, or +``~/.config/spack/$spack_instance_id/config.yaml``. See +:ref:`configuration-scopes` for details. --------------------- ``install_tree:root`` --------------------- The location where Spack will install packages and their dependencies. -The default is ``$spack/opt/spack``. +The default is ``$spack/opt/data/installs``. --------------- ``projections`` @@ -94,7 +96,7 @@ By default, Spack's ``build_stage`` is configured like this: build_stage: - $tempdir/$user/spack-stage - - ~/.spack/stage + - $xdg_cache_home/stage This can be an ordered list of paths that Spack should search when trying to find a temporary directory for the build stage. The list is searched in @@ -120,8 +122,8 @@ deleted, but you can manually purge them with :ref:`spack clean --stage ``source_cache`` -------------------- -Location to cache downloaded tarballs and repositories. By default, these -are stored in ``$spack/var/spack/cache``. These are stored indefinitely +Location to cache downloaded tarballs and repositories. By default, +these are stored in ``$spack/opt/data``. These are stored indefinitely by default and can be purged with :ref:`spack clean --downloads `. @@ -131,9 +133,10 @@ by default and can be purged with :ref:`spack clean --downloads ``misc_cache`` -------------------- -Temporary directory to store long-lived cache files, such as indices of -packages available in repositories. Defaults to ``~/.spack/cache``. Can -be purged with :ref:`spack clean --misc-cache `. +Temporary directory to store long-lived cache files, such as indices +of packages available in repositories. Defaults to +``~/.local/state/spack/$spack_instance_id/spack``. Can be purged with +:ref:`spack clean --misc-cache `. -------------------- ``verify_ssl`` diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 64d372ede070e0..9f039d15d85e9b 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -37,10 +37,10 @@ Here is an example ``config.yaml`` file: .. code-block:: yaml config: - install_tree: $spack/opt/spack + install_tree: $spack/opt/data/installs build_stage: - $tempdir/$user/spack-stage - - ~/.spack/stage + - $xdg_cache_home/stage Each Spack configuration file is nested under a top-level section corresponding to its name. So, ``config.yaml`` starts with ``config:``, @@ -73,18 +73,35 @@ are multiple configuration scopes. From lowest to highest precedence: packages. These settings are presumably controlled by someone with root access on the machine. They override the defaults scope. +#. **site-admin**: Stored in ``$spack/etc/site-admin``, which does not + exist by default. This scope contains private settings shared by + those administering a Spack instance on behalf of other + users. These settings affect only *this instance* of Spack. This + scope is meant to have read access restricted to control which + users are affected by these settings. + #. **site**: Stored in ``$(prefix)/etc/spack/``. Settings here affect - only *this instance* of Spack, and they override the defaults and system - scopes. The site scope can be used for per-project settings (one - Spack instance per project) or for site-wide settings on a multi-user - machine (e.g., for a common Spack instance). + only *this instance* of Spack, and they override the defaults, + system, and site-admin scopes. The site scope can be used for + per-project settings (one Spack instance per project) or for + site-wide settings on a multi-user machine (e.g., for a common + Spack instance). #. **plugin**: Read from a Python package's entry points. Settings here affect all instances of Spack running with the same Python installation. This scope takes higher precedence than site, system, and default scopes. -#. **user**: Stored in the home directory: ``~/.spack/``. These settings - affect all instances of Spack and take higher precedence than site, - system, plugin, or defaults scopes. +#. **user**: Stored in the home directory: + ``~/.local/config/spack/``. These settings affect all instances of + Spack and take higher precedence than site, system, plugin, or + defaults scopes. + +#. **per-spack**: Stored in the home directory: + ``~/.local/config/spack/$spack_instance_id/``. These settings + affect the instance of spack that corresponds to the specified + ``$spack_instance_id``. This scope is most useful for overriding + configurations for Spack instances that are not (or should not be) + writable by the user, such as those provided by a system + adminstrator or other package manager (like pip). #. **custom**: Stored in a custom directory specified by ``--config-scope``. If multiple scopes are listed on the command line, they are ordered @@ -138,6 +155,22 @@ If multiple scopes are provided: #. Each must be preceded with the ``--config-scope`` or ``-C`` flag. #. They must be ordered from lowest to highest precedence. +.. _scopes-and-xdg-compliance: + +^^^^^^^^^^^^^^^^^^^^^^^^^ +Scopes and XDG Compliance +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Spack respects XDG variables, and the search location for user +configuration files can be affected by them. When ``XDG_CONFIG_HOME`` +is not defined, Spack assumes the XDG default value of ``~/.config``, +and will apply user and per-spack configuration files located in +``~/.config/spack``. Defining ``XDG_CONFIG_HOME`` will change where +Spack searches for configuration files. To override this behavior, +define ``SPACK_USER_CONFIG_PATH`` to be the desired path. For more +information, see the :ref:`xdg_overrides` for more details. + + """"""""""""""""""""""""""""""""""""""""""" Example: scopes for release and development """"""""""""""""""""""""""""""""""""""""""" @@ -258,19 +291,22 @@ Platform-specific Configuration placed :ref:`include.yaml ` file. There is often a need for platform-specific configuration settings. -For example, on most platforms, GCC is the preferred compiler. However, -on macOS (darwin), Clang often works for more packages, and is set as -the default compiler. This configuration is set in -``$(prefix)/etc/spack/defaults/darwin/packages.yaml``, which is included -as by ``$(prefix)/etc/spack/defaults/include.yaml``. Since it is an included -configuration of the ``defaults`` scope, settings in the ``defaults`` scope -will take precedence. You can override the values by specifying settings in -``system``, ``site``, ``user``, or ``custom``, where scope precedence is: +For example, on most platforms, GCC is the preferred +compiler. However, on macOS (darwin), Clang often works for more +packages, and is set as the default compiler. This configuration is +set in ``$(prefix)/etc/spack/defaults/darwin/packages.yaml``, which is +included as by ``$(prefix)/etc/spack/defaults/include.yaml``. Since it +is an included configuration of the ``defaults`` scope, settings in +the ``defaults`` scope will take precedence. You can override the +values by specifying settings in ``system``, ``site-admin``, ``site``, +``user``, ``per-spack``, or ``custom``, where scope precedence is: #. ``defaults`` #. ``system`` +#. ``site-admin`` #. ``site`` #. ``user`` +#. ``per-spack`` #. ``custom`` and settings in each scope taking precedence over those found in configuration @@ -293,10 +329,10 @@ then, on macOS (``darwin``), configuration settings for files under the --platform``. Platform-specific configuration files can similarly be set up for the -``system``, ``site``, and ``user`` scopes by creating an ``include.yaml`` -similar to the one above for ``defaults`` -- under the appropriate -configuration paths (see :ref:`config-overrides`) and creating a subdirectory -with the platform name that contains the configuration files. +other scopes by creating an ``include.yaml`` similar to the one above +for ``defaults`` -- under the appropriate configuration paths (see +:ref:`config-overrides`) and creating a subdirectory with the platform +name that contains the configuration files. .. note:: @@ -342,14 +378,14 @@ your configurations look like this: :caption: $(prefix)/etc/spack/defaults/config.yaml config: - install_tree: $spack/opt/spack + install_tree: $spack/opt/data/installs build_stage: - $tempdir/$user/spack-stage - - ~/.spack/stage + - $xdg_cache_home/stage .. code-block:: yaml - :caption: ~/.spack/config.yaml + :caption: ~/.config/spack/config.yaml config: install_tree: /some/other/directory @@ -368,7 +404,7 @@ command: install_tree: /some/other/directory build_stage: - $tempdir/$user/spack-stage - - ~/.spack/stage + - $xdg_cache_home/stage .. _config-prepend-append: @@ -384,7 +420,7 @@ string concatenation at the end of a key in a configuration file. For example: .. code-block:: yaml :emphasize-lines: 1 - :caption: ~/.spack/config.yaml + :caption: ~/.config/spack/config.yaml config: install_tree-: /my/custom/suffix/ @@ -399,14 +435,14 @@ Spack will then append to the lower-precedence configuration under the install_tree: /some/other/directory/my/custom/suffix build_stage: - $tempdir/$user/spack-stage - - ~/.spack/stage + - $xdg_cache_home/stage Similarly, ``+:`` can be used to *prepend* to a path or name: .. code-block:: yaml :emphasize-lines: 1 - :caption: ~/.spack/config.yaml + :caption: ~/.config/spack/config.yaml config: install_tree+: /my/custom/suffix/ @@ -425,7 +461,7 @@ at the end of a key in a configuration file. For example: .. code-block:: yaml :emphasize-lines: 1 - :caption: ~/.spack/config.yaml + :caption: ~/.config/spack/config.yaml config:: install_tree: /some/other/directory @@ -452,14 +488,14 @@ Let's revisit the ``config.yaml`` example one more time. The build_stage: - $tempdir/$user/spack-stage - - ~/.spack/stage + - $xdg_cache_home/stage Suppose the user configuration adds its *own* list of ``build_stage`` paths: .. code-block:: yaml - :caption: ~/.spack/config.yaml + :caption: ~/.config/spack/config.yaml build_stage: - /lustre-scratch/$user/spack @@ -467,7 +503,7 @@ paths: Spack will first look at the paths in the defaults ``config.yaml``, then the -paths in the user's ``~/.spack/config.yaml``. The list in the +paths in the user's ``~/.config/spack/config.yaml``. The list in the higher-precedence scope is *prepended* to the defaults. ``spack config get config`` shows the result: @@ -481,7 +517,7 @@ get config`` shows the result: - /lustre-scratch/$user/spack - ~/mystage - $tempdir/$user/spack-stage - - ~/.spack/stage + - $xdg_cache_home/stage As in :ref:`config-overrides`, the higher-precedence scope can @@ -490,11 +526,12 @@ user config looked like this: .. code-block:: yaml :emphasize-lines: 1 - :caption: ~/.spack/config.yaml + :caption: ~/.config/spack/config.yaml - build_stage:: - - /lustre-scratch/$user/spack - - ~/mystage + config: + build_stage:: + - /lustre-scratch/$user/spack + - ~/mystage The merged configuration would look like this: @@ -551,7 +588,12 @@ Spack understands over a dozen special variables. These are: detected by ArchSpec. E.g. ``x86_64`` or ``aarch64``. * ``$date``: the current date in the format YYYY-MM-DD * ``$spack_short_version``: the Spack version truncated to the first components. - +* ``$spack_instance_id``: a hash that distinguishes Spack instances on the filesystem. +* ``$xdg_state_home``: the XDG location for long-lived but not-essential cache. +* ``$xdg_cache_home``: the XDG location for temporary data. +* ``$default_download_root``: the location where downloads go by default. +* ``$default_envs_root``: the location where environments are managed by default. +* ``$default_install_root``: the location where installs go by default. Note that, as with shell variables, you can write these as ``$varname`` or with braces to distinguish the variable from surrounding characters: @@ -635,16 +677,16 @@ account all scopes. For example, to see the fully merged verify_ssl: true dirty: false build_jobs: 8 - install_tree: $spack/opt/spack + install_tree: + root: $default_install_root template_dirs: - $spack/templates directory_layout: {architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash} build_stage: - $tempdir/$user/spack-stage - - ~/.spack/stage - - $spack/var/spack/stage - source_cache: $spack/var/spack/cache - misc_cache: ~/.spack/cache + - $xdg_cache_home/stage + source_cache: $default_download_root + misc_cache: $xdg_state_home/$spack_instance_id/cache locks: true Likewise, this will show the fully merged ``packages.yaml``: @@ -688,10 +730,9 @@ down the source of the configuration: /home/myuser/spack/etc/spack/defaults/config.yaml:28 directory_layout: {architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash} /home/myuser/spack/etc/spack/defaults/config.yaml:49 build_stage: /home/myuser/spack/etc/spack/defaults/config.yaml:50 - $tempdir/$user/spack-stage - /home/myuser/spack/etc/spack/defaults/config.yaml:51 - ~/.spack/stage - /home/myuser/spack/etc/spack/defaults/config.yaml:52 - $spack/var/spack/stage - /home/myuser/spack/etc/spack/defaults/config.yaml:57 source_cache: $spack/var/spack/cache - /home/myuser/spack/etc/spack/defaults/config.yaml:62 misc_cache: ~/.spack/cache + /home/myuser/spack/etc/spack/defaults/config.yaml:51 - $xdg_cache_home/stage + /home/myuser/spack/etc/spack/defaults/config.yaml:57 source_cache: $default_download_root + /home/myuser/spack/etc/spack/defaults/config.yaml:62 misc_cache: $xdg_state_home/$spack_instance_id/cache /home/myuser/spack/etc/spack/defaults/config.yaml:86 locks: True You can see above that the ``build_jobs`` and ``debug`` settings are @@ -713,26 +754,62 @@ installation, these scopes can be undesirable. For example, users may want to op global system configuration, or they may want to ignore their own home directory settings when running in a continuous integration environment. -Spack also, by default, keeps various caches and user data in ``~/.spack``, but +Spack also, by default, keeps various caches and user data in ``~/.config/spack``, but users may want to override these locations. Spack provides three environment variables that allow you to override or opt out of configuration locations: * ``SPACK_USER_CONFIG_PATH``: Override the path to use for the - ``user`` scope (``~/.spack`` by default). + ``user`` scope (``~/.config/spack`` by default). Note that + ``XDG_CONFIG_HOME`` has the same effect, but with lower precedence. * ``SPACK_SYSTEM_CONFIG_PATH``: Override the path to use for the ``system`` scope (``/etc/spack`` by default). * ``SPACK_DISABLE_LOCAL_CONFIG``: Set this environment variable to completely disable - **both** the system and user configuration directories. Spack will then only consider its + **all** configurations from the system, site-admin, and user directories. Spack will then only consider its own defaults and ``site`` configuration locations. And one that allows you to move the default cache location: -* ``SPACK_USER_CACHE_PATH``: Override the default path to use for user data - (misc_cache, tests, reports, etc.) +* ``SPACK_USER_CACHE_PATH``: Override the default path to use for user + data (misc_cache, tests, reports, etc.). With these settings, if you want to isolate Spack in a CI environment, you can do this:: export SPACK_DISABLE_LOCAL_CONFIG=true export SPACK_USER_CACHE_PATH=/tmp/spack + +.. _xdg_overrides: + +------------------------------------------- +Overriding default paths with XDG variables +------------------------------------------- + +While Spack will by default use locations within the ``$spack`` and +the user's home directory, it can now store all state in locations +goverened by XDG variables. Many default storage locations have +changed in 1.0, so this section will summarize the changes, and how +XDG variables can help users organize their Spack artifacts. + +``XDG_DATA_HOME`` is used to store long-lived data. Its default value +is ``$HOME/.local/share``. The following items are always stored by +default using ``$XDG_DATA_HOME``: +* Modules: ``$XDG_DATA_HOME/share/spack`` +* Package indices: ``$XDG_DATA_HOME/$spack_instance_id/spack`` + +The data listed below will be placed in ``$spack`` by default. However, +if ``XDG_DATA_HOME`` is set, they will be stored under that path. The +default values for these are as follows: +* Source caches: ``$HOME/.local/share`` or ``$XDG_DATA_HOME/spack/downloads``. +* The install tree: ``$spack/opt/data/installs`` or ``$XDG_DATA_HOME/spack/installs``. +* Environment management: ``$spack/opt/data/environments`` or ``$XDG_DATA_HOME/spack/environments`` + +``XDG_STATE_HOME`` is used to store data that is useful if persistent, +but not integral to Spack's functionality. If not defined, its default +value is ``~/.local/state``. ``misc_cache`` is placed by default using +this variable. +* ``misc_cache``: ``$XDG_STATE_HOME/$spack_instance_id/spack`` + +The user configuration scope's files will be located with ``XDG_CONFIG_HOME``. Its +default value is ``~/.config``. +* User configuration scope: ``$XDG_CONFIG_HOME/spack`` diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index eb4db6baac7a3e..26782a3505ad9b 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -1146,12 +1146,14 @@ the key that we just created: .. code-block:: console - gpgconf: socketdir is '/run/user/1000/gnupg' - /home/spackuser/spack/opt/spack/gpg/pubring.kbx - ---------------------------------------------------------- - pub rsa4096 2021-03-25 [SC] - 60D2685DAB647AD4DB54125961E09BB6F2A0ADCB - uid [ultimate] dinosaur (GPG created for Spack) + gpg: checking the trustdb + gpg: marginals needed: 3 completes needed: 1 trust model: pgp + gpg: depth: 0 valid: 1 signed: 0 trust: 0-, 0q, 0n, 0m, 0f, 1u + /home/spackuser/.local/share/spack/gpg/pubring.kbx + ------------------------------------------------ + pub rsa4096 2025-06-06 [SC] + 85E7556B528FC163410BA1F9C30374F7F1248184 + uid [ultimate] dinosaur (GPG created for Spack) Note that the name "dinosaur" can be seen under the uid, which is the unique From 3eac9581a9faa4899f1072cce539955a6fbcd77b Mon Sep 17 00:00:00 2001 From: "Matthew L. Curry" Date: Fri, 6 Jun 2025 14:10:12 -0400 Subject: [PATCH 145/506] Update for reorder of config scopes --- lib/spack/docs/configuration.rst | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 9f039d15d85e9b..fd1d42419af8ee 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -66,19 +66,21 @@ are multiple configuration scopes. From lowest to highest precedence: here, but should override them in other configuration scopes. The defaults here will change from version to version of Spack. -#. **system**: Stored in ``/etc/spack/``. These are settings for this - machine or for all machines on which this file system is - mounted. The system scope can be used for settings idiosyncratic to a - particular machine, such as the locations of compilers or external - packages. These settings are presumably controlled by someone with - root access on the machine. They override the defaults scope. - #. **site-admin**: Stored in ``$spack/etc/site-admin``, which does not exist by default. This scope contains private settings shared by those administering a Spack instance on behalf of other users. These settings affect only *this instance* of Spack. This scope is meant to have read access restricted to control which - users are affected by these settings. + users are affected by these settings. This scope only overrides + defaults. + +#. **system**: Stored in ``/etc/spack/``. These are settings for this + machine or for all machines on which this file system is + mounted. The system scope can be used for settings idiosyncratic to + a particular machine, such as the locations of compilers or + external packages. These settings are presumably controlled by + someone with root access on the machine. They override the defaults + and site-admin scopes. #. **site**: Stored in ``$(prefix)/etc/spack/``. Settings here affect only *this instance* of Spack, and they override the defaults, @@ -298,12 +300,12 @@ set in ``$(prefix)/etc/spack/defaults/darwin/packages.yaml``, which is included as by ``$(prefix)/etc/spack/defaults/include.yaml``. Since it is an included configuration of the ``defaults`` scope, settings in the ``defaults`` scope will take precedence. You can override the -values by specifying settings in ``system``, ``site-admin``, ``site``, +values by specifying settings in ``site-admin``, ``system``, ``site``, ``user``, ``per-spack``, or ``custom``, where scope precedence is: #. ``defaults`` -#. ``system`` #. ``site-admin`` +#. ``system`` #. ``site`` #. ``user`` #. ``per-spack`` From 119cceeec7d499766cbbfb8ca2f31a6ce3b9fd6b Mon Sep 17 00:00:00 2001 From: "Matthew L. Curry" Date: Fri, 6 Jun 2025 12:33:55 -0600 Subject: [PATCH 146/506] Missed things --- lib/spack/docs/config_yaml.rst | 12 +++++++----- lib/spack/docs/configuration.rst | 5 +++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index 20f63e23d0b433..4f054482dedeb6 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -62,7 +62,7 @@ projections. For example: config: install_tree: - root: $spack/opt/spack + root: $default_install_root projections: all: "{name}/{version}/{hash:16}" @@ -102,10 +102,12 @@ This can be an ordered list of paths that Spack should search when trying to find a temporary directory for the build stage. The list is searched in order, and Spack will use the first directory to which it has write access. -Specifying `~/.spack/stage` first will ensure each user builds in their home -directory. The historic Spack stage path `$spack/var/spack/stage` will build -directly inside the Spack instance. See :ref:`config-file-variables` for more -on ``$tempdir`` and ``$spack``. +Specifying `$xdg_cache_home` first will ensure each user builds in +their home directory (or wherever the user overrides XDG_CACHE_HOME to +be; see :ref:`xdg_overrides` for more information). The historic Spack +stage path `$spack/var/spack/stage` will build directly inside the +Spack instance. See :ref:`config-file-variables` for more on +``$tempdir``, XDG variables, and ``$spack``. When Spack builds a package, it creates a temporary directory within the ``build_stage``. After the package is successfully installed, Spack deletes diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index fd1d42419af8ee..1b4f948ecdbb4f 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -812,6 +812,11 @@ value is ``~/.local/state``. ``misc_cache`` is placed by default using this variable. * ``misc_cache``: ``$XDG_STATE_HOME/$spack_instance_id/spack`` +``XDG_CACHE_HOME`` is used to store temporary data. If not defined, +its default value is ``~/.cache``. Build stages are placed by default +using this variable. +* Build stages: ``$XDG_CACHE_HOME/spack`` + The user configuration scope's files will be located with ``XDG_CONFIG_HOME``. Its default value is ``~/.config``. * User configuration scope: ``$XDG_CONFIG_HOME/spack`` From 3cbb70dee76838a976b18b31bc6431ffb0443e48 Mon Sep 17 00:00:00 2001 From: "Matthew L. Curry" Date: Fri, 6 Jun 2025 12:42:50 -0600 Subject: [PATCH 147/506] More updates --- lib/spack/docs/packages_yaml.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/spack/docs/packages_yaml.rst b/lib/spack/docs/packages_yaml.rst index 52f5afb2b9564a..cfc4466754a408 100644 --- a/lib/spack/docs/packages_yaml.rst +++ b/lib/spack/docs/packages_yaml.rst @@ -34,9 +34,10 @@ or you can specify that certain settings should apply to *all* packages. The types of settings you can customize are described in detail below. Spack's build defaults are in the default -``etc/spack/defaults/packages.yaml`` file. You can override them in -``~/.spack/packages.yaml`` or ``etc/spack/packages.yaml``. For more -details on how this works, see :ref:`configuration-scopes`. +``$spack/etc/spack/defaults/packages.yaml`` file. You can override +them in ``~/.config/spack/packages.yaml`` or +``$spack/etc/spack/packages.yaml``. For more details on how this +works, see :ref:`configuration-scopes`. .. _sec-external-packages: From 7c3df41d616beb28df9c1fb68969d7b44b25e624 Mon Sep 17 00:00:00 2001 From: "Matthew L. Curry" Date: Fri, 6 Jun 2025 12:52:20 -0600 Subject: [PATCH 148/506] Update config file comments to reflect overrides --- etc/spack/defaults/base/packages.yaml | 2 +- etc/spack/defaults/concretizer.yaml | 5 +++-- etc/spack/defaults/config.yaml | 2 +- etc/spack/defaults/darwin/modules.yaml | 2 +- etc/spack/defaults/darwin/packages.yaml | 2 +- etc/spack/defaults/linux/modules.yaml | 2 +- etc/spack/defaults/modules.yaml | 2 +- etc/spack/defaults/repos.yaml | 2 +- etc/spack/defaults/windows/packages.yaml | 2 +- 9 files changed, 11 insertions(+), 10 deletions(-) diff --git a/etc/spack/defaults/base/packages.yaml b/etc/spack/defaults/base/packages.yaml index 6cc366511b59d6..f3c397ee8d66ef 100644 --- a/etc/spack/defaults/base/packages.yaml +++ b/etc/spack/defaults/base/packages.yaml @@ -11,7 +11,7 @@ # $SPACK_ROOT/etc/spack/packages.yaml # # Per-user settings (overrides default and site settings): -# ~/.spack/packages.yaml +# ~/.config/spack/packages.yaml # ------------------------------------------------------------------------- packages: all: diff --git a/etc/spack/defaults/concretizer.yaml b/etc/spack/defaults/concretizer.yaml index 0d1042dee258f3..3acaf831e0ee08 100644 --- a/etc/spack/defaults/concretizer.yaml +++ b/etc/spack/defaults/concretizer.yaml @@ -6,8 +6,9 @@ # file to keep it current. # # Users can override these settings by editing -# `$SPACK_ROOT/etc/spack/concretizer.yaml`, `~/.spack/concretizer.yaml`, -# or by adding a `concretizer:` section to an environment. +# `$SPACK_ROOT/etc/spack/concretizer.yaml`, +# `~/.config/spack/concretizer.yaml`, or by adding a `concretizer:` +# section to an environment. # ------------------------------------------------------------------------- concretizer: # Whether to consider installed packages or packages from buildcaches when diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 3a123a5f0cde69..2b9ba5771a2706 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -11,7 +11,7 @@ # $SPACK_ROOT/etc/spack/config.yaml # # Per-user settings (overrides default and site settings): -# ~/.spack/config.yaml +# ~/.config/spack/config.yaml # ------------------------------------------------------------------------- config: # This is the path to the root of the Spack install tree. diff --git a/etc/spack/defaults/darwin/modules.yaml b/etc/spack/defaults/darwin/modules.yaml index b913d1029ea75b..9192f8cc76f5d1 100644 --- a/etc/spack/defaults/darwin/modules.yaml +++ b/etc/spack/defaults/darwin/modules.yaml @@ -11,7 +11,7 @@ # $SPACK_ROOT/etc/spack/modules.yaml # # Per-user settings (overrides default and site settings): -# ~/.spack/modules.yaml +# ~/.config/spack/modules.yaml # ------------------------------------------------------------------------- modules: prefix_inspections: diff --git a/etc/spack/defaults/darwin/packages.yaml b/etc/spack/defaults/darwin/packages.yaml index 32a3b940409014..60dfaf8279af16 100644 --- a/etc/spack/defaults/darwin/packages.yaml +++ b/etc/spack/defaults/darwin/packages.yaml @@ -11,7 +11,7 @@ # $SPACK_ROOT/etc/spack/packages.yaml # # Per-user settings (overrides default and site settings): -# ~/.spack/packages.yaml +# ~/.config/spack/packages.yaml # ------------------------------------------------------------------------- packages: all: diff --git a/etc/spack/defaults/linux/modules.yaml b/etc/spack/defaults/linux/modules.yaml index a80f87b16ab400..947ea7dc360049 100644 --- a/etc/spack/defaults/linux/modules.yaml +++ b/etc/spack/defaults/linux/modules.yaml @@ -11,6 +11,6 @@ # $SPACK_ROOT/etc/spack/modules.yaml # # Per-user settings (overrides default and site settings): -# ~/.spack/modules.yaml +# ~/.config/spack/modules.yaml # ------------------------------------------------------------------------- modules: {} diff --git a/etc/spack/defaults/modules.yaml b/etc/spack/defaults/modules.yaml index b152e967a66b5d..26a085b4a5c52b 100644 --- a/etc/spack/defaults/modules.yaml +++ b/etc/spack/defaults/modules.yaml @@ -11,7 +11,7 @@ # $SPACK_ROOT/etc/spack/modules.yaml # # Per-user settings (overrides default and site settings): -# ~/.spack/modules.yaml +# ~/.config/spack/modules.yaml # ------------------------------------------------------------------------- modules: # This maps paths in the package install prefix to environment variables diff --git a/etc/spack/defaults/repos.yaml b/etc/spack/defaults/repos.yaml index 48a9469d6df3b9..57c9c9321a1c79 100644 --- a/etc/spack/defaults/repos.yaml +++ b/etc/spack/defaults/repos.yaml @@ -8,7 +8,7 @@ # $SPACK_ROOT/etc/spack/repos.yaml # # Per-user settings (overrides default and site settings): -# ~/.spack/repos.yaml +# ~/.config/spack/repos.yaml # ------------------------------------------------------------------------- repos: builtin: diff --git a/etc/spack/defaults/windows/packages.yaml b/etc/spack/defaults/windows/packages.yaml index f18ea28023f586..71e1bf9b278fcd 100644 --- a/etc/spack/defaults/windows/packages.yaml +++ b/etc/spack/defaults/windows/packages.yaml @@ -11,7 +11,7 @@ # $SPACK_ROOT/etc/spack/packages.yaml # # Per-user settings (overrides default and site settings): -# ~/.spack/packages.yaml +# ~/.config/spack/packages.yaml # ------------------------------------------------------------------------- packages: all: From df85b09d2baefd80e32e01c344ca1a0c23aa4a71 Mon Sep 17 00:00:00 2001 From: "Matthew L. Curry" Date: Fri, 6 Jun 2025 13:07:10 -0600 Subject: [PATCH 149/506] Bug with bullets --- lib/spack/docs/configuration.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 1b4f948ecdbb4f..60dcf649e8cc93 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -796,12 +796,14 @@ XDG variables can help users organize their Spack artifacts. ``XDG_DATA_HOME`` is used to store long-lived data. Its default value is ``$HOME/.local/share``. The following items are always stored by default using ``$XDG_DATA_HOME``: + * Modules: ``$XDG_DATA_HOME/share/spack`` * Package indices: ``$XDG_DATA_HOME/$spack_instance_id/spack`` The data listed below will be placed in ``$spack`` by default. However, if ``XDG_DATA_HOME`` is set, they will be stored under that path. The default values for these are as follows: + * Source caches: ``$HOME/.local/share`` or ``$XDG_DATA_HOME/spack/downloads``. * The install tree: ``$spack/opt/data/installs`` or ``$XDG_DATA_HOME/spack/installs``. * Environment management: ``$spack/opt/data/environments`` or ``$XDG_DATA_HOME/spack/environments`` @@ -810,13 +812,16 @@ default values for these are as follows: but not integral to Spack's functionality. If not defined, its default value is ``~/.local/state``. ``misc_cache`` is placed by default using this variable. + * ``misc_cache``: ``$XDG_STATE_HOME/$spack_instance_id/spack`` ``XDG_CACHE_HOME`` is used to store temporary data. If not defined, its default value is ``~/.cache``. Build stages are placed by default using this variable. + * Build stages: ``$XDG_CACHE_HOME/spack`` The user configuration scope's files will be located with ``XDG_CONFIG_HOME``. Its default value is ``~/.config``. + * User configuration scope: ``$XDG_CONFIG_HOME/spack`` From 3045c3427479f64ec6a1e0a20275d8dfaeeb0e06 Mon Sep 17 00:00:00 2001 From: "Matthew L. Curry" Date: Fri, 6 Jun 2025 14:24:10 -0600 Subject: [PATCH 150/506] More --- lib/spack/docs/environments.rst | 8 ++++---- lib/spack/docs/mirrors.rst | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index a9301f64551349..a13d4feff02358 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -74,17 +74,17 @@ An environment is created by: $ spack env create myenv -The directory ``$SPACK_ROOT/var/spack/environments/myenv`` is created +The directory ``$SPACK_ROOT/opt/data/environments/myenv`` is created to manage the environment. .. note:: By default, all managed environments are stored in the - ``$SPACK_ROOT/var/spack/environments`` folder. This location can be changed + ``$SPACK_ROOT/opt/data/environments`` folder. This location can be changed by setting the ``environments_root`` variable in ``config.yaml``. Spack creates the file ``spack.yaml``, hidden directory ``.spack-env``, and -``spack.lock`` file under ``$SPACK_ROOT/var/spack/environments/myenv``. User +``spack.lock`` file under ``$SPACK_ROOT/opt/data/environments/environments/myenv``. User interaction occurs through the ``spack.yaml`` file and the Spack commands that affect it. Metadata and, by default, the view are stored in the ``.spack-env`` directory. When the environment is concretized, Spack creates @@ -138,7 +138,7 @@ guaranteed to initially have the same concrete specs as the original. Environment creation also accepts a full path to the file. - If the path is not under the ``$SPACK_ROOT/var/spack/environments`` + If the path is not under the ``$SPACK_ROOT/opt/data/environments`` directory then the source is referred to as an :ref:`independent environment `. diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index f6ab067867ebc4..da5df9838b44d4 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -260,7 +260,7 @@ Spack caches resources that are downloaded as part of installations. The cache i a valid Spack mirror: it uses the same directory structure and naming scheme as other Spack mirrors (so it can be copied anywhere and referenced with a URL like other mirrors). The mirror is maintained locally (within the Spack -installation directory) at :file:`var/spack/cache/`. It is always enabled (and +installation directory) at :file:`$spack/opt/data`. It is always enabled (and is always searched first when attempting to retrieve files for an installation) but can be cleared with :ref:`clean `; the cache directory can also be deleted manually without issue. From 9158c32621385bfb6932f1cf2bac88436a85cdf4 Mon Sep 17 00:00:00 2001 From: "Matthew L. Curry" Date: Fri, 6 Jun 2025 14:25:43 -0600 Subject: [PATCH 151/506] Bug fix --- lib/spack/docs/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 60dcf649e8cc93..45e34de8fcde28 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -797,7 +797,7 @@ XDG variables can help users organize their Spack artifacts. is ``$HOME/.local/share``. The following items are always stored by default using ``$XDG_DATA_HOME``: -* Modules: ``$XDG_DATA_HOME/share/spack`` +* Modules: ``$XDG_DATA_HOME/spack/`` * Package indices: ``$XDG_DATA_HOME/$spack_instance_id/spack`` The data listed below will be placed in ``$spack`` by default. However, From ef72c75220d0185aff21328f5f5ac7a5b901a843 Mon Sep 17 00:00:00 2001 From: "Matthew L. Curry" Date: Fri, 6 Jun 2025 15:01:40 -0600 Subject: [PATCH 152/506] More --- lib/spack/docs/build_systems/inteloneapipackage.rst | 2 +- lib/spack/docs/chain.rst | 2 +- lib/spack/docs/configuration.rst | 2 +- lib/spack/docs/developer_guide.rst | 2 +- lib/spack/docs/getting_started.rst | 2 +- lib/spack/docs/mirrors.rst | 2 +- lib/spack/docs/module_file_support.rst | 10 +++++----- lib/spack/docs/packaging_guide.rst | 2 +- lib/spack/docs/repositories.rst | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/spack/docs/build_systems/inteloneapipackage.rst b/lib/spack/docs/build_systems/inteloneapipackage.rst index 51e5af9788a4a3..941dbd158635ee 100644 --- a/lib/spack/docs/build_systems/inteloneapipackage.rst +++ b/lib/spack/docs/build_systems/inteloneapipackage.rst @@ -133,7 +133,7 @@ Libraries If you want Spack to use oneMKL that you have installed without Spack in the default location, then add the following to -``~/.spack/packages.yaml``, adjusting the version as appropriate:: +``~/.config/spack/packages.yaml``, adjusting the version as appropriate:: intel-oneapi-mkl: externals: diff --git a/lib/spack/docs/chain.rst b/lib/spack/docs/chain.rst index 28aa84b3d5ade7..9ca9e83559ec51 100644 --- a/lib/spack/docs/chain.rst +++ b/lib/spack/docs/chain.rst @@ -21,7 +21,7 @@ you can add it as an entry to ``upstreams.yaml`` at any of the spack-instance-2: install_tree: /path/to/another/spack/opt/spack -The ``install_tree`` must point to the ``opt/spack`` directory inside of the +The ``install_tree`` must point to the ``opt/data/installs`` directory inside of the Spack base directory, or the location of the ``install_tree`` defined in :ref:`config.yaml `. diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 45e34de8fcde28..b53a904038ecf2 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -575,7 +575,7 @@ Spack understands over a dozen special variables. These are: `_ variable. * ``$user``: name of the current user -* ``$user_cache_path``: user cache directory (``~/.spack`` unless +* ``$user_cache_path``: user cache directory (``~/.local/share/spack`` unless :ref:`overridden `) * ``$architecture``: the architecture triple of the current host, as detected by Spack. diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index 4b8efce4b845e4..372eb9c972c4dc 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -84,7 +84,7 @@ with a high-level view of Spack's directory structure: etc/ spack/ <- Spack config files. - Can be overridden by files in ~/.spack. + Can be overridden by files in ~/.config/spack. var/ spack/ <- build & stage directories diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index 26782a3505ad9b..c0514d6777d19b 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -188,7 +188,7 @@ The Bootstrap Store """"""""""""""""""" All the tools Spack needs for its own functioning are installed in a separate store, which lives -under the ``${HOME}/.spack`` directory. The software installed there can be queried with: +under the ``${HOME}/.local/share/spack`` directory. The software installed there can be queried with: .. code-block:: console diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index da5df9838b44d4..783b6f26c5fd46 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -238,7 +238,7 @@ To remove a mirror by name, run: Mirror precedence ----------------- -Adding a mirror really adds a line in ``~/.spack/mirrors.yaml``: +Adding a mirror really adds a line in ``~/.config/spack/mirrors.yaml``: .. code-block:: yaml diff --git a/lib/spack/docs/module_file_support.rst b/lib/spack/docs/module_file_support.rst index cd392353a92296..ca8139f2e40c20 100644 --- a/lib/spack/docs/module_file_support.rst +++ b/lib/spack/docs/module_file_support.rst @@ -57,7 +57,7 @@ to interact with them: $ module avail - --------------------------------------------------------------- ~/spack/share/spack/modules/linux-ubuntu14-x86_64 --------------------------------------------------------------- + --------------------------------------------------------------- ~/.local/share/spack/modules/linux-ubuntu14-x86_64 --------------------------------------------------------------- autoconf/2.69-gcc-4.8-qextxkq hwloc/1.11.6-gcc-6.3.0-akcisez m4/1.4.18-gcc-4.8-ev2znoc openblas/0.2.19-gcc-6.3.0-dhkmed6 py-setuptools/34.2.0-gcc-6.3.0-fadur4s automake/1.15-gcc-4.8-maqvukj isl/0.18-gcc-4.8-afi6taq m4/1.4.18-gcc-6.3.0-uppywnz openmpi/2.1.0-gcc-6.3.0-go2s4z5 py-six/1.10.0-gcc-6.3.0-p4dhkaw binutils/2.28-gcc-4.8-5s7c6rs libiconv/1.15-gcc-4.8-at46wg3 mawk/1.3.4-gcc-4.8-acjez57 openssl/1.0.2k-gcc-4.8-dkls5tk python/2.7.13-gcc-6.3.0-tyehea7 @@ -95,9 +95,9 @@ the different file formats that can be generated by Spack: +-----------+--------------+------------------------------+----------------------------------------------+----------------------+ | | Hierarchical | **Default root directory** | **Default template file** | **Compatible tools** | +===========+==============+==============================+==============================================+======================+ - | ``tcl`` | No | share/spack/modules | share/spack/templates/modules/modulefile.tcl | Env. Modules/Lmod | + | ``tcl`` | No | ~/.local/share/spack/modules | share/spack/templates/modules/modulefile.tcl | Env. Modules/Lmod | +-----------+--------------+------------------------------+----------------------------------------------+----------------------+ - | ``lmod`` | Yes | share/spack/lmod | share/spack/templates/modules/modulefile.lua | Lmod | + | ``lmod`` | Yes | ~/.local/share/spack/lmod | share/spack/templates/modules/modulefile.lua | Lmod | +-----------+--------------+------------------------------+----------------------------------------------+----------------------+ @@ -175,8 +175,8 @@ Changing the modules root ^^^^^^^^^^^^^^^^^^^^^^^^^ As shown in the table above, the default module root for ``lmod`` is -``$spack/share/spack/lmod`` and the default root for ``tcl`` is -``$spack/share/spack/modules``. This can be overridden for any module +``~/.local/share/spack/lmod`` and the default root for ``tcl`` is +``~/.local/share/spack/modules``. This can be overridden for any module set by changing the ``roots`` key of the configuration. .. code-block:: yaml diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 9a87e015cbd4d0..176e4d0fc8b7b3 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -5255,7 +5255,7 @@ Configuring the test stage directory Stand-alone tests utilize a test stage directory to build, run, and track tests in the same way Spack uses a build stage directory to install software. -The default test stage root directory, ``$HOME/.spack/test``, is defined in +The default test stage root directory, ``$user_cache_path/test``, is defined in :ref:`config.yaml `. This location is customizable by adding or changing the ``test_stage`` path such that: diff --git a/lib/spack/docs/repositories.rst b/lib/spack/docs/repositories.rst index d7089351cb35c5..eeeeabfd9a9540 100644 --- a/lib/spack/docs/repositories.rst +++ b/lib/spack/docs/repositories.rst @@ -61,7 +61,7 @@ them in your own repository. ``repos.yaml`` --------------------- -Spack uses the ``repos.yaml`` file in ``~/.spack`` (and :ref:`elsewhere +Spack uses the ``repos.yaml`` file in ``~/.config/spack`` (and :ref:`elsewhere `) to find repositories. Note that the ``repos.yaml`` configuration file is distinct from the ``repo.yaml`` file in each repository. For more on the YAML format, and on how configuration file From c09c420c69307c45de97dc6c15a2a18b1b31468e Mon Sep 17 00:00:00 2001 From: "Matthew L. Curry" Date: Fri, 6 Jun 2025 15:15:52 -0600 Subject: [PATCH 153/506] Add discussion about SPACK_*_HOME variables --- lib/spack/docs/configuration.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index b53a904038ecf2..bf26234a91977d 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -825,3 +825,11 @@ The user configuration scope's files will be located with ``XDG_CONFIG_HOME``. I default value is ``~/.config``. * User configuration scope: ``$XDG_CONFIG_HOME/spack`` + +Spack also includes the variables ``SPACK_DATA_HOME``, +``SPACK_CONFIG_HOME``, and ``SPACK_STATE_HOME`` that map directly to +the XDG variables described above. They work the same way, but have +higher precedence than the XDG variables. If the Spack-specific +variables *are not* defined, Spack uses the XDG variables with a suffix +of "/spack". If they are defined, they are used directly without any +additional suffix. From bcc6967741f42d8be606a807abd83072a44f4a61 Mon Sep 17 00:00:00 2001 From: "Matthew L. Curry" Date: Fri, 6 Jun 2025 15:26:54 -0600 Subject: [PATCH 154/506] Change from xdg_ to spack_ --- lib/spack/docs/config_yaml.rst | 6 +++--- lib/spack/docs/configuration.rst | 31 ++++++++++++++++--------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index 4f054482dedeb6..6d251d66dbd32a 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -96,14 +96,14 @@ By default, Spack's ``build_stage`` is configured like this: build_stage: - $tempdir/$user/spack-stage - - $xdg_cache_home/stage + - $spack_cache_home/stage This can be an ordered list of paths that Spack should search when trying to find a temporary directory for the build stage. The list is searched in order, and Spack will use the first directory to which it has write access. -Specifying `$xdg_cache_home` first will ensure each user builds in -their home directory (or wherever the user overrides XDG_CACHE_HOME to +Specifying `$spack_cache_home` first will ensure each user builds in +their home directory (or wherever the user overrides ``XDG_CACHE_HOME`` to be; see :ref:`xdg_overrides` for more information). The historic Spack stage path `$spack/var/spack/stage` will build directly inside the Spack instance. See :ref:`config-file-variables` for more on diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index bf26234a91977d..7486e6203c980c 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -40,7 +40,7 @@ Here is an example ``config.yaml`` file: install_tree: $spack/opt/data/installs build_stage: - $tempdir/$user/spack-stage - - $xdg_cache_home/stage + - $spack_cache_home/stage Each Spack configuration file is nested under a top-level section corresponding to its name. So, ``config.yaml`` starts with ``config:``, @@ -383,7 +383,7 @@ your configurations look like this: install_tree: $spack/opt/data/installs build_stage: - $tempdir/$user/spack-stage - - $xdg_cache_home/stage + - $spack_cache_home/stage .. code-block:: yaml @@ -406,7 +406,7 @@ command: install_tree: /some/other/directory build_stage: - $tempdir/$user/spack-stage - - $xdg_cache_home/stage + - $spack_cache_home/stage .. _config-prepend-append: @@ -437,7 +437,7 @@ Spack will then append to the lower-precedence configuration under the install_tree: /some/other/directory/my/custom/suffix build_stage: - $tempdir/$user/spack-stage - - $xdg_cache_home/stage + - $spack_cache_home/stage Similarly, ``+:`` can be used to *prepend* to a path or name: @@ -490,7 +490,7 @@ Let's revisit the ``config.yaml`` example one more time. The build_stage: - $tempdir/$user/spack-stage - - $xdg_cache_home/stage + - $spack_cache_home/stage Suppose the user configuration adds its *own* list of ``build_stage`` @@ -519,7 +519,7 @@ get config`` shows the result: - /lustre-scratch/$user/spack - ~/mystage - $tempdir/$user/spack-stage - - $xdg_cache_home/stage + - $spack_cache_home/stage As in :ref:`config-overrides`, the higher-precedence scope can @@ -591,8 +591,8 @@ Spack understands over a dozen special variables. These are: * ``$date``: the current date in the format YYYY-MM-DD * ``$spack_short_version``: the Spack version truncated to the first components. * ``$spack_instance_id``: a hash that distinguishes Spack instances on the filesystem. -* ``$xdg_state_home``: the XDG location for long-lived but not-essential cache. -* ``$xdg_cache_home``: the XDG location for temporary data. +* ``$spack_state_home``: the XDG-derived location for long-lived but not-essential cache. +* ``$spack_cache_home``: the XDG-derived location for temporary data. * ``$default_download_root``: the location where downloads go by default. * ``$default_envs_root``: the location where environments are managed by default. * ``$default_install_root``: the location where installs go by default. @@ -686,9 +686,9 @@ account all scopes. For example, to see the fully merged directory_layout: {architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash} build_stage: - $tempdir/$user/spack-stage - - $xdg_cache_home/stage + - $spack_cache_home/stage source_cache: $default_download_root - misc_cache: $xdg_state_home/$spack_instance_id/cache + misc_cache: $spack_state_home/$spack_instance_id/cache locks: true Likewise, this will show the fully merged ``packages.yaml``: @@ -732,9 +732,9 @@ down the source of the configuration: /home/myuser/spack/etc/spack/defaults/config.yaml:28 directory_layout: {architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash} /home/myuser/spack/etc/spack/defaults/config.yaml:49 build_stage: /home/myuser/spack/etc/spack/defaults/config.yaml:50 - $tempdir/$user/spack-stage - /home/myuser/spack/etc/spack/defaults/config.yaml:51 - $xdg_cache_home/stage + /home/myuser/spack/etc/spack/defaults/config.yaml:51 - $spack_cache_home/stage /home/myuser/spack/etc/spack/defaults/config.yaml:57 source_cache: $default_download_root - /home/myuser/spack/etc/spack/defaults/config.yaml:62 misc_cache: $xdg_state_home/$spack_instance_id/cache + /home/myuser/spack/etc/spack/defaults/config.yaml:62 misc_cache: $spack_state_home/$spack_instance_id/cache /home/myuser/spack/etc/spack/defaults/config.yaml:86 locks: True You can see above that the ``build_jobs`` and ``debug`` settings are @@ -830,6 +830,7 @@ Spack also includes the variables ``SPACK_DATA_HOME``, ``SPACK_CONFIG_HOME``, and ``SPACK_STATE_HOME`` that map directly to the XDG variables described above. They work the same way, but have higher precedence than the XDG variables. If the Spack-specific -variables *are not* defined, Spack uses the XDG variables with a suffix -of "/spack". If they are defined, they are used directly without any -additional suffix. +environment variables *are not* defined, Spack will uses the XDG +variables with a suffix of "/spack" to define ``$spack_state_home``, +``$spack_data_home``, and ``spack_cache_home``. If they are defined, +they are used directly without any additional suffix. From a4654787e3d721a48b67cb96294b5e833257ad03 Mon Sep 17 00:00:00 2001 From: "Matthew L. Curry" Date: Fri, 6 Jun 2025 15:40:10 -0600 Subject: [PATCH 155/506] Peter feedback edits --- lib/spack/docs/config_yaml.rst | 6 +++--- lib/spack/docs/configuration.rst | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index 6d251d66dbd32a..c5425fc2745b98 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -125,9 +125,9 @@ deleted, but you can manually purge them with :ref:`spack clean --stage -------------------- Location to cache downloaded tarballs and repositories. By default, -these are stored in ``$spack/opt/data``. These are stored indefinitely -by default and can be purged with :ref:`spack clean --downloads -`. +these are stored in ``$spack/opt/data/downloads``. These are stored +indefinitely by default and can be purged with :ref:`spack +clean --downloads `. .. _Misc Cache: diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 7486e6203c980c..615d8a69d0458c 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -97,7 +97,7 @@ are multiple configuration scopes. From lowest to highest precedence: Spack and take higher precedence than site, system, plugin, or defaults scopes. -#. **per-spack**: Stored in the home directory: +#. **this-spack**: Stored in the home directory: ``~/.local/config/spack/$spack_instance_id/``. These settings affect the instance of spack that corresponds to the specified ``$spack_instance_id``. This scope is most useful for overriding @@ -166,7 +166,7 @@ Scopes and XDG Compliance Spack respects XDG variables, and the search location for user configuration files can be affected by them. When ``XDG_CONFIG_HOME`` is not defined, Spack assumes the XDG default value of ``~/.config``, -and will apply user and per-spack configuration files located in +and will apply user and this-spack configuration files located in ``~/.config/spack``. Defining ``XDG_CONFIG_HOME`` will change where Spack searches for configuration files. To override this behavior, define ``SPACK_USER_CONFIG_PATH`` to be the desired path. For more @@ -301,14 +301,14 @@ included as by ``$(prefix)/etc/spack/defaults/include.yaml``. Since it is an included configuration of the ``defaults`` scope, settings in the ``defaults`` scope will take precedence. You can override the values by specifying settings in ``site-admin``, ``system``, ``site``, -``user``, ``per-spack``, or ``custom``, where scope precedence is: +``user``, ``this-spack``, or ``custom``, where scope precedence is: #. ``defaults`` #. ``site-admin`` #. ``system`` #. ``site`` #. ``user`` -#. ``per-spack`` +#. ``this-spack`` #. ``custom`` and settings in each scope taking precedence over those found in configuration From ef4b34c41e3d3ca4f5ac1c3b894144a00c046e00 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 15:18:19 -0700 Subject: [PATCH 156/506] remove comment (was from debugging, for reviewers) --- lib/spack/spack/test/data/config/config.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/spack/spack/test/data/config/config.yaml b/lib/spack/spack/test/data/config/config.yaml index 2524faba3b208b..fc50b4b7c02f0b 100644 --- a/lib/spack/spack/test/data/config/config.yaml +++ b/lib/spack/spack/test/data/config/config.yaml @@ -7,12 +7,6 @@ config: - $spack/lib/spack/spack/test/data/templates_again build_stage: - $tempdir/$user/spack-stage - # You can also unset these next two to "solve" the - # problem: it goes away because in a multi-package - # install, the installations subprocs won't be - # trying to simultaneously generate the cache indices - # at all (for some reason, the test config is not - # transmitted to the subprocesses on windows) source_cache: $user_cache_path/source misc_cache: $user_cache_path/cache verify_ssl: true From c12222c67c7c1eeef4dd0adbe43b525f105cd60b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 16:18:43 -0700 Subject: [PATCH 157/506] use more-general concurrency limit just added to main branch --- lib/spack/spack/installer.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index cb6292a9bc7daa..f628d53babc55f 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -1428,16 +1428,6 @@ def complete(self): self.record.fail(e) -if sys.platform == "win32": - # This seems like the real problem: AFAIK we don't have - # locks on Windows (we still skip the entire llnl.util.lock - # test module there) and we shouldn't be doing concurrent - # installs there. - _default_concurrent_procs = 1 -else: - _default_concurrent_procs = 4 - - class PackageInstaller: """ Class for managing the install process for a Spack instance based on a bottom-up DAG approach. @@ -1474,7 +1464,7 @@ def __init__( unsigned: Optional[bool] = None, use_cache: bool = False, verbose: bool = False, - concurrent_packages: int = _default_concurrent_procs, + concurrent_packages: int = 4, ) -> None: """ Arguments: From 5a7c990bbe28c8c3c46ada680a2a5d30e8fe0ce0 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 17:13:19 -0700 Subject: [PATCH 158/506] minor sync of docs and logic --- lib/spack/docs/config_yaml.rst | 4 +--- lib/spack/docs/environments.rst | 2 +- lib/spack/spack/paths.py | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index 078190afde8b71..763df2a60e06f4 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -104,9 +104,7 @@ order, and Spack will use the first directory to which it has write access. Specifying `$spack_cache_home` first will ensure each user builds in their home directory (or wherever the user overrides ``XDG_CACHE_HOME`` to -be; see :ref:`xdg_overrides` for more information). The historic Spack -stage path `$spack/var/spack/stage` will build directly inside the -Spack instance. See :ref:`config-file-variables` for more on +be; see :ref:`xdg_overrides` and :ref:`config-file-variables` for more on ``$tempdir``, XDG variables, and ``$spack``. When Spack builds a package, it creates a temporary directory within the diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index 1322a200c3837e..25a4e6b1c90215 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -263,7 +263,7 @@ the Spack instance. The same rule applies to the ``install`` and ==> No binary for zlib-1.2.8-yfc7epf57nsfn2gn4notccaiyxha6z7x found: installing from source ==> zlib: Executing phase: 'install' [+] ~/spack/opt/spack/linux-rhel7-broadwell/gcc-8.1.0/zlib-1.2.8-yfc7epf57nsfn2gn4notccaiyxha6z7x - ==> Updating view at ~/spack/var/spack/environments/myenv/.spack-env/view + ==> Updating view at ~/spack/opt/data/environments/myenv/.spack-env/view $ spack find ==> In environment myenv diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index d4f9cb44673589..cbec134a85c457 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -243,7 +243,7 @@ def __init__(self, _prefix=None): if dir_is_occupied(os.path.join(self.share_path, module_dir)): self.modules_base = self.share_path if not self.modules_base: - self.modules_base = os.path.join(self.user_cache_path, "modules") + self.modules_base = self.user_cache_path # ------ Next section # Spack can also write data into the following locations, and their From 48a510a9e0742c087fcb19b9b7720fcfc21f14e9 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 17:49:46 -0700 Subject: [PATCH 159/506] I think aligning default config with paths.py will fix mac os test issue (because subprocs dont get config) --- etc/spack/defaults/concretizer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/spack/defaults/concretizer.yaml b/etc/spack/defaults/concretizer.yaml index fd042aecbfdb13..f4ec31b1595c55 100644 --- a/etc/spack/defaults/concretizer.yaml +++ b/etc/spack/defaults/concretizer.yaml @@ -93,4 +93,4 @@ concretizer: concretization_cache: enable: true - url: $user_cache_path/concretization + url: $spack_state_home/$spack_instance_id/cache/concretization From 1cb21f1117c45bec6629899982406928089f3080 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 18:01:59 -0700 Subject: [PATCH 160/506] actually match --- etc/spack/defaults/concretizer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/spack/defaults/concretizer.yaml b/etc/spack/defaults/concretizer.yaml index f4ec31b1595c55..d670467a61d8f4 100644 --- a/etc/spack/defaults/concretizer.yaml +++ b/etc/spack/defaults/concretizer.yaml @@ -93,4 +93,4 @@ concretizer: concretization_cache: enable: true - url: $spack_state_home/$spack_instance_id/cache/concretization + url: $spack_state_home/$spack_instance_id/misc-cache/concretization From fe917c23d295086bb877b70ca58d7640acedf30b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 21:31:44 -0700 Subject: [PATCH 161/506] another attempt to sync misc cache and concretization_cache url --- etc/spack/defaults/concretizer.yaml | 2 +- etc/spack/defaults/config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/spack/defaults/concretizer.yaml b/etc/spack/defaults/concretizer.yaml index d670467a61d8f4..00d4355b8f725e 100644 --- a/etc/spack/defaults/concretizer.yaml +++ b/etc/spack/defaults/concretizer.yaml @@ -93,4 +93,4 @@ concretizer: concretization_cache: enable: true - url: $spack_state_home/$spack_instance_id/misc-cache/concretization + url: $user_cache_path/cache/concretization diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 2b9ba5771a2706..93ffd14b548d3e 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -105,7 +105,7 @@ config: # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` - misc_cache: $spack_state_home/$spack_instance_id/cache + misc_cache: $user_cache_path/cache # Abort downloads after this many seconds if not data is received. From 3f54cdc0432cbe55eb9a9a0b80c41891fcfbdca2 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 22:12:32 -0700 Subject: [PATCH 162/506] other sync I missed for misc cache / concretizer cache --- lib/spack/spack/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index cbec134a85c457..d717deeba80f26 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -276,7 +276,7 @@ def __init__(self, _prefix=None): #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) #: overridden by `config:misc_cache` self.default_misc_cache_path = os.path.join( - self.spack_state_home, self.spack_instance_id, "misc-cache" + self.user_cache_path, "cache" ) #: concretization cache for Spack concretizations From 9d53c9b84cda7297b293ee16a14be50b5013d335 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 22:17:13 -0700 Subject: [PATCH 163/506] style fix --- lib/spack/spack/paths.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index d717deeba80f26..95431b3924df6f 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -275,9 +275,7 @@ def __init__(self, _prefix=None): #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) #: overridden by `config:misc_cache` - self.default_misc_cache_path = os.path.join( - self.user_cache_path, "cache" - ) + self.default_misc_cache_path = os.path.join(self.user_cache_path, "cache") #: concretization cache for Spack concretizations #: overridden by `config:concretization_cache:url` From a38efaf0a48bf5c90c59f36a502a2a7e429b0246 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Jun 2025 23:36:01 -0700 Subject: [PATCH 164/506] ok now try to sync it to be the thing I want it to be --- etc/spack/defaults/concretizer.yaml | 2 +- etc/spack/defaults/config.yaml | 2 +- lib/spack/spack/paths.py | 5 +++-- lib/spack/spack/test/data/config/config.yaml | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/etc/spack/defaults/concretizer.yaml b/etc/spack/defaults/concretizer.yaml index 00d4355b8f725e..f4ec31b1595c55 100644 --- a/etc/spack/defaults/concretizer.yaml +++ b/etc/spack/defaults/concretizer.yaml @@ -93,4 +93,4 @@ concretizer: concretization_cache: enable: true - url: $user_cache_path/cache/concretization + url: $spack_state_home/$spack_instance_id/cache/concretization diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 93ffd14b548d3e..2b9ba5771a2706 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -105,7 +105,7 @@ config: # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` - misc_cache: $user_cache_path/cache + misc_cache: $spack_state_home/$spack_instance_id/cache # Abort downloads after this many seconds if not data is received. diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 95431b3924df6f..4d09271a20aa30 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -275,8 +275,9 @@ def __init__(self, _prefix=None): #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) #: overridden by `config:misc_cache` - self.default_misc_cache_path = os.path.join(self.user_cache_path, "cache") - + self.default_misc_cache_path = os.path.join( + self.spack_state_home, self.spack_instance_id, "cache" + ) #: concretization cache for Spack concretizations #: overridden by `config:concretization_cache:url` self.default_conc_cache_path = os.path.join(self.default_misc_cache_path, "concretization") diff --git a/lib/spack/spack/test/data/config/config.yaml b/lib/spack/spack/test/data/config/config.yaml index e6867adb3db9b2..62ce38134c8341 100644 --- a/lib/spack/spack/test/data/config/config.yaml +++ b/lib/spack/spack/test/data/config/config.yaml @@ -8,7 +8,7 @@ config: build_stage: - $tempdir/$user/spack-stage source_cache: $user_cache_path/source - misc_cache: $user_cache_path/cache + misc_cache: $spack_state_home/$spack_instance_id/cache verify_ssl: true ssl_certs: $SSL_CERT_FILE checksum: true From a246f854255fcab08b7013e6547f909401c16c82 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sat, 7 Jun 2025 15:41:43 -0700 Subject: [PATCH 165/506] try dist loadscope to reduce parallelism in tests (not a real fix) --- .github/workflows/unit_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 7b6d45d738f8ed..3d6f99797d892c 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -221,7 +221,7 @@ jobs: $(which spack) bootstrap disable spack-install $(which spack) solve zlib common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x) - $(which spack) unit-test --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}" + $(which spack) unit-test --dist=loadscope --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}" - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b with: name: coverage-${{ matrix.os }}-python${{ matrix.python-version }} From c0a46415b27253dcfd963dcd99e9da7f802c640a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sat, 7 Jun 2025 20:18:39 -0700 Subject: [PATCH 166/506] keep cache keys separate for parallel tests --- lib/spack/spack/test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 7dc5241dd20ef2..56b799ffb4d091 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -1068,7 +1068,7 @@ def install_mockery(temporary_store: spack.store.Store, mutable_config, mock_pac temporary_store.failure_tracker.clear_all() -@pytest.fixture(scope="module") +@pytest.fixture(scope="function") def temporary_mirror_dir(tmpdir_factory): dir = tmpdir_factory.mktemp("mirror") yield str(dir) From 7f9719e5983940865964553bb44870d8ec6b1781 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sat, 7 Jun 2025 20:24:23 -0700 Subject: [PATCH 167/506] Revert "keep cache keys separate for parallel tests" This reverts commit c0a46415b27253dcfd963dcd99e9da7f802c640a. --- lib/spack/spack/test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 56b799ffb4d091..7dc5241dd20ef2 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -1068,7 +1068,7 @@ def install_mockery(temporary_store: spack.store.Store, mutable_config, mock_pac temporary_store.failure_tracker.clear_all() -@pytest.fixture(scope="function") +@pytest.fixture(scope="module") def temporary_mirror_dir(tmpdir_factory): dir = tmpdir_factory.mktemp("mirror") yield str(dir) From 520f018ebac0f85e2288cf47fdda303e0ce4bbf9 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sun, 8 Jun 2025 14:54:12 -0700 Subject: [PATCH 168/506] rm --dist=loadscope (was already set to file anyway, and I think this is cross-module parallelism) --- .github/workflows/unit_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 3d6f99797d892c..7b6d45d738f8ed 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -221,7 +221,7 @@ jobs: $(which spack) bootstrap disable spack-install $(which spack) solve zlib common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x) - $(which spack) unit-test --dist=loadscope --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}" + $(which spack) unit-test --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}" - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b with: name: coverage-${{ matrix.os }}-python${{ matrix.python-version }} From 708188423ea6bca795212e2c5e90ec7f304c9f4e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sun, 8 Jun 2025 14:57:48 -0700 Subject: [PATCH 169/506] swap misc cache to old value --- etc/spack/defaults/config.yaml | 2 +- lib/spack/spack/paths.py | 4 +--- lib/spack/spack/test/data/config/config.yaml | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 2b9ba5771a2706..93ffd14b548d3e 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -105,7 +105,7 @@ config: # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` - misc_cache: $spack_state_home/$spack_instance_id/cache + misc_cache: $user_cache_path/cache # Abort downloads after this many seconds if not data is received. diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 4d09271a20aa30..d583c825098d5e 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -275,9 +275,7 @@ def __init__(self, _prefix=None): #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) #: overridden by `config:misc_cache` - self.default_misc_cache_path = os.path.join( - self.spack_state_home, self.spack_instance_id, "cache" - ) + self.default_misc_cache_path = os.path.join(self.user_cache_path, "cache") #: concretization cache for Spack concretizations #: overridden by `config:concretization_cache:url` self.default_conc_cache_path = os.path.join(self.default_misc_cache_path, "concretization") diff --git a/lib/spack/spack/test/data/config/config.yaml b/lib/spack/spack/test/data/config/config.yaml index d4f688bfb6a28e..fc50b4b7c02f0b 100644 --- a/lib/spack/spack/test/data/config/config.yaml +++ b/lib/spack/spack/test/data/config/config.yaml @@ -8,7 +8,7 @@ config: build_stage: - $tempdir/$user/spack-stage source_cache: $user_cache_path/source - misc_cache: $spack_state_home/$spack_instance_id/cache + misc_cache: $user_cache_path/cache verify_ssl: true ssl_certs: $SSL_CERT_FILE checksum: true From bf6427d7a168b8bdb8934bd7ea1241a128c74818 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sun, 8 Jun 2025 16:00:14 -0700 Subject: [PATCH 170/506] updated config wrt moving misc cache --- lib/spack/docs/config_yaml.rst | 2 +- lib/spack/docs/configuration.rst | 21 +++++++-------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index a554d1b17b6fce..7a4b32d25b8af7 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -135,7 +135,7 @@ clean --downloads `. Temporary directory to store long-lived cache files, such as indices of packages available in repositories. Defaults to -``~/.local/state/spack/$spack_instance_id/spack``. Can be purged with +``$user_cache_path/cache``. Can be purged with :ref:`spack clean --misc-cache `. -------------------- diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 615d8a69d0458c..6da9da654f897c 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -93,12 +93,12 @@ are multiple configuration scopes. From lowest to highest precedence: all instances of Spack running with the same Python installation. This scope takes higher precedence than site, system, and default scopes. #. **user**: Stored in the home directory: - ``~/.local/config/spack/``. These settings affect all instances of + ``~/.config/spack/``. These settings affect all instances of Spack and take higher precedence than site, system, plugin, or defaults scopes. #. **this-spack**: Stored in the home directory: - ``~/.local/config/spack/$spack_instance_id/``. These settings + ``~/.config/spack/$spack_instance_id/``. These settings affect the instance of spack that corresponds to the specified ``$spack_instance_id``. This scope is most useful for overriding configurations for Spack instances that are not (or should not be) @@ -688,7 +688,7 @@ account all scopes. For example, to see the fully merged - $tempdir/$user/spack-stage - $spack_cache_home/stage source_cache: $default_download_root - misc_cache: $spack_state_home/$spack_instance_id/cache + misc_cache: $user_cache_path/cache locks: true Likewise, this will show the fully merged ``packages.yaml``: @@ -734,7 +734,7 @@ down the source of the configuration: /home/myuser/spack/etc/spack/defaults/config.yaml:50 - $tempdir/$user/spack-stage /home/myuser/spack/etc/spack/defaults/config.yaml:51 - $spack_cache_home/stage /home/myuser/spack/etc/spack/defaults/config.yaml:57 source_cache: $default_download_root - /home/myuser/spack/etc/spack/defaults/config.yaml:62 misc_cache: $spack_state_home/$spack_instance_id/cache + /home/myuser/spack/etc/spack/defaults/config.yaml:62 misc_cache: $user_cache_path/cache /home/myuser/spack/etc/spack/defaults/config.yaml:86 locks: True You can see above that the ``build_jobs`` and ``debug`` settings are @@ -774,7 +774,7 @@ configuration locations: And one that allows you to move the default cache location: * ``SPACK_USER_CACHE_PATH``: Override the default path to use for user - data (misc_cache, tests, reports, etc.). + data (modules, misc_cache, tests, reports, etc.). With these settings, if you want to isolate Spack in a CI environment, you can do this:: @@ -794,11 +794,7 @@ changed in 1.0, so this section will summarize the changes, and how XDG variables can help users organize their Spack artifacts. ``XDG_DATA_HOME`` is used to store long-lived data. Its default value -is ``$HOME/.local/share``. The following items are always stored by -default using ``$XDG_DATA_HOME``: - -* Modules: ``$XDG_DATA_HOME/spack/`` -* Package indices: ``$XDG_DATA_HOME/$spack_instance_id/spack`` +is ``$HOME/.local/share``. It is the default value for ``$user_cache_path``. The data listed below will be placed in ``$spack`` by default. However, if ``XDG_DATA_HOME`` is set, they will be stored under that path. The @@ -810,10 +806,7 @@ default values for these are as follows: ``XDG_STATE_HOME`` is used to store data that is useful if persistent, but not integral to Spack's functionality. If not defined, its default -value is ``~/.local/state``. ``misc_cache`` is placed by default using -this variable. - -* ``misc_cache``: ``$XDG_STATE_HOME/$spack_instance_id/spack`` +value is ``~/.local/state``. ``XDG_CACHE_HOME`` is used to store temporary data. If not defined, its default value is ``~/.cache``. Build stages are placed by default From 7129b2a38d0ae7e54e63ad350096054ac5c62d8c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Jun 2025 14:05:55 -0700 Subject: [PATCH 171/506] Revert "updated config wrt moving misc cache" This reverts commit bf6427d7a168b8bdb8934bd7ea1241a128c74818. --- lib/spack/docs/config_yaml.rst | 2 +- lib/spack/docs/configuration.rst | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index 7a4b32d25b8af7..a554d1b17b6fce 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -135,7 +135,7 @@ clean --downloads `. Temporary directory to store long-lived cache files, such as indices of packages available in repositories. Defaults to -``$user_cache_path/cache``. Can be purged with +``~/.local/state/spack/$spack_instance_id/spack``. Can be purged with :ref:`spack clean --misc-cache `. -------------------- diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 6da9da654f897c..615d8a69d0458c 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -93,12 +93,12 @@ are multiple configuration scopes. From lowest to highest precedence: all instances of Spack running with the same Python installation. This scope takes higher precedence than site, system, and default scopes. #. **user**: Stored in the home directory: - ``~/.config/spack/``. These settings affect all instances of + ``~/.local/config/spack/``. These settings affect all instances of Spack and take higher precedence than site, system, plugin, or defaults scopes. #. **this-spack**: Stored in the home directory: - ``~/.config/spack/$spack_instance_id/``. These settings + ``~/.local/config/spack/$spack_instance_id/``. These settings affect the instance of spack that corresponds to the specified ``$spack_instance_id``. This scope is most useful for overriding configurations for Spack instances that are not (or should not be) @@ -688,7 +688,7 @@ account all scopes. For example, to see the fully merged - $tempdir/$user/spack-stage - $spack_cache_home/stage source_cache: $default_download_root - misc_cache: $user_cache_path/cache + misc_cache: $spack_state_home/$spack_instance_id/cache locks: true Likewise, this will show the fully merged ``packages.yaml``: @@ -734,7 +734,7 @@ down the source of the configuration: /home/myuser/spack/etc/spack/defaults/config.yaml:50 - $tempdir/$user/spack-stage /home/myuser/spack/etc/spack/defaults/config.yaml:51 - $spack_cache_home/stage /home/myuser/spack/etc/spack/defaults/config.yaml:57 source_cache: $default_download_root - /home/myuser/spack/etc/spack/defaults/config.yaml:62 misc_cache: $user_cache_path/cache + /home/myuser/spack/etc/spack/defaults/config.yaml:62 misc_cache: $spack_state_home/$spack_instance_id/cache /home/myuser/spack/etc/spack/defaults/config.yaml:86 locks: True You can see above that the ``build_jobs`` and ``debug`` settings are @@ -774,7 +774,7 @@ configuration locations: And one that allows you to move the default cache location: * ``SPACK_USER_CACHE_PATH``: Override the default path to use for user - data (modules, misc_cache, tests, reports, etc.). + data (misc_cache, tests, reports, etc.). With these settings, if you want to isolate Spack in a CI environment, you can do this:: @@ -794,7 +794,11 @@ changed in 1.0, so this section will summarize the changes, and how XDG variables can help users organize their Spack artifacts. ``XDG_DATA_HOME`` is used to store long-lived data. Its default value -is ``$HOME/.local/share``. It is the default value for ``$user_cache_path``. +is ``$HOME/.local/share``. The following items are always stored by +default using ``$XDG_DATA_HOME``: + +* Modules: ``$XDG_DATA_HOME/spack/`` +* Package indices: ``$XDG_DATA_HOME/$spack_instance_id/spack`` The data listed below will be placed in ``$spack`` by default. However, if ``XDG_DATA_HOME`` is set, they will be stored under that path. The @@ -806,7 +810,10 @@ default values for these are as follows: ``XDG_STATE_HOME`` is used to store data that is useful if persistent, but not integral to Spack's functionality. If not defined, its default -value is ``~/.local/state``. +value is ``~/.local/state``. ``misc_cache`` is placed by default using +this variable. + +* ``misc_cache``: ``$XDG_STATE_HOME/$spack_instance_id/spack`` ``XDG_CACHE_HOME`` is used to store temporary data. If not defined, its default value is ``~/.cache``. Build stages are placed by default From bc2d2ee89044f66fc84566ab2ea57e346cf08fc9 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Jun 2025 14:06:13 -0700 Subject: [PATCH 172/506] Revert "swap misc cache to old value" This reverts commit 708188423ea6bca795212e2c5e90ec7f304c9f4e. --- etc/spack/defaults/config.yaml | 2 +- lib/spack/spack/paths.py | 4 +++- lib/spack/spack/test/data/config/config.yaml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index 93ffd14b548d3e..2b9ba5771a2706 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -105,7 +105,7 @@ config: # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` - misc_cache: $user_cache_path/cache + misc_cache: $spack_state_home/$spack_instance_id/cache # Abort downloads after this many seconds if not data is received. diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index d583c825098d5e..4d09271a20aa30 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -275,7 +275,9 @@ def __init__(self, _prefix=None): #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) #: overridden by `config:misc_cache` - self.default_misc_cache_path = os.path.join(self.user_cache_path, "cache") + self.default_misc_cache_path = os.path.join( + self.spack_state_home, self.spack_instance_id, "cache" + ) #: concretization cache for Spack concretizations #: overridden by `config:concretization_cache:url` self.default_conc_cache_path = os.path.join(self.default_misc_cache_path, "concretization") diff --git a/lib/spack/spack/test/data/config/config.yaml b/lib/spack/spack/test/data/config/config.yaml index fc50b4b7c02f0b..d4f688bfb6a28e 100644 --- a/lib/spack/spack/test/data/config/config.yaml +++ b/lib/spack/spack/test/data/config/config.yaml @@ -8,7 +8,7 @@ config: build_stage: - $tempdir/$user/spack-stage source_cache: $user_cache_path/source - misc_cache: $user_cache_path/cache + misc_cache: $spack_state_home/$spack_instance_id/cache verify_ssl: true ssl_certs: $SSL_CERT_FILE checksum: true From 4632f8e2961dd4215bb3b96f9b794f51d4a5b60a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 14 Jul 2025 11:08:42 -0700 Subject: [PATCH 173/506] update shell completion --- share/spack/spack-completion.fish | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index a484d3b79396ad..dc96b4742e23cc 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -2493,7 +2493,7 @@ complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'co set -g __fish_spack_optspecs_spack_mirror_ls h/help scope= complete -c spack -n '__fish_spack_using_command mirror ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -d 'configuration scope to read from' # spack module @@ -2881,7 +2881,7 @@ complete -c spack -n '__fish_spack_using_command repo update' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command repo update' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command repo update' -l remote -s r -r -f -a remote complete -c spack -n '__fish_spack_using_command repo update' -l remote -s r -r -d 'name of remote to check for branches, tags, or commits' -complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -f -a '_builtin defaults:base defaults system site user command_line' +complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -f -a '_builtin defaults:base defaults system end-user site user this-spack command_line' complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command repo update' -l branch -s b -r -f -a branch complete -c spack -n '__fish_spack_using_command repo update' -l branch -s b -r -d 'name of a branch to change to' From f0de5f534e9280874dfb5d248e0bd88e3c43dc6e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 14 Jul 2025 12:37:07 -0700 Subject: [PATCH 174/506] update new paths tests for tmp_path Signed-off-by: Peter Scheibel --- lib/spack/spack/test/paths.py | 49 +++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 5a615f4b557211..91070517d6045a 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -8,11 +8,16 @@ import spack.paths as paths -def test_install_location(working_env, tmpdir): +def _ensure_dir(pathlike): + pathlike.mkdir(parents=True, exist_ok=True) + return str(pathlike) + + +def test_install_location(working_env, tmp_path): # With no direction from env vars, a fresh clone of Spack # should default to using the Spack prefix. It was moved from # where it used to be - base_prefix = str(tmpdir.join("base-prefix").ensure(dir=True)) + base_prefix = _ensure_dir(tmp_path / "base-prefix") p1 = paths.SpackPaths(base_prefix) assert p1.default_install_location == str( pathlib.Path(base_prefix) / "opt" / "data" / "installs" @@ -26,48 +31,48 @@ def test_install_location(working_env, tmpdir): assert p1.default_install_location == str(preexisting_install_dir.parent) # XDG_DATA_HOME overrides all the above - xdg_data_home = str(tmpdir.join("xdg_data_home")) + xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") os.environ["XDG_DATA_HOME"] = xdg_data_home p1 = paths.SpackPaths(base_prefix) assert p1.default_install_location == str(pathlib.Path(xdg_data_home) / "spack" / "installs") # Check that SPACK_DATA_HOME overrides all the above - spack_data_home = str(tmpdir.join("spack_data_home")) + spack_data_home = _ensure_dir(tmp_path / "spack_data_home") os.environ["SPACK_DATA_HOME"] = spack_data_home p2 = paths.SpackPaths(base_prefix) assert p2.default_install_location == str(pathlib.Path(spack_data_home) / "installs") -def test_system_config_path_is_overridable(working_env, tmpdir): - redirect_syscfg_path = str(pathlib.Path(tmpdir) / "redirected_syscfg") +def test_system_config_path_is_overridable(working_env, tmp_path): + redirect_syscfg_path = str(pathlib.Path(tmp_path) / "redirected_syscfg") os.environ["SPACK_SYSTEM_CONFIG_PATH"] = redirect_syscfg_path - p1 = paths.SpackPaths(str(tmpdir.join("base-prefix").ensure(dir=True))) + p1 = paths.SpackPaths(_ensure_dir(tmp_path / "base-prefix")) assert p1.system_config_path == redirect_syscfg_path -def test_system_config_path_is_default_when_env_var_is_empty(working_env, tmpdir): +def test_system_config_path_is_default_when_env_var_is_empty(working_env, tmp_path): os.environ["SPACK_SYSTEM_CONFIG_PATH"] = "" - p1 = paths.SpackPaths(str(tmpdir)) + p1 = paths.SpackPaths(str(tmp_path)) assert os.sep + os.path.join("etc", "spack") == p1.system_config_path -def test_user_config_path_is_overridable(working_env, tmpdir): - redirect_usrcfg_path = str(pathlib.Path(tmpdir) / "redirected_usrcfg") +def test_user_config_path_is_overridable(working_env, tmp_path): + redirect_usrcfg_path = str(pathlib.Path(tmp_path) / "redirected_usrcfg") os.environ["SPACK_USER_CONFIG_PATH"] = redirect_usrcfg_path - p1 = paths.SpackPaths(str(tmpdir.join("base-prefix").ensure(dir=True))) + p1 = paths.SpackPaths(_ensure_dir(tmp_path / "base-prefix")) assert p1.user_config_path == redirect_usrcfg_path -def test_user_config_path_is_default_when_env_var_is_empty(working_env, tmpdir): +def test_user_config_path_is_default_when_env_var_is_empty(working_env, tmp_path): os.environ["SPACK_USER_CONFIG_PATH"] = "" - p1 = paths.SpackPaths(str(tmpdir)) + p1 = paths.SpackPaths(str(tmp_path)) assert os.path.expanduser(os.path.join("~", ".config", "spack")) == p1.user_config_path -def test_user_cache_path_is_overridable(working_env, tmpdir): - redirect_usr_cache = str(pathlib.Path(tmpdir) / "redirected_usr_cache") +def test_user_cache_path_is_overridable(working_env, tmp_path): + redirect_usr_cache = str(pathlib.Path(tmp_path) / "redirected_usr_cache") os.environ["SPACK_USER_CACHE_PATH"] = redirect_usr_cache - p1 = paths.SpackPaths(str(tmpdir.join("base-prefix").ensure(dir=True))) + p1 = paths.SpackPaths(_ensure_dir(tmp_path / "base-prefix")) assert p1.user_cache_path == redirect_usr_cache # Check that things that are supposed to be bundled inside of @@ -75,9 +80,9 @@ def test_user_cache_path_is_overridable(working_env, tmpdir): assert p1.package_repos_path == str(pathlib.Path(redirect_usr_cache) / "package_repos") -def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmpdir): - user_cache_path = str(tmpdir.join("user_cache")) - base_prefix = str(tmpdir.join("base-prefix").ensure(dir=True)) +def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmp_path): + user_cache_path = _ensure_dir(tmp_path / "user-cache") + base_prefix = _ensure_dir(tmp_path / "base-prefix") os.environ["SPACK_USER_CACHE_PATH"] = user_cache_path p1 = paths.SpackPaths(base_prefix) @@ -103,7 +108,7 @@ def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmpdir): assert p1.gpg_keys_path == str(old_gpg_keys_dir) -def test_user_cache_path_is_default_when_env_var_is_empty(working_env, tmpdir): +def test_user_cache_path_is_default_when_env_var_is_empty(working_env, tmp_path): os.environ["SPACK_USER_CACHE_PATH"] = "" - p1 = paths.SpackPaths(str(tmpdir)) + p1 = paths.SpackPaths(str(tmp_path)) assert os.path.expanduser(os.path.join("~", ".local", "share", "spack")) == p1.user_cache_path From 7ebf1cc479b5acd85f30d4b2072fde6fe2006edb Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 12 Sep 2025 14:03:54 -0700 Subject: [PATCH 175/506] run the auto-formatter on new config docs --- .../docs/build_systems/inteloneapipackage.rst | 4 +- lib/spack/docs/chain.rst | 4 +- lib/spack/docs/config_yaml.rst | 17 ++--- lib/spack/docs/configuration.rst | 64 ++++++++----------- lib/spack/docs/repositories.rst | 3 +- 5 files changed, 36 insertions(+), 56 deletions(-) diff --git a/lib/spack/docs/build_systems/inteloneapipackage.rst b/lib/spack/docs/build_systems/inteloneapipackage.rst index 64d8ced8b575c1..31b8180522ab00 100644 --- a/lib/spack/docs/build_systems/inteloneapipackage.rst +++ b/lib/spack/docs/build_systems/inteloneapipackage.rst @@ -141,9 +141,7 @@ Before 2024, the directory structure was different: Libraries ^^^^^^^^^ -If you want Spack to use oneMKL that you have installed without Spack in -the default location, then add the following to -``~/.config/spack/packages.yaml``, adjusting the version as appropriate: +If you want Spack to use oneMKL that you have installed without Spack in the default location, then add the following to ``~/.config/spack/packages.yaml``, adjusting the version as appropriate: .. code-block:: yaml diff --git a/lib/spack/docs/chain.rst b/lib/spack/docs/chain.rst index 0acb499290e2ae..6ab8ba12a89b2d 100644 --- a/lib/spack/docs/chain.rst +++ b/lib/spack/docs/chain.rst @@ -20,9 +20,7 @@ To register the other Spack instance, you can add it as an entry to ``upstreams. spack-instance-2: install_tree: /path/to/another/spack/opt/spack -The ``install_tree`` must point to the ``opt/data/installs`` directory inside of the -Spack base directory, or the location of the ``install_tree`` defined -in :ref:`config.yaml `. +The ``install_tree`` must point to the ``opt/data/installs`` directory inside of the Spack base directory, or the location of the ``install_tree`` defined in :ref:`config.yaml `. Once the upstream Spack instance has been added, ``spack find`` will automatically check the upstream instance when querying installed packages, and new package installations for the local Spack installation will use any dependencies that are installed in the upstream instance. diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index 39a1deaa9ab5aa..a1b7cdbe8583b0 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -17,10 +17,8 @@ You can see the default settings by looking at ``etc/spack/defaults/config.yaml` .. literalinclude:: _spack_root/etc/spack/defaults/config.yaml :language: yaml -These settings can be overridden in ``etc/spack/config.yaml``, or -``~/.config/spack/config.yaml``, or -``~/.config/spack/$spack_instance_id/config.yaml``. See -:ref:`configuration-scopes` for details. +These settings can be overridden in ``etc/spack/config.yaml``, or ``~/.config/spack/config.yaml``, or ``~/.config/spack/$spack_instance_id/config.yaml``. +See :ref:`configuration-scopes` for details. ``install_tree:root`` --------------------- @@ -92,10 +90,7 @@ By default, Spack's ``build_stage`` is configured like this: This can be an ordered list of paths that Spack should search when trying to find a temporary directory for the build stage. The list is searched in order, and Spack will use the first directory to which it has write access. -Specifying `$spack_cache_home` first will ensure each user builds in -their home directory, or wherever the user overrides ``XDG_CACHE_HOME`` to -be - see :ref:`xdg_overrides` and :ref:`config-file-variables` for more on -``$tempdir``, XDG variables, and ``$spack``. +Specifying `$spack_cache_home` first will ensure each user builds in their home directory, or wherever the user overrides ``XDG_CACHE_HOME`` to be - see :ref:`xdg_overrides` and :ref:`config-file-variables` for more on ``$tempdir``, XDG variables, and ``$spack``. When Spack builds a package, it creates a temporary directory within the ``build_stage``. After the package is successfully installed, Spack deletes the temporary directory it used to build. @@ -108,9 +103,9 @@ Unsuccessful builds are not deleted, but you can manually purge them with ``spac ``source_cache`` -------------------- -Location to cache downloaded tarballs and repositories. By default, -these are stored in ``$spack/opt/data/downloads``. These are stored -indefinitely by default and can be purged with ``spack clean --downloads``. +Location to cache downloaded tarballs and repositories. +By default, these are stored in ``$spack/opt/data/downloads``. +These are stored indefinitely by default and can be purged with ``spack clean --downloads``. .. _Misc Cache: diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 5aaeb837f8d026..c81ea3c5f2c555 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -131,14 +131,11 @@ If multiple scopes are provided: Scopes and XDG Compliance ^^^^^^^^^^^^^^^^^^^^^^^^^ -Spack respects XDG variables, and the search location for user -configuration files can be affected by them. When ``XDG_CONFIG_HOME`` -is not defined, Spack assumes the XDG default value of ``~/.config``, -and will apply user and this-spack configuration files located in -``~/.config/spack``. Defining ``XDG_CONFIG_HOME`` will change where -Spack searches for configuration files. To override this behavior, -define ``SPACK_USER_CONFIG_PATH`` to be the desired path. For more -information, see the :ref:`xdg_overrides` for more details. +Spack respects XDG variables, and the search location for user configuration files can be affected by them. +When ``XDG_CONFIG_HOME`` is not defined, Spack assumes the XDG default value of ``~/.config``, and will apply user and this-spack configuration files located in ``~/.config/spack``. +Defining ``XDG_CONFIG_HOME`` will change where Spack searches for configuration files. +To override this behavior, define ``SPACK_USER_CONFIG_PATH`` to be the desired path. +For more information, see the :ref:`xdg_overrides` for more details. """"""""""""""""""""""""""""""""""""""""""" @@ -688,54 +685,45 @@ With these settings, if you want to isolate Spack in a CI environment, you can d .. _xdg_overrides: -------------------------------------------- Overriding default paths with XDG variables -------------------------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -While Spack will by default use locations within the ``$spack`` and -the user's home directory, it can now store all state in locations -goverened by XDG variables. Many default storage locations have -changed in 1.0, so this section will summarize the changes, and how -XDG variables can help users organize their Spack artifacts. +While Spack will by default use locations within the ``$spack`` and the user's home directory, it can now store all state in locations goverened by XDG variables. +Many default storage locations have changed in 1.0, so this section will summarize the changes, and how XDG variables can help users organize their Spack artifacts. -``XDG_DATA_HOME`` is used to store long-lived data. Its default value -is ``$HOME/.local/share``. The following items are always stored by -default using ``$XDG_DATA_HOME``: +``XDG_DATA_HOME`` is used to store long-lived data. +Its default value is ``$HOME/.local/share``. +The following items are always stored by default using ``$XDG_DATA_HOME``: * Modules: ``$XDG_DATA_HOME/spack/`` * Package indices: ``$XDG_DATA_HOME/$spack_instance_id/spack`` -The data listed below will be placed in ``$spack`` by default. However, -if ``XDG_DATA_HOME`` is set, they will be stored under that path. The -default values for these are as follows: +The data listed below will be placed in ``$spack`` by default. +However, if ``XDG_DATA_HOME`` is set, they will be stored under that path. +The default values for these are as follows: * Source caches: ``$HOME/.local/share`` or ``$XDG_DATA_HOME/spack/downloads``. * The install tree: ``$spack/opt/data/installs`` or ``$XDG_DATA_HOME/spack/installs``. * Environment management: ``$spack/opt/data/environments`` or ``$XDG_DATA_HOME/spack/environments`` -``XDG_STATE_HOME`` is used to store data that is useful if persistent, -but not integral to Spack's functionality. If not defined, its default -value is ``~/.local/state``. ``misc_cache`` is placed by default using -this variable. +``XDG_STATE_HOME`` is used to store data that is useful if persistent, but not integral to Spack's functionality. +If not defined, its default value is ``~/.local/state``. +``misc_cache`` is placed by default using this variable. * ``misc_cache``: ``$XDG_STATE_HOME/$spack_instance_id/spack`` -``XDG_CACHE_HOME`` is used to store temporary data. If not defined, -its default value is ``~/.cache``. Build stages are placed by default -using this variable. +``XDG_CACHE_HOME`` is used to store temporary data. +If not defined, its default value is ``~/.cache``. +Build stages are placed by default using this variable. * Build stages: ``$XDG_CACHE_HOME/spack`` -The user configuration scope's files will be located with ``XDG_CONFIG_HOME``. Its -default value is ``~/.config``. +The user configuration scope's files will be located with ``XDG_CONFIG_HOME``. +Its default value is ``~/.config``. * User configuration scope: ``$XDG_CONFIG_HOME/spack`` -Spack also includes the variables ``SPACK_DATA_HOME``, -``SPACK_CONFIG_HOME``, and ``SPACK_STATE_HOME`` that map directly to -the XDG variables described above. They work the same way, but have -higher precedence than the XDG variables. If the Spack-specific -environment variables *are not* defined, Spack will uses the XDG -variables with a suffix of "/spack" to define ``$spack_state_home``, -``$spack_data_home``, and ``spack_cache_home``. If they are defined, -they are used directly without any additional suffix. +Spack also includes the variables ``SPACK_DATA_HOME``, ``SPACK_CONFIG_HOME``, and ``SPACK_STATE_HOME`` that map directly to the XDG variables described above. +They work the same way, but have higher precedence than the XDG variables. +If the Spack-specific environment variables *are not* defined, Spack will uses the XDG variables with a suffix of "/spack" to define ``$spack_state_home``, ``$spack_data_home``, and ``spack_cache_home``. +If they are defined, they are used directly without any additional suffix. diff --git a/lib/spack/docs/repositories.rst b/lib/spack/docs/repositories.rst index cb7dae56291803..cc5cf9699d9105 100644 --- a/lib/spack/docs/repositories.rst +++ b/lib/spack/docs/repositories.rst @@ -147,7 +147,8 @@ For example, to use ``~/custom_packages_clone`` for ``my_remote_repo``: destination: ~/custom_packages_clone -Spack uses the ``repos.yaml`` file in ``~/.config/spack`` (and :ref:`elsewhere `) to find repositories. Note that the ``repos.yaml`` configuration file is distinct from the ``repo.yaml`` file in each repository. +Spack uses the ``repos.yaml`` file in ``~/.config/spack`` (and :ref:`elsewhere `) to find repositories. +Note that the ``repos.yaml`` configuration file is distinct from the ``repo.yaml`` file in each repository. For more on the YAML format, and on how configuration file precedence works in Spack, see :ref:`configuration `. If the ``git`` URL is defined in a lower-precedence configuration (like Spack's defaults for ``builtin``), you only need to specify the ``destination`` in your user-level ``repos.yaml``. From 34351c869adb0d83fa9335ab1f6ceedd0c227365 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 13 Oct 2025 14:26:04 -0700 Subject: [PATCH 176/506] people didn't like that new spack, when putting data in the spack prefix, chose a different location vs. the old spack; this makes the new default locations (when XDG_ or SPACK_DATA_HOME are not set) match their old locations --- etc/spack/defaults/config.yaml | 9 +++------ lib/spack/docs/configuration.rst | 10 +++++----- lib/spack/spack/paths.py | 4 +--- lib/spack/spack/test/paths.py | 15 +++------------ 4 files changed, 12 insertions(+), 26 deletions(-) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index b917d4d4d49673..9c64aef23e3bfc 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -21,8 +21,7 @@ config: # This default value tells spack to check (in order of precedence): # $SPACK_DATA_HOME/installs # $XDG_DATA_HOME/spack/installs - # $spack/opt/spack if previously used by an older version of spack - # $spack/opt/data/installs + # $spack/opt/spack root: $default_install_root projections: all: "{architecture.platform}-{architecture.target}/{name}-{version}-{hash}" @@ -88,8 +87,7 @@ config: # This default value tells spack to check (in order of precedence): # $SPACK_DATA_HOME/downloads # $XDG_DATA_HOME/spack/downloads - # $spack/var/spack/cache if previously used by an older version of spack - # $spack/opt/data/downloads + # $spack/var/spack/cache source_cache: $default_download_root @@ -98,8 +96,7 @@ config: # This default value tells spack to check (in order of precedence): # $SPACK_DATA_HOME/environments # $XDG_DATA_HOME/spack/environments - # $spack/var/spack/environments if previously used by an older version of spack - # $spack/opt/data/environments + # $spack/var/spack/environments # environments_root: $default_envs_root diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 6837d26706503d..e4c38c13d2c983 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -38,7 +38,7 @@ Here is an example ``config.yaml`` file: config: install_tree: - root: $spack/opt/data/installs + root: $spack/opt/spack build_stage: - $tempdir/$user/spack-stage - $spack_cache_home/stage @@ -324,7 +324,7 @@ If your configurations look like this: config: install_tree: - root: $spack/opt/data/installs + root: $spack/opt/spack build_stage: - $tempdir/$user/spack-stage - $spack_cache_home/stage @@ -714,9 +714,9 @@ The data listed below will be placed in ``$spack`` by default. However, if ``XDG_DATA_HOME`` is set, they will be stored under that path. The default values for these are as follows: -* Source caches: ``$HOME/.local/share`` or ``$XDG_DATA_HOME/spack/downloads``. -* The install tree: ``$spack/opt/data/installs`` or ``$XDG_DATA_HOME/spack/installs``. -* Environment management: ``$spack/opt/data/environments`` or ``$XDG_DATA_HOME/spack/environments`` +* Source caches: ``$spack/var/spack/cache`` or ``$XDG_DATA_HOME/spack/downloads``. +* The install tree: ``$spack/opt/spack`` or ``$XDG_DATA_HOME/spack/installs``. +* Environment management: ``$spack/var/spack/environtments`` or ``$XDG_DATA_HOME/spack/environments`` ``XDG_STATE_HOME`` is used to store data that is useful if persistent, but not integral to Spack's functionality. If not defined, its default value is ``~/.local/state``. diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index cb222d9690e806..32a65a9e5a367b 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -292,10 +292,8 @@ def large_data_component(self, subdir, old_location): return os.path.expanduser( os.path.join(os.environ[XDG_vars.data_home.value], "spack", subdir) ) - elif dir_is_occupied(old_location): - return old_location else: - return os.path.join(self.prefix, "opt", "data", subdir) + return old_location locations = SpackPaths() diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 91070517d6045a..97912f3bd1d7e1 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -15,20 +15,11 @@ def _ensure_dir(pathlike): def test_install_location(working_env, tmp_path): # With no direction from env vars, a fresh clone of Spack - # should default to using the Spack prefix. It was moved from - # where it used to be + # should default to using $spack/opt/spack (this was the + # default <= 2025) base_prefix = _ensure_dir(tmp_path / "base-prefix") p1 = paths.SpackPaths(base_prefix) - assert p1.default_install_location == str( - pathlib.Path(base_prefix) / "opt" / "data" / "installs" - ) - - # If XDG_DATA_HOME and SPACK_DATA_HOME aren't set, and - # there are installs in the old prefix, use that - preexisting_install_dir = pathlib.Path(base_prefix) / "opt" / "spack" / ".spack-db" - (preexisting_install_dir).mkdir(parents=True) - p1 = paths.SpackPaths(base_prefix) - assert p1.default_install_location == str(preexisting_install_dir.parent) + assert p1.default_install_location == str(pathlib.Path(base_prefix) / "opt" / "spack") # XDG_DATA_HOME overrides all the above xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") From 2ca35e6b9e80551e2197e41666698d98439d1ab2 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 14 Nov 2025 15:48:35 -0800 Subject: [PATCH 177/506] relocate user scope WRT XDG --- etc/spack/include.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/spack/include.yaml b/etc/spack/include.yaml index 4bdc37ccb41895..7dff2f7f55d663 100644 --- a/etc/spack/include.yaml +++ b/etc/spack/include.yaml @@ -2,7 +2,7 @@ include: # user configuration scope - name: "user" path_override_env_var: SPACK_USER_CONFIG_PATH - path: "~/.spack" + path: "~/.config/spack/config.yaml" optional: true prefer_modify: true when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' From 1ccdb3cb646f8f9317d5ba0ff2bf5765d2590caf Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 14 Nov 2025 17:06:09 -0800 Subject: [PATCH 178/506] bring back site-admin (higher priority than spack scope) --- lib/spack/spack/config.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index e2a7d792bf1d6f..eaa3d8e494ca9a 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1333,6 +1333,21 @@ def create_incremental() -> Generator[Configuration, None, None]: # line scopes go above this. configuration_paths = [("spack", os.path.join(spack.paths.etc_path))] + # Site admin scope has two uses: (a) admins can share config with one + # another, but not with end users (b) pip/apt-installed spack can + # change the default install root + site_admin_path = os.path.join(spack.paths.etc_path, "site-admin") + site_admin_accessible = False + try: + if os.path.isdir(site_admin_path): + os.listdir(site_admin_path) + site_admin_accessible = True + except PermissionError: + pass + if site_admin_accessible: + # TODO: list this as an available scope, but not a writable one + configuration_paths.append(("site-admin", site_admin_path)) + # Python packages can register configuration scopes via entry_points configuration_paths.extend(config_paths_from_entry_points()) From 9233ddb815885a6b004496ccf8f14f25b23dae2b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 1 Dec 2025 11:30:05 -0800 Subject: [PATCH 179/506] removed end-user scope --- lib/spack/spack/config.py | 3 --- lib/spack/spack/main.py | 8 -------- 2 files changed, 11 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index eaa3d8e494ca9a..35cbc4ea1a18ff 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1310,9 +1310,6 @@ def config_paths_from_entry_points() -> List[Tuple[str, str]]: return config_paths -end_user_system_scope = True - - def create_incremental() -> Generator[Configuration, None, None]: """Singleton Configuration instance. diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 8f119900aeef0b..b6b6ae647bc1e5 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -467,11 +467,6 @@ def make_argument_parser(**kwargs): metavar="DIR|ENV", help="add directory or environment as read-only config scope", ) - config.add_argument( - "--disable-end-user-config", - action="store_true", - help="Disable system config scope for end users", - ) envs = config # parser.add_argument_group("environments") env_mutex = envs.add_mutually_exclusive_group() env_mutex.add_argument( @@ -639,9 +634,6 @@ def setup_main_options(args): if args.color is not None: color.set_color_when(args.color) - if args.disable_end_user_config: - spack.config.end_user_system_scope = False - def allows_unknown_args(command): """Implements really simple argument injection for unknown arguments. From 2c7956cfff0716d52b8dd3984980e003b3551b98 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 1 Dec 2025 12:19:35 -0800 Subject: [PATCH 180/506] update command ocmpletion Signed-off-by: Peter Scheibel --- share/spack/spack-completion.bash | 2 +- share/spack/spack-completion.fish | 64 ++++++++++++++++--------------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index ba842044f5266b..b8f57ce6ef8af8 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -398,7 +398,7 @@ SPACK_ALIASES="concretise:concretize;containerise:containerize;rm:remove" _spack() { if $list_options then - SPACK_COMPREPLY="--color -v --verbose -k --insecure -b --bootstrap -V --version -h --help -H --all-help -c --config -C --config-scope -e --env -D --env-dir -E --no-env --use-env-repo -d --debug -t --backtrace --pdb --timestamp -m --mock --print-shell-vars --stacktrace -l --enable-locks -L --disable-locks -p --profile --profile-file --sorted-profile --lines" + SPACK_COMPREPLY="--color -v --verbose -k --insecure -b --bootstrap -V --version -h --help -H --all-help -c --config -C --config-scope -e --env -D --env-dir -E --no-env --use-env-repo -d --debug -t --backtrace --pdb --timestamp -m --mock --print-shell-vars --stacktrace --warn-writes-into-spack -l --enable-locks -L --disable-locks -p --profile --profile-file --sorted-profile --lines" else SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse logs maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" fi diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 01ead8579753cf..3bca5351e4b584 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -346,7 +346,7 @@ complete -c spack --erase # Everything below here is auto-generated. # spack -set -g __fish_spack_optspecs_spack color= v/verbose k/insecure b/bootstrap V/version h/help H/all-help c/config= C/config-scope= e/env= D/env-dir= E/no-env use-env-repo d/debug t/backtrace pdb timestamp m/mock print-shell-vars= stacktrace l/enable-locks L/disable-locks p/profile profile-file= sorted-profile= lines= +set -g __fish_spack_optspecs_spack color= v/verbose k/insecure b/bootstrap V/version h/help H/all-help c/config= C/config-scope= e/env= D/env-dir= E/no-env use-env-repo d/debug t/backtrace pdb timestamp m/mock print-shell-vars= stacktrace warn-writes-into-spack l/enable-locks L/disable-locks p/profile profile-file= sorted-profile= lines= complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a add -d 'add a spec to an environment' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a arch -d 'print architecture information about this machine' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a audit -d 'audit configuration files, packages, etc.' @@ -467,6 +467,8 @@ complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -f -a complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -d 'print info needed by setup-env.*sh' complete -c spack -n '__fish_spack_using_command ' -l stacktrace -f -a stacktrace complete -c spack -n '__fish_spack_using_command ' -l stacktrace -d 'add stacktraces to all printed statements' +complete -c spack -n '__fish_spack_using_command ' -l warn-writes-into-spack -f -a warn_writes_into_spack +complete -c spack -n '__fish_spack_using_command ' -l warn-writes-into-spack -d 'Warn when Spack tries to write into its own prefix' complete -c spack -n '__fish_spack_using_command ' -s l -l enable-locks -f -a locks complete -c spack -n '__fish_spack_using_command ' -s l -l enable-locks -d 'use filesystem locking (default)' complete -c spack -n '__fish_spack_using_command ' -s L -l disable-locks -f -a locks @@ -603,7 +605,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_enable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap enable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap disable @@ -611,7 +613,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_disable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap disable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap reset @@ -626,14 +628,14 @@ set -g __fish_spack_optspecs_spack_bootstrap_root h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap root' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap list set -g __fish_spack_optspecs_spack_bootstrap_list h/help scope= complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap add @@ -642,7 +644,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap add' -f -a '(__ complete -c spack -n '__fish_spack_using_command_pos 1 bootstrap add' -f -a '(__fish_spack_environments)' complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -f -a trust complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -d 'enable the source immediately upon addition' @@ -819,7 +821,7 @@ complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirro complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -d 'override any configured mirrors with this mirror URL' complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -f -a output_file complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -d 'file where rebuild info should be written' -complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -d 'configuration scope containing mirrors to check' # spack buildcache download @@ -1097,7 +1099,7 @@ complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolcha complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolchain -d '(DEPRECATED) Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -d '(DEPRECATED) Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1111,7 +1113,7 @@ complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchai complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchain -d '(DEPRECATED) Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -d '(DEPRECATED) Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1123,7 +1125,7 @@ complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help - complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -d 'configuration scope to modify' # spack compiler rm @@ -1133,14 +1135,14 @@ complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -d 'configuration scope to modify' # spack compiler list set -g __fish_spack_optspecs_spack_compiler_list h/help scope= remote complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler list' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler list' -l remote -d 'list also compilers from registered buildcaches' @@ -1149,7 +1151,7 @@ complete -c spack -n '__fish_spack_using_command compiler list' -l remote -d 'li set -g __fish_spack_optspecs_spack_compiler_ls h/help scope= remote complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler ls' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler ls' -l remote -d 'list also compilers from registered buildcaches' @@ -1159,14 +1161,14 @@ set -g __fish_spack_optspecs_spack_compiler_info h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 compiler info' -f -a '(__fish_spack_installed_compilers)' complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -d 'configuration scope to read from' # spack compilers set -g __fish_spack_optspecs_spack_compilers h/help scope= remote complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command compilers' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compilers' -l remote -d 'list also compilers from registered buildcaches' @@ -1229,7 +1231,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a update -d ' complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a revert -d 'revert configuration files to their state before update' complete -c spack -n '__fish_spack_using_command config' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify' # spack config get @@ -1785,7 +1787,7 @@ complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -f complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -d 'packages to exclude from search' complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -f -a path complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -d 'one or more alternative search paths for finding externals' -complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command external find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command external find' -l all -f -a all complete -c spack -n '__fish_spack_using_command external find' -l all -d 'search for all packages that Spack knows about' @@ -2375,7 +2377,7 @@ set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsig complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use ``--type binary --type source`` (default)' @@ -2409,7 +2411,7 @@ set -g __fish_spack_optspecs_spack_mirror_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror remove' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -d 'configuration scope to modify' # spack mirror rm @@ -2417,7 +2419,7 @@ set -g __fish_spack_optspecs_spack_mirror_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror rm' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -d 'configuration scope to modify' # spack mirror set-url @@ -2429,7 +2431,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -f -a p complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -d 'set only the URL used for uploading' complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -f -a fetch complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -d 'set only the URL used for downloading' -complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2471,7 +2473,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a s complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache' complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed complete -c spack -n '__fish_spack_using_command mirror set' -l signed -d 'require signing and signature verification when pushing and installing from this build cache' -complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2496,14 +2498,14 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password-var set -g __fish_spack_optspecs_spack_mirror_list h/help scope= complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'configuration scope to read from' # spack mirror ls set -g __fish_spack_optspecs_spack_mirror_ls h/help scope= complete -c spack -n '__fish_spack_using_command mirror ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -d 'configuration scope to read from' # spack module @@ -2816,7 +2818,7 @@ complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirecto set -g __fish_spack_optspecs_spack_repo_list h/help scope= names namespaces complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command repo list' -l names -f -a names complete -c spack -n '__fish_spack_using_command repo list' -l names -d 'show configuration names only' @@ -2827,7 +2829,7 @@ complete -c spack -n '__fish_spack_using_command repo list' -l namespaces -d 'sh set -g __fish_spack_optspecs_spack_repo_ls h/help scope= names namespaces complete -c spack -n '__fish_spack_using_command repo ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command repo ls' -l names -f -a names complete -c spack -n '__fish_spack_using_command repo ls' -l names -d 'show configuration names only' @@ -2843,7 +2845,7 @@ complete -c spack -n '__fish_spack_using_command repo add' -l name -r -f -a name complete -c spack -n '__fish_spack_using_command repo add' -l name -r -d 'config name for the package repository, defaults to the namespace of the repository' complete -c spack -n '__fish_spack_using_command repo add' -l path -r -f -a path complete -c spack -n '__fish_spack_using_command repo add' -l path -r -d 'relative path to the Spack package repository inside a git repository. Can be repeated to add multiple package repositories in case of a monorepo' -complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -d 'configuration scope to modify' # spack repo set @@ -2855,7 +2857,7 @@ complete -c spack -n '__fish_spack_using_command repo set' -l destination -r -f complete -c spack -n '__fish_spack_using_command repo set' -l destination -r -d 'destination to clone git repository into' complete -c spack -n '__fish_spack_using_command repo set' -l path -r -f -a path complete -c spack -n '__fish_spack_using_command repo set' -l path -r -d 'relative path to the Spack package repository inside a git repository. Can be repeated to add multiple package repositories in case of a monorepo' -complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -d 'configuration scope to modify' # spack repo remove @@ -2863,7 +2865,7 @@ set -g __fish_spack_optspecs_spack_repo_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo remove' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -d 'configuration scope to modify' # spack repo rm @@ -2871,7 +2873,7 @@ set -g __fish_spack_optspecs_spack_repo_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo rm' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify' # spack repo migrate @@ -2891,7 +2893,7 @@ complete -c spack -n '__fish_spack_using_command repo update' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command repo update' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command repo update' -l remote -s r -r -f -a remote complete -c spack -n '__fish_spack_using_command repo update' -l remote -s r -r -d 'name of remote to check for branches, tags, or commits' -complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command repo update' -l branch -s b -r -f -a branch complete -c spack -n '__fish_spack_using_command repo update' -l branch -s b -r -d 'name of a branch to change to' From 1165fab51004371c45ad57743f6700d4a508aeb6 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 1 Dec 2025 14:22:21 -0800 Subject: [PATCH 181/506] typo in default config, and update completion Signed-off-by: Peter Scheibel --- etc/spack/include.yaml | 2 +- share/spack/spack-completion.fish | 60 +++++++++++++++---------------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/etc/spack/include.yaml b/etc/spack/include.yaml index 7dff2f7f55d663..b8ab4ebfeb3b8b 100644 --- a/etc/spack/include.yaml +++ b/etc/spack/include.yaml @@ -2,7 +2,7 @@ include: # user configuration scope - name: "user" path_override_env_var: SPACK_USER_CONFIG_PATH - path: "~/.config/spack/config.yaml" + path: "~/.config/spack/" optional: true prefer_modify: true when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 3bca5351e4b584..029820d8fc597c 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -605,7 +605,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_enable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap enable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap disable @@ -613,7 +613,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_disable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap disable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap reset @@ -628,14 +628,14 @@ set -g __fish_spack_optspecs_spack_bootstrap_root h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap root' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap list set -g __fish_spack_optspecs_spack_bootstrap_list h/help scope= complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap add @@ -644,7 +644,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap add' -f -a '(__ complete -c spack -n '__fish_spack_using_command_pos 1 bootstrap add' -f -a '(__fish_spack_environments)' complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -f -a trust complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -d 'enable the source immediately upon addition' @@ -821,7 +821,7 @@ complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirro complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -d 'override any configured mirrors with this mirror URL' complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -f -a output_file complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -d 'file where rebuild info should be written' -complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -d 'configuration scope containing mirrors to check' # spack buildcache download @@ -1099,7 +1099,7 @@ complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolcha complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolchain -d '(DEPRECATED) Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -d '(DEPRECATED) Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1113,7 +1113,7 @@ complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchai complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchain -d '(DEPRECATED) Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -f -a mixed_toolchain complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -d '(DEPRECATED) Do not allow mixed toolchains (for example: clang, clang++, gfortran)' -complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1125,7 +1125,7 @@ complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help - complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -d 'configuration scope to modify' # spack compiler rm @@ -1135,14 +1135,14 @@ complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -d 'configuration scope to modify' # spack compiler list set -g __fish_spack_optspecs_spack_compiler_list h/help scope= remote complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler list' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler list' -l remote -d 'list also compilers from registered buildcaches' @@ -1151,7 +1151,7 @@ complete -c spack -n '__fish_spack_using_command compiler list' -l remote -d 'li set -g __fish_spack_optspecs_spack_compiler_ls h/help scope= remote complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler ls' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler ls' -l remote -d 'list also compilers from registered buildcaches' @@ -1161,14 +1161,14 @@ set -g __fish_spack_optspecs_spack_compiler_info h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 compiler info' -f -a '(__fish_spack_installed_compilers)' complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -d 'configuration scope to read from' # spack compilers set -g __fish_spack_optspecs_spack_compilers h/help scope= remote complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command compilers' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compilers' -l remote -d 'list also compilers from registered buildcaches' @@ -1231,7 +1231,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a update -d ' complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a revert -d 'revert configuration files to their state before update' complete -c spack -n '__fish_spack_using_command config' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify' # spack config get @@ -1787,7 +1787,7 @@ complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -f complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -d 'packages to exclude from search' complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -f -a path complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -d 'one or more alternative search paths for finding externals' -complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command external find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command external find' -l all -f -a all complete -c spack -n '__fish_spack_using_command external find' -l all -d 'search for all packages that Spack knows about' @@ -2377,7 +2377,7 @@ set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsig complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use ``--type binary --type source`` (default)' @@ -2411,7 +2411,7 @@ set -g __fish_spack_optspecs_spack_mirror_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror remove' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -d 'configuration scope to modify' # spack mirror rm @@ -2419,7 +2419,7 @@ set -g __fish_spack_optspecs_spack_mirror_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 mirror rm' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -d 'configuration scope to modify' # spack mirror set-url @@ -2431,7 +2431,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -f -a p complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -d 'set only the URL used for uploading' complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -f -a fetch complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -d 'set only the URL used for downloading' -complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2473,7 +2473,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a s complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache' complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed complete -c spack -n '__fish_spack_using_command mirror set' -l signed -d 'require signing and signature verification when pushing and installing from this build cache' -complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2498,14 +2498,14 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password-var set -g __fish_spack_optspecs_spack_mirror_list h/help scope= complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'configuration scope to read from' # spack mirror ls set -g __fish_spack_optspecs_spack_mirror_ls h/help scope= complete -c spack -n '__fish_spack_using_command mirror ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -d 'configuration scope to read from' # spack module @@ -2818,7 +2818,7 @@ complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirecto set -g __fish_spack_optspecs_spack_repo_list h/help scope= names namespaces complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command repo list' -l names -f -a names complete -c spack -n '__fish_spack_using_command repo list' -l names -d 'show configuration names only' @@ -2829,7 +2829,7 @@ complete -c spack -n '__fish_spack_using_command repo list' -l namespaces -d 'sh set -g __fish_spack_optspecs_spack_repo_ls h/help scope= names namespaces complete -c spack -n '__fish_spack_using_command repo ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command repo ls' -l names -f -a names complete -c spack -n '__fish_spack_using_command repo ls' -l names -d 'show configuration names only' @@ -2845,7 +2845,7 @@ complete -c spack -n '__fish_spack_using_command repo add' -l name -r -f -a name complete -c spack -n '__fish_spack_using_command repo add' -l name -r -d 'config name for the package repository, defaults to the namespace of the repository' complete -c spack -n '__fish_spack_using_command repo add' -l path -r -f -a path complete -c spack -n '__fish_spack_using_command repo add' -l path -r -d 'relative path to the Spack package repository inside a git repository. Can be repeated to add multiple package repositories in case of a monorepo' -complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -d 'configuration scope to modify' # spack repo set @@ -2857,7 +2857,7 @@ complete -c spack -n '__fish_spack_using_command repo set' -l destination -r -f complete -c spack -n '__fish_spack_using_command repo set' -l destination -r -d 'destination to clone git repository into' complete -c spack -n '__fish_spack_using_command repo set' -l path -r -f -a path complete -c spack -n '__fish_spack_using_command repo set' -l path -r -d 'relative path to the Spack package repository inside a git repository. Can be repeated to add multiple package repositories in case of a monorepo' -complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -d 'configuration scope to modify' # spack repo remove @@ -2865,7 +2865,7 @@ set -g __fish_spack_optspecs_spack_repo_remove h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo remove' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -d 'configuration scope to modify' # spack repo rm @@ -2873,7 +2873,7 @@ set -g __fish_spack_optspecs_spack_repo_rm h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 repo rm' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify' # spack repo migrate @@ -2893,7 +2893,7 @@ complete -c spack -n '__fish_spack_using_command repo update' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command repo update' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command repo update' -l remote -s r -r -f -a remote complete -c spack -n '__fish_spack_using_command repo update' -l remote -s r -r -d 'name of remote to check for branches, tags, or commits' -complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -f -a '_builtin defaults:base defaults site spack command_line' +complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -f -a '_builtin defaults:base defaults site user spack command_line' complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command repo update' -l branch -s b -r -f -a branch complete -c spack -n '__fish_spack_using_command repo update' -l branch -s b -r -d 'name of a branch to change to' From 0400ff58dcdb8f1c249971cd93a753ae3d95d723 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 1 Dec 2025 16:36:55 -0800 Subject: [PATCH 182/506] missed a relocation of bootstrap cache Signed-off-by: Peter Scheibel --- .github/workflows/bootstrap.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index 525a024fa5e70e..6c7435ef0c324e 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -148,7 +148,7 @@ jobs: fi spack solve zlib done - tree ~/.spack/bootstrap/store + tree ~/.local/share/spack/bootstrap/store - name: Bootstrap GnuPG run: | . share/spack/setup-env.sh From 6ee206e22ccbaecce6471fd0844f2a2e8e5b229b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 3 Dec 2025 17:04:47 -0800 Subject: [PATCH 183/506] removing end-user config scope --- lib/spack/spack/paths.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index b2ac64cc3e4194..79ad1e50abbdbf 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -269,10 +269,6 @@ def __init__(self, _prefix=None): os.getenv("SPACK_SYSTEM_CONFIG_PATH") or os.sep + os.path.join("etc", "spack") ) - #: When Spack is provided by an admin to a user, the admin can - #: provide a config that only applies for the end-users - self.end_user_cfg_path = os.path.join(self.system_config_path, "end-user") - #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) #: overridden by `config:misc_cache` self.default_misc_cache_path = os.path.join( @@ -340,7 +336,6 @@ def large_data_component(self, subdir, old_location): modules_base = locations.modules_base user_config_path = locations.user_config_path system_config_path = locations.system_config_path -end_user_cfg_path = locations.end_user_cfg_path default_misc_cache_path = locations.default_misc_cache_path From 9a5781b4ac59e67ab3f0636c1f562b2070a139a7 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 4 Dec 2025 15:56:19 -0800 Subject: [PATCH 184/506] make sure all expected scopes are defined for shell completion --- lib/spack/spack/test/cmd/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index 37ed2358ddb6e8..891a7c08689219 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -258,7 +258,7 @@ def test_update_completion_arg(shell, tmp_path: pathlib.Path, monkeypatch): # Note: this test is never expected to be supported on Windows @pytest.mark.not_on_windows("Shell completion script generator fails on windows") @pytest.mark.parametrize("shell", ["bash", "fish"]) -def test_updated_completion_scripts(shell, tmp_path: pathlib.Path): +def test_updated_completion_scripts(shell, tmp_path: pathlib.Path, mutable_config): """Make sure our shell tab completion scripts remain up-to-date.""" width = 72 From 3faa83716974f6b50e51048d53ffec9c1aa2f70e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 5 Dec 2025 00:15:08 -0800 Subject: [PATCH 185/506] for the command autocompletion unit test, it depends on the user config scope existing) Signed-off-by: Peter Scheibel --- .github/workflows/unit_tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 79153a1b76f764..61b7f8e6c47691 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -82,6 +82,7 @@ jobs: COVERAGE_FILE: coverage/.coverage-${{ matrix.os }}-python${{ matrix.python-version }} UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.11' }} run: | + mkdir ~/.config/spack share/spack/qa/run-unit-tests - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b with: From 2cfca8fcb54baefe11def9c3150d0b7f5a9fd69f Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 5 Dec 2025 10:34:33 -0800 Subject: [PATCH 186/506] this was just a problem for rhel; I updated the wrong test run --- .github/workflows/unit_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 61b7f8e6c47691..22fceb36e87747 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -82,7 +82,6 @@ jobs: COVERAGE_FILE: coverage/.coverage-${{ matrix.os }}-python${{ matrix.python-version }} UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.11' }} run: | - mkdir ~/.config/spack share/spack/qa/run-unit-tests - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b with: @@ -152,6 +151,7 @@ jobs: - name: Run unit tests shell: runuser -u spack-test -- bash {0} run: | + mkdir ~/.config/spack source share/spack/setup-env.sh spack -d bootstrap now --dev spack unit-test -k 'not cvs and not svn and not hg' -x --verbose From 2ef39fe3f05dd3a0edaf330fd839a4b7e3d8db8a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 9 Dec 2025 10:30:01 -0800 Subject: [PATCH 187/506] undo making scope; it should be visible regardless of whether it exists now --- .github/workflows/unit_tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 55d84612933dc6..7a090a91f70a23 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -152,7 +152,6 @@ jobs: - name: Bootstrap Spack development environment and run unit tests shell: runuser -u spack-test -- bash {0} run: | - mkdir ~/.config/spack source share/spack/setup-env.sh spack debug report spack -d bootstrap now --dev From ff90f540aad6cced725606e5a545bf3f1cf78758 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 12 Dec 2025 11:36:12 -0800 Subject: [PATCH 188/506] higher-level paths can use config; lower-level paths (e.g. locations inside the spack prefix) are used by config --- lib/spack/spack/cmd/__init__.py | 4 +- lib/spack/spack/config.py | 12 +- lib/spack/spack/llnl/util/lang.py | 22 +-- lib/spack/spack/main.py | 8 +- lib/spack/spack/paths.py | 191 ++++++-------------------- lib/spack/spack/paths_base.py | 103 ++++++++++++++ lib/spack/spack/subprocess_context.py | 6 +- lib/spack/spack/trace.py | 10 +- lib/spack/spack/util/path.py | 3 +- 9 files changed, 177 insertions(+), 182 deletions(-) create mode 100644 lib/spack/spack/paths_base.py diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 3f621e902b66b6..bbd65466f8366e 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -20,7 +20,7 @@ import spack.extensions import spack.llnl.string import spack.llnl.util.tty as tty -import spack.paths +import spack.paths_base import spack.repo import spack.spec import spack.spec_parser @@ -87,7 +87,7 @@ def all_commands(): global _all_commands if _all_commands is None: _all_commands = [] - command_paths = [spack.paths.command_path] # Built-in commands + command_paths = [spack.paths_base.command_path] # Built-in commands command_paths += spack.extensions.get_command_paths() # Extensions for path in command_paths: for file in os.listdir(path): diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 400165630bd392..a089bdee21a27d 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -41,7 +41,7 @@ from spack.vendor import jsonschema import spack.error -import spack.paths +import spack.paths_base import spack.schema import spack.schema.bootstrap import spack.schema.cdash @@ -103,7 +103,7 @@ } #: Path to the main configuration scope -CONFIGURATION_DEFAULTS_PATH = ("defaults", os.path.join(spack.paths.etc_path, "defaults")) +CONFIGURATION_DEFAULTS_PATH = ("defaults", os.path.join(spack.paths_base.etc_path, "defaults")) #: Hard-coded default values for some key configuration options. #: This ensures that Spack will still work even if config.yaml in @@ -117,7 +117,7 @@ "dirty": False, "build_jobs": min(16, cpus_available()), "build_stage": "$tempdir/spack-stage", - "license_dir": spack.paths.default_license_dir, + "license_dir": spack.paths_base.default_license_dir, }, "concretizer": {"externals": {"completion": "default_variants"}}, } @@ -138,7 +138,7 @@ def _include_cache_location(): """Location to cache included configuration files.""" - return os.path.join(spack.paths.user_cache_path, "includes") + return os.path.join(spack.paths_base.default_xdg_cache_home, "includes") class ConfigScope: @@ -1383,12 +1383,12 @@ def create_incremental() -> Generator[Configuration, None, None]: # Initial topmost scope is spack (the config scope in the spack instance). # It includes the user, site, and system scopes. Environments and command # line scopes go above this. - configuration_paths = [("spack", os.path.join(spack.paths.etc_path))] + configuration_paths = [("spack", os.path.join(spack.paths_base.etc_path))] # Site admin scope has two uses: (a) admins can share config with one # another, but not with end users (b) pip/apt-installed spack can # change the default install root - site_admin_path = os.path.join(spack.paths.etc_path, "site-admin") + site_admin_path = os.path.join(spack.paths_base.etc_path, "site-admin") site_admin_accessible = False try: if os.path.isdir(site_admin_path): diff --git a/lib/spack/spack/llnl/util/lang.py b/lib/spack/spack/llnl/util/lang.py index f12c2d7e47f8ce..b1d854b3611a12 100644 --- a/lib/spack/spack/llnl/util/lang.py +++ b/lib/spack/spack/llnl/util/lang.py @@ -723,16 +723,18 @@ def __init__(self, factory: Callable[[], object]): @property def instance(self): if self._instance is None: - instance = self.factory() - - if isinstance(instance, types.GeneratorType): - # if it's a generator, assign every value - for value in instance: - self._instance = value - else: - # if not, just assign the result like a normal singleton - self._instance = instance - + try: + instance = self.factory() + + if isinstance(instance, types.GeneratorType): + # if it's a generator, assign every value + for value in instance: + self._instance = value + else: + # if not, just assign the result like a normal singleton + self._instance = instance + except AttributeError as e: + raise Exception("AttrbuteError during creation of Singleton instance") from e return self._instance def __getattr__(self, name): diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 16ff0e580360b6..9f056f412dfa2a 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -36,7 +36,7 @@ import spack.llnl.util.tty as tty import spack.llnl.util.tty.colify import spack.llnl.util.tty.color as color -import spack.paths +import spack.paths_base import spack.platforms import spack.solver.asp import spack.spec @@ -602,7 +602,7 @@ def setup_main_options(args): # override lock configuration if passed on command line if args.locks is not None: if args.locks is False: - spack.util.lock.check_lock_safety(spack.paths.prefix) + spack.util.lock.check_lock_safety(spack.paths_base.prefix) spack.config.set("config:locks", args.locks, scope="command_line") if args.mock: @@ -611,7 +611,7 @@ def setup_main_options(args): key = syaml.syaml_str("repos") key.override = True spack.config.CONFIG.scopes["command_line"].sections["repos"] = syaml.syaml_dict( - [(key, [spack.paths.mock_packages_path])] + [(key, [spack.paths_base.mock_packages_path])] ) # If the user asked for it, don't check ssl certs. @@ -1101,7 +1101,7 @@ def finish_parse_and_run(parser, cmd_name, main_args, env_format_error): raise env_format_error # many operations will fail without a working directory. - spack.paths.set_working_dir() + spack.paths_base.set_working_dir() # now we can actually execute the command. if main_args.spack_profile or main_args.sorted_profile or main_args.profile_file: diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 79ad1e50abbdbf..76430c6fb42cd8 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -15,8 +15,9 @@ from enum import Enum from pathlib import PurePath -import spack.llnl.util.filesystem import spack.util.hash as hash +import spack.paths_base as paths_base +import spack.config as config class XDG_vars(Enum): @@ -93,52 +94,11 @@ def dir_is_occupied(x, except_for=None): class SpackPaths: - def __init__(self, _prefix=None): - #: This file lives in $prefix/lib/spack/spack/__file__ - self.prefix = _prefix or str(PurePath(spack.llnl.util.filesystem.ancestor(__file__, 4))) - - #: synonym for prefix - self.spack_root = self.prefix - - #: bin directory in the spack prefix - self.bin_path = os.path.join(self.prefix, "bin") - - #: The spack script itself - self.spack_script = os.path.join(self.bin_path, "spack") - - #: The sbang script in the spack installation - self.sbang_script = os.path.join(self.bin_path, "sbang") - - # spack directory hierarchy - self.lib_path = os.path.join(self.prefix, "lib", "spack") - self.external_path = os.path.join(self.lib_path, "external") - self.module_path = os.path.join(self.lib_path, "spack") - self.vendor_path = os.path.join(self.module_path, "vendor") - self.command_path = os.path.join(self.module_path, "cmd") - self.analyzers_path = os.path.join(self.module_path, "analyzers") - self.platform_path = os.path.join(self.module_path, "platforms") - self.compilers_path = os.path.join(self.module_path, "compilers") - self.operating_system_path = os.path.join(self.module_path, "operating_systems") - self.test_path = os.path.join(self.module_path, "test") - self.hooks_path = os.path.join(self.module_path, "hooks") - self.share_path = os.path.join(self.prefix, "share", "spack") - self.etc_path = os.path.join(self.prefix, "etc", "spack") - self.default_license_dir = os.path.join(self.etc_path, "licenses") - self.var_path = os.path.join(self.prefix, "var", "spack") - - # $spack/var/spack is generally read-only. Older instances may - # write gpg keys or environments into ...var/ - self.repos_path = os.path.join(self.var_path, "repos") - self.test_repos_path = os.path.join(self.var_path, "test_repos") - self.mock_packages_path = os.path.join(self.test_repos_path, "spack_repo", "builtin_mock") - - self.mock_gpg_data_path = os.path.join(self.var_path, "gpg.mock", "data") - self.mock_gpg_keys_path = os.path.join(self.var_path, "gpg.mock", "keys") - + def __init__(self, base): #: Not a location itself, but used for when Spack instances #: share the same cache base directory for caches that should #: not be shared between those instances. - self.spack_instance_id = hash.b32_hash(self.prefix)[:7] + self.spack_instance_id = hash.b32_hash(base.prefix)[:7] # Resolved XDG_x_HOME variables, with additional "spack" subdirectory. # Resolves to default value from XDG spec if unset. @@ -147,50 +107,12 @@ def __init__(self, _prefix=None): self.spack_cache_home = _spack_xdg_or_backup(XDG_mappings.cache_home.value) self.spack_data_home = _spack_xdg_or_backup(XDG_mappings.data_home.value) - # ------ Next section - # Spack can write a lot of data into the next 3 locations, and - # they used to be inside of Spack by default. They can be set - # to any location with `config:` settings, and those have priority. - # They can also be redirected by setting the SPACK_DATA_HOME or - # XDG_DATA_HOME environment variables. If none of those are - # set, then they point to inside of the Spack prefix. - # - # Precedence: - # 1. config: setting (code consults the config before this - # module) - # 2. explicitly defined SPACK_DATA_HOME - # 3. explicitly defined XDG_DATA_HOME - # 4. old default path, if occupied (inside spack prefix) - # 5. inside spack prefix (slightly different compared to old - # install path) - old_install_path = os.path.join(self.prefix, "opt", "spack") - self.default_install_location = self.large_data_component("installs", old_install_path) - - old_envs_path = os.path.join(self.var_path, "environments") - self.default_envs_path = self.large_data_component("environments", old_envs_path) - - old_fetch_cache_path = os.path.join(self.var_path, "cache") + self.default_install_location = self.large_data_component("installs", base.old_install_path) + self.default_envs_path = self.large_data_component("environments", base.old_envs_path) self.default_fetch_cache_path = self.large_data_component( - "downloads", old_fetch_cache_path + "downloads", base.old_fetch_cache_path ) - # ------ Next section - # Spack can write data into the following locations, but it - # isn't expected to be substantial, so Spack can choose to set - # "~" as a default. They are all organized under a single - # directory that users can refer to in config as $user_cache_path - # - # You can override the top-level directory (the user cache path) by - # setting `SPACK_USER_CACHE_PATH`, `SPACK_DATA_HOME`, or - # `XDG_DATA_HOME`; if none of those are set, then the default for - # `XDG_DATA_HOME` is used (~/.local/share). - # - # Precedence: - # 1. Config setting (not available for all of these) - # 2. SPACK_USER_CACHE_PATH - # 3. explicitly defined SPACK_DATA_HOME - # 4. explicitly defined XDG_DATA_HOME - # 5. default for XDG_DATA_HOME self.user_cache_path = str( PurePath( os.path.expanduser( @@ -218,21 +140,13 @@ def __init__(self, _prefix=None): #: overridden by `bootstrap:root` self.default_user_bootstrap_path = os.path.join(self.user_cache_path, "bootstrap") - # ------ Next section - # The next three locations used to be written inside of the - # Spack prefix, and are now organized under $user_cache_path - # by default, *except* for old installs of Spack that have - # data written into the old locations (in which case, when - # they pull this update, they will continue to use those - # locations) - - old_gpg_path = os.path.join(self.prefix, "opt", "spack", "gpg") + old_gpg_path = os.path.join(base.prefix, "opt", "spack", "gpg") if dir_is_occupied(old_gpg_path): self.gpg_path = old_gpg_path else: self.gpg_path = os.path.join(self.user_cache_path, "gpg") - old_gpg_keys_path = os.path.join(self.var_path, "gpg") + old_gpg_keys_path = os.path.join(base.var_path, "gpg") if dir_is_occupied(old_gpg_keys_path): self.gpg_keys_path = old_gpg_keys_path else: @@ -240,25 +154,11 @@ def __init__(self, _prefix=None): self.modules_base = None for module_dir in ["lmod", "modules"]: - if dir_is_occupied(os.path.join(self.share_path, module_dir)): - self.modules_base = self.share_path + if dir_is_occupied(os.path.join(base.share_path, module_dir)): + self.modules_base = base.share_path if not self.modules_base: self.modules_base = self.user_cache_path - # ------ Next section - # Spack can also write data into the following locations, and their - # defaults are not controlled by SPACK/XDG_DATA_HOME or - # SPACK_USER_CACHE_PATH. Like the prior section, the data written - # into these locations isn't expected to take up much space, so in - # some cases defaults to "~" (in those cases in compliance with - # XDG defaults). - - # There are three environment variables you can use to isolate spack from - # the host environment: - # - `SPACK_USER_CONFIG_PATH`: override `~/.spack` location (for config and caches) - # - `SPACK_SYSTEM_CONFIG_PATH`: override `/etc/spack` configuration scope. - # - `SPACK_DISABLE_LOCAL_CONFIG`: disable both of these locations. - #: User configuration location self.user_config_path = os.path.expanduser( os.getenv("SPACK_USER_CONFIG_PATH") or self.spack_config_home @@ -289,33 +189,8 @@ def large_data_component(self, subdir, old_location): return old_location -locations = SpackPaths() - -prefix = locations.prefix -spack_root = locations.spack_root -bin_path = locations.bin_path -spack_script = locations.spack_script -sbang_script = locations.sbang_script -lib_path = locations.lib_path -external_path = locations.external_path -module_path = locations.module_path -vendor_path = locations.vendor_path -command_path = locations.command_path -analyzers_path = locations.analyzers_path -platform_path = locations.platform_path -compilers_path = locations.compilers_path -operating_system_path = locations.operating_system_path -test_path = locations.test_path -hooks_path = locations.hooks_path -share_path = locations.share_path -etc_path = locations.etc_path -default_license_dir = locations.default_license_dir -var_path = locations.var_path -repos_path = locations.repos_path -test_repos_path = locations.test_repos_path -mock_packages_path = locations.mock_packages_path -mock_gpg_data_path = locations.mock_gpg_data_path -mock_gpg_keys_path = locations.mock_gpg_keys_path +locations = SpackPaths(paths_base.locations) + spack_instance_id = locations.spack_instance_id spack_state_home = locations.spack_state_home spack_config_home = locations.spack_config_home @@ -338,16 +213,30 @@ def large_data_component(self, subdir, old_location): system_config_path = locations.system_config_path default_misc_cache_path = locations.default_misc_cache_path - -#: Recorded directory where spack command was originally invoked -spack_working_dir = None - - -def set_working_dir(): - """Change the working directory to getcwd, or spack prefix if no cwd.""" - global spack_working_dir - try: - spack_working_dir = os.getcwd() - except OSError: - os.chdir(prefix) - spack_working_dir = prefix +# Copy from paths_base +prefix = paths_base.prefix +spack_root = paths_base.spack_root +bin_path = paths_base.bin_path +spack_script = paths_base.spack_script +sbang_script = paths_base.sbang_script +lib_path = paths_base.lib_path +external_path = paths_base.external_path +module_path = paths_base.module_path +vendor_path = paths_base.vendor_path +command_path = paths_base.command_path +analyzers_path = paths_base.analyzers_path +platform_path = paths_base.platform_path +compilers_path = paths_base.compilers_path +operating_system_path = paths_base.operating_system_path +test_path = paths_base.test_path +hooks_path = paths_base.hooks_path +share_path = paths_base.share_path +etc_path = paths_base.etc_path +default_license_dir = paths_base.default_license_dir +var_path = paths_base.var_path +repos_path = paths_base.repos_path +test_repos_path = paths_base.test_repos_path +mock_packages_path = paths_base.mock_packages_path +mock_gpg_data_path = paths_base.mock_gpg_data_path +mock_gpg_keys_path = paths_base.mock_gpg_keys_path +default_xdg_cache_home = paths_base.default_xdg_cache_home \ No newline at end of file diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py new file mode 100644 index 00000000000000..26c93b66417366 --- /dev/null +++ b/lib/spack/spack/paths_base.py @@ -0,0 +1,103 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from pathlib import PurePath +import os + +import spack.llnl.util.filesystem + + +class SpackPathsBase: + def __init__(self, _prefix=None): + #: This file lives in $prefix/lib/spack/spack/__file__ + self.prefix = _prefix or str(PurePath(spack.llnl.util.filesystem.ancestor(__file__, 4))) + + #: This file lives in $prefix/lib/spack/spack/__file__ + self.prefix = _prefix or str(PurePath(spack.llnl.util.filesystem.ancestor(__file__, 4))) + + #: synonym for prefix + self.spack_root = self.prefix + + #: bin directory in the spack prefix + self.bin_path = os.path.join(self.prefix, "bin") + + #: The spack script itself + self.spack_script = os.path.join(self.bin_path, "spack") + + #: The sbang script in the spack installation + self.sbang_script = os.path.join(self.bin_path, "sbang") + + # spack directory hierarchy + self.lib_path = os.path.join(self.prefix, "lib", "spack") + self.external_path = os.path.join(self.lib_path, "external") + self.module_path = os.path.join(self.lib_path, "spack") + self.vendor_path = os.path.join(self.module_path, "vendor") + self.command_path = os.path.join(self.module_path, "cmd") + self.analyzers_path = os.path.join(self.module_path, "analyzers") + self.platform_path = os.path.join(self.module_path, "platforms") + self.compilers_path = os.path.join(self.module_path, "compilers") + self.operating_system_path = os.path.join(self.module_path, "operating_systems") + self.test_path = os.path.join(self.module_path, "test") + self.hooks_path = os.path.join(self.module_path, "hooks") + self.share_path = os.path.join(self.prefix, "share", "spack") + self.etc_path = os.path.join(self.prefix, "etc", "spack") + self.default_license_dir = os.path.join(self.etc_path, "licenses") + self.var_path = os.path.join(self.prefix, "var", "spack") + + # $spack/var/spack is generally read-only. Older instances may + # write gpg keys or environments into ...var/ + self.repos_path = os.path.join(self.var_path, "repos") + self.test_repos_path = os.path.join(self.var_path, "test_repos") + self.mock_packages_path = os.path.join(self.test_repos_path, "spack_repo", "builtin_mock") + + self.mock_gpg_data_path = os.path.join(self.var_path, "gpg.mock", "data") + self.mock_gpg_keys_path = os.path.join(self.var_path, "gpg.mock", "keys") + + self.old_install_path = os.path.join(self.prefix, "opt", "spack") + self.old_envs_path = os.path.join(self.var_path, "environments") + self.old_fetch_cache_path = os.path.join(self.var_path, "cache") + + self.default_xdg_cache_home = os.path.join("~", ".cache", "spack") + +locations = SpackPathsBase() +prefix = locations.prefix +spack_root = locations.spack_root +bin_path = locations.bin_path +spack_script = locations.spack_script +sbang_script = locations.sbang_script +lib_path = locations.lib_path +external_path = locations.external_path +module_path = locations.module_path +vendor_path = locations.vendor_path +command_path = locations.command_path +analyzers_path = locations.analyzers_path +platform_path = locations.platform_path +compilers_path = locations.compilers_path +operating_system_path = locations.operating_system_path +test_path = locations.test_path +hooks_path = locations.hooks_path +share_path = locations.share_path +etc_path = locations.etc_path +default_license_dir = locations.default_license_dir +var_path = locations.var_path +repos_path = locations.repos_path +test_repos_path = locations.test_repos_path +mock_packages_path = locations.mock_packages_path +mock_gpg_data_path = locations.mock_gpg_data_path +mock_gpg_keys_path = locations.mock_gpg_keys_path +default_xdg_cache_home = locations.default_xdg_cache_home + + +#: Recorded directory where spack command was originally invoked +spack_working_dir = None + + +def set_working_dir(): + """Change the working directory to getcwd, or spack prefix if no cwd.""" + global spack_working_dir + try: + spack_working_dir = os.getcwd() + except OSError: + os.chdir(prefix) + spack_working_dir = prefix diff --git a/lib/spack/spack/subprocess_context.py b/lib/spack/spack/subprocess_context.py index 2eef70dc21ac53..717145fa96df40 100644 --- a/lib/spack/spack/subprocess_context.py +++ b/lib/spack/spack/subprocess_context.py @@ -19,7 +19,7 @@ from typing import Any import spack.config -import spack.paths +import spack.paths_base import spack.platforms import spack.repo import spack.store @@ -82,10 +82,10 @@ def __init__(self, pkg, *, ctx=None): self.global_state = None self.test_patches = None self.env = active_environment() - self.spack_working_dir = spack.paths.spack_working_dir + self.spack_working_dir = spack.paths_base.spack_working_dir def restore(self): - spack.paths.spack_working_dir = self.spack_working_dir + spack.paths_base.spack_working_dir = self.spack_working_dir # Activating the environment modifies the global configuration, so globals have to # be restored afterward, in case other modifications were applied on top (e.g. from # command line) diff --git a/lib/spack/spack/trace.py b/lib/spack/spack/trace.py index d78b3425916652..980f6c6ac1377f 100644 --- a/lib/spack/spack/trace.py +++ b/lib/spack/spack/trace.py @@ -6,7 +6,7 @@ import pathlib import warnings -import spack.paths as paths +import spack.paths_base as paths_base def _most_recent_internal_call(): @@ -17,7 +17,7 @@ def _most_recent_internal_call(): stack = inspect.stack() this_file = str(pathlib.Path(__file__).resolve()) - spack_prefix = pathlib.Path(paths.prefix).resolve() + spack_prefix = pathlib.Path(paths_base.prefix).resolve() for frame in stack: frame_loc = pathlib.Path(frame.filename).resolve() if str(frame_loc) != this_file and spack_prefix in frame_loc.parents: @@ -58,17 +58,17 @@ def _guard_writes(event, args): return abs_path = os.path.abspath(path) intent_to_modify = bool((set(mode) & set("wax")) or "r+" in mode) - if abs_path.startswith(paths.prefix) and intent_to_modify: + if abs_path.startswith(paths_base.prefix) and intent_to_modify: _attempted_modify_internal(f"Open {path} in mode [{mode}]") elif event in ["shutil.copyfile", "os.rename", "shutil.move"]: _, dst = args[:2] abs_dst = os.path.abspath(dst) - if abs_dst.startswith(paths.prefix): + if abs_dst.startswith(paths_base.prefix): _attempted_modify_internal(f"copy dst {abs_dst}") elif event == "os.mkdir": path = args[0] abs_path = os.path.abspath(path) - if abs_path.startswith(paths.prefix): + if abs_path.startswith(paths_base.prefix): _attempted_modify_internal(f"mkdir {abs_path}") diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index f710eef2fc3043..2c274c641c25ee 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -58,11 +58,12 @@ def replacements(): import spack import spack.environment as ev import spack.paths + import spack.paths_base arch = architecture() return { - "spack": lambda: spack.paths.prefix, + "spack": lambda: spack.paths_base.prefix, "user": lambda: get_user(), "tempdir": lambda: tempfile.gettempdir(), "user_cache_path": lambda: spack.paths.user_cache_path, From 41b6e2fc820048d2559c05bfd17a116e08faa3fa Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 12 Dec 2025 15:36:18 -0800 Subject: [PATCH 189/506] partial work: spack.paths is now using config, but this is causing circular imports --- lib/spack/spack/__init__.py | 6 +- lib/spack/spack/caches.py | 5 +- lib/spack/spack/llnl/util/lang.py | 3 + lib/spack/spack/paths.py | 135 ++++++++++++++++-------------- lib/spack/spack/paths_base.py | 15 ++++ lib/spack/spack/schema/config.py | 9 ++ 6 files changed, 105 insertions(+), 68 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 0c25558b512a8b..6cc1f9e0975e75 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -6,7 +6,7 @@ import re from typing import Optional -import spack.paths +import spack.paths_base import spack.util.git #: PEP440 canonical ... string @@ -43,7 +43,7 @@ def get_spack_commit() -> Optional[str]: Returns: (str or None) the commit sha if available, otherwise None """ - git_path = os.path.join(spack.paths.prefix, ".git") + git_path = os.path.join(spack.paths_base.prefix, ".git") if not os.path.exists(git_path): return None @@ -53,7 +53,7 @@ def get_spack_commit() -> Optional[str]: rev = git( "-C", - spack.paths.prefix, + spack.paths_base.prefix, "rev-parse", "HEAD", output=str, diff --git a/lib/spack/spack/caches.py b/lib/spack/spack/caches.py index a01189cf2f0248..d7aaecf50c4b06 100644 --- a/lib/spack/spack/caches.py +++ b/lib/spack/spack/caches.py @@ -8,7 +8,6 @@ import spack.config import spack.fetch_strategy import spack.llnl.util.lang -import spack.paths import spack.util.file_cache import spack.util.path from spack.llnl.util.filesystem import mkdirp @@ -20,6 +19,8 @@ def misc_cache_location(): Currently the ``MISC_CACHE`` stores indexes for virtual dependency providers and for which packages provide which tags. """ + import spack.paths + path = spack.config.get("config:misc_cache", spack.paths.default_misc_cache_path) return spack.util.path.canonicalize_path(path) @@ -41,6 +42,8 @@ def fetch_cache_location(): This prevents Spack from repeatedly fetch the same files when building the same package different ways or multiple times. """ + import spack.paths + path = spack.config.get("config:source_cache") if not path: path = spack.paths.default_fetch_cache_path diff --git a/lib/spack/spack/llnl/util/lang.py b/lib/spack/spack/llnl/util/lang.py index b1d854b3611a12..76efb1c1e94fc7 100644 --- a/lib/spack/spack/llnl/util/lang.py +++ b/lib/spack/spack/llnl/util/lang.py @@ -734,6 +734,9 @@ def instance(self): # if not, just assign the result like a normal singleton self._instance = instance except AttributeError as e: + import traceback + import pdb; pdb.set_trace() + traceback.print_exc() raise Exception("AttrbuteError during creation of Singleton instance") from e return self._instance diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 76430c6fb42cd8..947c7f5e0c2927 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -100,102 +100,109 @@ def __init__(self, base): #: not be shared between those instances. self.spack_instance_id = hash.b32_hash(base.prefix)[:7] - # Resolved XDG_x_HOME variables, with additional "spack" subdirectory. - # Resolves to default value from XDG spec if unset. - self.spack_state_home = _spack_xdg_or_backup(XDG_mappings.state_home.value) - self.spack_config_home = _spack_xdg_or_backup(XDG_mappings.config_home.value) - self.spack_cache_home = _spack_xdg_or_backup(XDG_mappings.cache_home.value) - self.spack_data_home = _spack_xdg_or_backup(XDG_mappings.data_home.value) - - self.default_install_location = self.large_data_component("installs", base.old_install_path) - self.default_envs_path = self.large_data_component("environments", base.old_envs_path) - self.default_fetch_cache_path = self.large_data_component( - "downloads", base.old_fetch_cache_path - ) + self.state_home = self.resolve_a_home(["SPACK_USER_CACHE_PATH", "SPACK_STATE_HOME"], "XDG_STATE_HOME", "state", ".local/state/spack") + self.cache_home = self.resolve_a_home("SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", ".cache/spack") + self.data_home = self.resolve_a_home("SPACK_DATA_HOME", "XDG_DATA_HOME", "data", ".local/share/spack") + self.user_cache_path = self.state_home - self.user_cache_path = str( - PurePath( - os.path.expanduser( - os.getenv("SPACK_USER_CACHE_PATH") or self.data_home_for_small_data() - ) - ) - ) + self.default_install_location = self.prefer_old_location(base.old_install_path, os.path.join(self.data_home, "installs")) + self.default_envs_path = self.prefer_old_location(base.old_envs_path, os.path.join(self.data_home, "envs")) + self.default_fetch_cache_path = self.prefer_old_location(base.old_fetch_cache_path, os.path.join(self.data_home, "downloads")) #: junit, cdash, etc. reports about builds - self.reports_path = os.path.join(self.user_cache_path, "reports") + self.reports_path = os.path.join(self.state_home, "reports") #: installation test (spack test) output - self.default_test_path = os.path.join(self.user_cache_path, "test") + self.default_test_path = os.path.join(self.state_home, "test") #: spack monitor analysis directories self.default_monitor_path = os.path.join(self.reports_path, "monitor") #: git repositories fetched to compare commits to versions - self.user_repos_cache_path = os.path.join(self.user_cache_path, "git_repos") + self.user_repos_cache_path = os.path.join(self.state_home, "git_repos") #: default location where remote package repositories are cloned - self.package_repos_path = os.path.join(self.user_cache_path, "package_repos") + self.package_repos_path = os.path.join(self.state_home, "package_repos") #: bootstrap store for bootstrapping clingo and other tools #: overridden by `bootstrap:root` - self.default_user_bootstrap_path = os.path.join(self.user_cache_path, "bootstrap") - - old_gpg_path = os.path.join(base.prefix, "opt", "spack", "gpg") - if dir_is_occupied(old_gpg_path): - self.gpg_path = old_gpg_path - else: - self.gpg_path = os.path.join(self.user_cache_path, "gpg") + self.default_user_bootstrap_path = os.path.join(self.state_home, "bootstrap") - old_gpg_keys_path = os.path.join(base.var_path, "gpg") - if dir_is_occupied(old_gpg_keys_path): - self.gpg_keys_path = old_gpg_keys_path - else: - self.gpg_keys_path = os.path.join(self.user_cache_path, "gpg-keys") + self.gpg_path = self.prefer_old_location(base.old_gpg_path, os.path.join(self.data_home, "gpg")) + self.gpg_keys_path = self.prefer_old_location(base.old_gpg_keys_path, os.path.join(self.data_home, "gpg-keys")) self.modules_base = None for module_dir in ["lmod", "modules"]: if dir_is_occupied(os.path.join(base.share_path, module_dir)): self.modules_base = base.share_path if not self.modules_base: - self.modules_base = self.user_cache_path - - #: User configuration location - self.user_config_path = os.path.expanduser( - os.getenv("SPACK_USER_CONFIG_PATH") or self.spack_config_home - ) - - #: System configuration location - self.system_config_path = os.path.expanduser( - os.getenv("SPACK_SYSTEM_CONFIG_PATH") or os.sep + os.path.join("etc", "spack") - ) + self.modules_base = self.data_home #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) #: overridden by `config:misc_cache` self.default_misc_cache_path = os.path.join( - self.spack_state_home, self.spack_instance_id, "cache" + self.state_home, self.spack_instance_id, "cache" ) - def data_home_for_small_data(self): - return self.spack_data_home - - def large_data_component(self, subdir, old_location): - if XDG_overrides.data_home.value in os.environ: - return os.path.join(os.environ[XDG_overrides.data_home.value], subdir) - elif XDG_vars.data_home.value in os.environ: - return os.path.expanduser( - os.path.join(os.environ[XDG_vars.data_home.value], "spack", subdir) - ) - else: + def resolve_a_home(self, env_vars, xdg_var, config_var, home_rel): + disable_env = config.get("config:locations:disable_env", False) + + def spack_env_check(): + if disable_env: + return + if isinstance(env_vars, str): + x = [env_vars] + else: + x = env_vars + for n in x: + if n in os.environ: + return os.environ[n] + + def xdg_env_check(): + if disable_env: + return + return os.environ.get(xdg_var, None) + + def spack_home_env_check(): + if disable_env: + return + if "SPACK_HOME" in os.environ: + return os.path.join(os.environ["SPACK_HOME"], home_rel) + + def cfg_check(): + return config.get(f"config:locations:{config_var}", None) + + def spack_home_cfg_check(): + h = config.get("config:locations:home", None) + if h: + return os.path.join(h, home_rel) + + for check in [spack_env_check, xdg_env_check, spack_home_env_check, cfg_check, spack_home_cfg_check]: + possible_resolution = check() + if possible_resolution: + return possible_resolution + + return os.path.join("~", home_rel) + + def prefer_old_location(self, old_location, new_location): + # TODO: perhaps it should be configurable whether old locations + # are used. Other option is to relocate downloads & gpg keys. + # TODO: if user sets SPACK/XDG_DATA_HOME, should we move installs + # there even if old dir is occupied? (right now that's what is + # happening here) + if dir_is_occupied(old_location): return old_location + else: + return new_location locations = SpackPaths(paths_base.locations) spack_instance_id = locations.spack_instance_id -spack_state_home = locations.spack_state_home -spack_config_home = locations.spack_config_home -spack_cache_home = locations.spack_cache_home -spack_data_home = locations.spack_data_home +spack_state_home = locations.state_home +#spack_config_home = locations.spack_config_home +spack_cache_home = locations.cache_home +spack_data_home = locations.data_home default_install_location = locations.default_install_location default_envs_path = locations.default_envs_path default_fetch_cache_path = locations.default_fetch_cache_path @@ -210,7 +217,6 @@ def large_data_component(self, subdir, old_location): gpg_keys_path = locations.gpg_keys_path modules_base = locations.modules_base user_config_path = locations.user_config_path -system_config_path = locations.system_config_path default_misc_cache_path = locations.default_misc_cache_path # Copy from paths_base @@ -239,4 +245,5 @@ def large_data_component(self, subdir, old_location): mock_packages_path = paths_base.mock_packages_path mock_gpg_data_path = paths_base.mock_gpg_data_path mock_gpg_keys_path = paths_base.mock_gpg_keys_path -default_xdg_cache_home = paths_base.default_xdg_cache_home \ No newline at end of file +default_xdg_cache_home = paths_base.default_xdg_cache_home +system_config_path = paths_base.system_config_path \ No newline at end of file diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index 26c93b66417366..f2c85a93d51afd 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -57,9 +57,22 @@ def __init__(self, _prefix=None): self.old_install_path = os.path.join(self.prefix, "opt", "spack") self.old_envs_path = os.path.join(self.var_path, "environments") self.old_fetch_cache_path = os.path.join(self.var_path, "cache") + self.old_gpg_path = os.path.join(self.prefix, "opt", "spack", "gpg") + self.old_gpg_keys_path = os.path.join(self.var_path, "gpg") self.default_xdg_cache_home = os.path.join("~", ".cache", "spack") + #: User configuration location + self.user_config_path = os.path.expanduser( + os.getenv("SPACK_USER_CONFIG_PATH") or os.path.join("~", ".config", "spack") + ) + + #: System configuration location + self.system_config_path = os.path.expanduser( + os.getenv("SPACK_SYSTEM_CONFIG_PATH") or os.sep + os.path.join("etc", "spack") + ) + + locations = SpackPathsBase() prefix = locations.prefix spack_root = locations.spack_root @@ -87,6 +100,8 @@ def __init__(self, _prefix=None): mock_gpg_data_path = locations.mock_gpg_data_path mock_gpg_keys_path = locations.mock_gpg_keys_path default_xdg_cache_home = locations.default_xdg_cache_home +system_config_path = locations.system_config_path +user_config_path = locations.user_config_path #: Recorded directory where spack command was originally invoked diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py index 2b157863449500..fc183da46e3368 100644 --- a/lib/spack/spack/schema/config.py +++ b/lib/spack/spack/schema/config.py @@ -46,6 +46,15 @@ **spack.schema.projections.properties, }, }, + "locations": { + "type": "object", + "properties": { + "home": {"type": "string"}, + "cache": {"type": "string"}, + "state": {"type": "string"}, + "disable_env": {"type": "boolean"}, + }, + }, "install_hash_length": {"type": "integer", "minimum": 1}, "build_stage": { "oneOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}] From 7466ac6d03434cfb6a107fd6ec8438106e9335d5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 12 Dec 2025 16:48:11 -0800 Subject: [PATCH 190/506] importing paths no longer automatically invokes the config system - that only occurs when anything tries to resolve a path that is config-based --- lib/spack/spack/binary_distribution.py | 4 +- lib/spack/spack/bootstrap/config.py | 4 +- lib/spack/spack/build_environment.py | 4 +- lib/spack/spack/caches.py | 8 +- lib/spack/spack/ci/__init__.py | 10 +- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/cmd/commands.py | 10 +- lib/spack/spack/cmd/common/__init__.py | 12 +- lib/spack/spack/cmd/edit.py | 12 +- lib/spack/spack/cmd/gpg.py | 4 +- lib/spack/spack/cmd/install.py | 4 +- lib/spack/spack/cmd/license.py | 8 +- lib/spack/spack/cmd/location.py | 6 +- lib/spack/spack/cmd/make_installer.py | 4 +- lib/spack/spack/cmd/style.py | 24 +-- lib/spack/spack/cmd/tutorial.py | 6 +- lib/spack/spack/cmd/unit_test.py | 10 +- lib/spack/spack/environment/depfile.py | 4 +- lib/spack/spack/environment/environment.py | 4 +- lib/spack/spack/hooks/sbang.py | 6 +- lib/spack/spack/install_test.py | 4 +- lib/spack/spack/modules/common.py | 4 +- lib/spack/spack/new_installer.py | 4 +- lib/spack/spack/paths.py | 144 +++++++++--------- lib/spack/spack/repo.py | 6 +- lib/spack/spack/reporters/cdash.py | 4 +- lib/spack/spack/solver/core.py | 6 +- lib/spack/spack/spec.py | 4 +- lib/spack/spack/spec_parser.py | 4 +- lib/spack/spack/store.py | 4 +- lib/spack/spack/test/builder.py | 4 +- lib/spack/spack/test/ci.py | 6 +- lib/spack/spack/test/cmd/blame.py | 10 +- lib/spack/spack/test/cmd/ci.py | 8 +- lib/spack/spack/test/cmd/clean.py | 6 +- lib/spack/spack/test/cmd/commands.py | 8 +- lib/spack/spack/test/cmd/diff.py | 4 +- lib/spack/spack/test/cmd/edit.py | 4 +- lib/spack/spack/test/cmd/env.py | 10 +- lib/spack/spack/test/cmd/find.py | 4 +- lib/spack/spack/test/cmd/license.py | 6 +- lib/spack/spack/test/cmd/list.py | 6 +- lib/spack/spack/test/cmd/location.py | 12 +- lib/spack/spack/test/cmd/pkg.py | 4 +- lib/spack/spack/test/cmd/style.py | 32 ++-- lib/spack/spack/test/cmd/test.py | 4 +- .../test/concretization/compiler_runtimes.py | 4 +- lib/spack/spack/test/concretization/core.py | 8 +- .../spack/test/concretization/flag_mixing.py | 4 +- .../spack/test/concretization/requirements.py | 4 +- lib/spack/spack/test/config.py | 16 +- lib/spack/spack/test/conftest.py | 33 ++-- lib/spack/spack/test/database.py | 4 +- lib/spack/spack/test/directory_layout.py | 6 +- lib/spack/spack/test/link_paths.py | 6 +- lib/spack/spack/test/llnl/util/file_list.py | 4 +- lib/spack/spack/test/llnl/util/filesystem.py | 8 +- lib/spack/spack/test/main.py | 4 +- lib/spack/spack/test/patch.py | 4 +- lib/spack/spack/test/repo.py | 26 ++-- lib/spack/spack/test/spec_semantics.py | 4 +- lib/spack/spack/test/spec_yaml.py | 6 +- lib/spack/spack/test/util/package_hash.py | 4 +- lib/spack/spack/test/util/path.py | 4 +- .../spack/test/util/remote_file_cache.py | 4 +- lib/spack/spack/test/web.py | 4 +- lib/spack/spack/util/gpg.py | 4 +- lib/spack/spack/util/path.py | 20 +-- lib/spack/spack/version/git_ref_lookup.py | 4 +- 69 files changed, 321 insertions(+), 322 deletions(-) diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index b92b99e53f90f4..0170f59651934b 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -43,7 +43,7 @@ import spack.oci.image import spack.oci.oci import spack.oci.opener -import spack.paths +from spack.paths import locations as paths import spack.platforms import spack.relocate as relocate import spack.spec @@ -549,7 +549,7 @@ def get_buildinfo_dict(spec): return { "sbang_install_path": spack.hooks.sbang.sbang_install_path(), "buildpath": spack.store.STORE.layout.root, - "spackprefix": spack.paths.prefix, + "spackprefix": paths.prefix, "relative_prefix": os.path.relpath(spec.prefix, spack.store.STORE.layout.root), # "relocate_textfiles": [], # "relocate_binaries": [], diff --git a/lib/spack/spack/bootstrap/config.py b/lib/spack/spack/bootstrap/config.py index ef048e24d23022..3fb59a6a1e1993 100644 --- a/lib/spack/spack/bootstrap/config.py +++ b/lib/spack/spack/bootstrap/config.py @@ -11,7 +11,7 @@ import spack.config import spack.environment import spack.modules -import spack.paths +from spack.paths import locations as paths import spack.platforms import spack.repo import spack.store @@ -43,7 +43,7 @@ def spec_for_current_python() -> str: def root_path() -> str: """Root of all the bootstrap related folders""" return spack.util.path.canonicalize_path( - spack.config.get("bootstrap:root", spack.paths.default_user_bootstrap_path) + spack.config.get("bootstrap:root", paths.default_user_bootstrap_path) ) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index bc1666354b3ef8..afccbd21c01b2d 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -71,7 +71,7 @@ import spack.llnl.util.tty as tty import spack.multimethod import spack.package_base -import spack.paths +from spack.paths import locations as paths import spack.platforms import spack.schema.environment import spack.spec @@ -487,7 +487,7 @@ def set_wrapper_variables(pkg, env): env.set(SPACK_DEBUG, "TRUE") env.set(SPACK_SHORT_SPEC, pkg.spec.short_spec) env.set(SPACK_DEBUG_LOG_ID, pkg.spec.format("{name}-{hash:7}")) - env.set(SPACK_DEBUG_LOG_DIR, spack.paths.spack_working_dir) + env.set(SPACK_DEBUG_LOG_DIR, paths.spack_working_dir) if spack.config.get("config:ccache"): # Enable ccache in the compiler wrapper diff --git a/lib/spack/spack/caches.py b/lib/spack/spack/caches.py index d7aaecf50c4b06..e0ef6c5d9933ac 100644 --- a/lib/spack/spack/caches.py +++ b/lib/spack/spack/caches.py @@ -11,7 +11,7 @@ import spack.util.file_cache import spack.util.path from spack.llnl.util.filesystem import mkdirp - +from spack.paths import locations as paths def misc_cache_location(): """The ``MISC_CACHE`` is Spack's cache for small data. @@ -21,7 +21,7 @@ def misc_cache_location(): """ import spack.paths - path = spack.config.get("config:misc_cache", spack.paths.default_misc_cache_path) + path = spack.config.get("config:misc_cache", paths.default_misc_cache_path) return spack.util.path.canonicalize_path(path) @@ -42,11 +42,9 @@ def fetch_cache_location(): This prevents Spack from repeatedly fetch the same files when building the same package different ways or multiple times. """ - import spack.paths - path = spack.config.get("config:source_cache") if not path: - path = spack.paths.default_fetch_cache_path + path = paths.default_fetch_cache_path path = spack.util.path.canonicalize_path(path) return path diff --git a/lib/spack/spack/ci/__init__.py b/lib/spack/spack/ci/__init__.py index e7a719590c7e71..2c461716176541 100644 --- a/lib/spack/spack/ci/__init__.py +++ b/lib/spack/spack/ci/__init__.py @@ -27,7 +27,7 @@ import spack.llnl.util.tty as tty import spack.main import spack.mirrors.mirror -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.spec import spack.stage @@ -748,11 +748,11 @@ def download_and_extract_artifacts(url: str, work_dir: str) -> str: def get_spack_info(): """If spack is running from a git repo, return the most recent git log entry, otherwise, return a string containing the spack version.""" - git_path = os.path.join(spack.paths.prefix, ".git") + git_path = os.path.join(paths.prefix, ".git") if os.path.exists(git_path): git = spack.util.git.git() if git: - with fs.working_dir(spack.paths.prefix): + with fs.working_dir(paths.prefix): git_log = git("log", "-1", output=str, error=os.devnull, fail_on_error=False) return git_log @@ -787,12 +787,12 @@ def setup_spack_repro_version( tty.info(f"checkout_commit: {checkout_commit}") tty.info(f"merge_commit: {merge_commit}") - dot_git_path = os.path.join(spack.paths.prefix, ".git") + dot_git_path = os.path.join(paths.prefix, ".git") if not os.path.exists(dot_git_path): tty.error("Unable to find the path to your local spack clone") return False - spack_git_path = spack.paths.prefix + spack_git_path = paths.prefix git = spack.util.git.git() if not git: diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 8079e093d19af3..2e6d1f9cef63f5 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -11,7 +11,7 @@ import spack.config import spack.llnl.util.filesystem import spack.llnl.util.tty as tty -import spack.paths as paths +from spack.paths import locations as paths import spack.stage import spack.store import spack.util.path diff --git a/lib/spack/spack/cmd/commands.py b/lib/spack/spack/cmd/commands.py index e88fb359749f31..a51eee8726b184 100644 --- a/lib/spack/spack/cmd/commands.py +++ b/lib/spack/spack/cmd/commands.py @@ -15,7 +15,7 @@ import spack.config import spack.llnl.util.tty as tty import spack.main -import spack.paths +from spack.paths import locations as paths import spack.platforms from spack.llnl.util.argparsewriter import ArgparseRstWriter, ArgparseWriter, Command from spack.llnl.util.tty.colify import colify @@ -36,14 +36,14 @@ "bash": { "aliases": True, "format": "bash", - "header": os.path.join(spack.paths.share_path, "bash", "spack-completion.bash"), - "update": os.path.join(spack.paths.share_path, "spack-completion.bash"), + "header": os.path.join(paths.share_path, "bash", "spack-completion.bash"), + "update": os.path.join(paths.share_path, "spack-completion.bash"), }, "fish": { "aliases": True, "format": "fish", - "header": os.path.join(spack.paths.share_path, "fish", "spack-completion.fish"), - "update": os.path.join(spack.paths.share_path, "spack-completion.fish"), + "header": os.path.join(paths.share_path, "fish", "spack-completion.fish"), + "update": os.path.join(paths.share_path, "spack-completion.fish"), }, } diff --git a/lib/spack/spack/cmd/common/__init__.py b/lib/spack/spack/cmd/common/__init__.py index 4f6bad5c2c88a1..8c29ae83ed2148 100644 --- a/lib/spack/spack/cmd/common/__init__.py +++ b/lib/spack/spack/cmd/common/__init__.py @@ -4,7 +4,7 @@ import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color -import spack.paths +import spack.paths_base def shell_init_instructions(cmd, equivalent): @@ -25,19 +25,19 @@ def shell_init_instructions(cmd, equivalent): "To set up shell support, run the command below for your shell.", "", color.colorize("@*c{For bash/zsh/sh:}"), - " . %s/setup-env.sh" % spack.paths.share_path, + " . %s/setup-env.sh" % spack.paths_base.share_path, "", color.colorize("@*c{For csh/tcsh:}"), - " source %s/setup-env.csh" % spack.paths.share_path, + " source %s/setup-env.csh" % spack.paths_base.share_path, "", color.colorize("@*c{For fish:}"), - " source %s/setup-env.fish" % spack.paths.share_path, + " source %s/setup-env.fish" % spack.paths_base.share_path, "", color.colorize("@*c{For Windows batch:}"), - " %s\\spack_cmd.bat" % spack.paths.bin_path, + " %s\\spack_cmd.bat" % spack.paths_base.bin_path, "", color.colorize("@*c{For PowerShell:}"), - " %s\\setup-env.ps1" % spack.paths.share_path, + " %s\\setup-env.ps1" % spack.paths_base.share_path, "", "Or, if you do not want to use shell support, run " + ("one of these" if shell_specific else "this") diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index 67fe9a15e84283..e26b58292e7e17 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -10,7 +10,7 @@ import spack.cmd import spack.llnl.util.tty as tty -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.util.editor @@ -38,7 +38,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: "--command", dest="path", action="store_const", - const=spack.paths.command_path, + const=paths.command_path, help="edit the command with the supplied name", ) excl_args.add_argument( @@ -46,7 +46,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: "--docs", dest="path", action="store_const", - const=os.path.join(spack.paths.lib_path, "docs"), + const=os.path.join(paths.lib_path, "docs"), help="edit the docs with the supplied name", ) excl_args.add_argument( @@ -54,7 +54,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: "--test", dest="path", action="store_const", - const=spack.paths.test_path, + const=paths.test_path, help="edit the test with the supplied name", ) excl_args.add_argument( @@ -62,7 +62,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: "--module", dest="path", action="store_const", - const=spack.paths.module_path, + const=paths.module_path, help="edit the main spack module with the supplied name", ) @@ -117,7 +117,7 @@ def locate_build_system(name: str, repo: Optional[spack.repo.Repo]) -> str: def locate_file(name: str, path: str) -> str: # convert command names to python module name - if path == spack.paths.command_path: + if path == paths.command_path: name = spack.cmd.python_name(name) file_path = os.path.join(path, name) diff --git a/lib/spack/spack/cmd/gpg.py b/lib/spack/spack/cmd/gpg.py index 66c9d9a41a64f4..4aa8a8bd8b3528 100644 --- a/lib/spack/spack/cmd/gpg.py +++ b/lib/spack/spack/cmd/gpg.py @@ -8,7 +8,7 @@ import spack.binary_distribution import spack.mirrors.mirror -import spack.paths +from spack.paths import locations as paths import spack.stage import spack.util.gpg import spack.util.url @@ -187,7 +187,7 @@ def gpg_init(args): """add the default keys to the keyring""" import_dir = args.import_dir if import_dir is None: - import_dir = spack.paths.gpg_keys_path + import_dir = paths.gpg_keys_path for root, _, filenames in os.walk(import_dir): for filename in filenames: diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 10a097f65d63fd..19e8c641e2e895 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -12,7 +12,7 @@ import spack.config import spack.environment as ev import spack.llnl.util.filesystem as fs -import spack.paths +from spack.paths import locations as paths import spack.spec import spack.store from spack.cmd.common import arguments @@ -229,7 +229,7 @@ def default_log_file(spec): the corresponding directory if not present """ basename = spec.format_path("test-{name}-{version}-{hash}.xml") - dirname = fs.os.path.join(spack.paths.reports_path, "junit") + dirname = fs.os.path.join(paths.reports_path, "junit") fs.mkdirp(dirname) return fs.os.path.join(dirname, basename) diff --git a/lib/spack/spack/cmd/license.py b/lib/spack/spack/cmd/license.py index 09fa5ba7a31722..edcf6f689e0aec 100644 --- a/lib/spack/spack/cmd/license.py +++ b/lib/spack/spack/cmd/license.py @@ -10,7 +10,7 @@ from typing import Dict, Generator import spack.llnl.util.tty as tty -import spack.paths +from spack.paths import locations as paths description = "list and check license headers on files in spack" section = "query" @@ -62,7 +62,7 @@ ] -def _licensed_files(root: str = spack.paths.prefix) -> Generator[str, None, None]: +def _licensed_files(root: str = paths.prefix) -> Generator[str, None, None]: """Generates paths of licensed files.""" licensed_files = re.compile("|".join(f"(?:{pattern})" for pattern in licensed_files_patterns)) dirs = [ @@ -85,7 +85,7 @@ def _licensed_files(root: str = spack.paths.prefix) -> Generator[str, None, None def list_files(args): """list files in spack that should have license headers""" for relpath in sorted(_licensed_files(args.root)): - print(os.path.join(spack.paths.spack_root, relpath)) + print(os.path.join(paths.spack_root, relpath)) # Error codes for license verification. All values are chosen such that @@ -190,7 +190,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "--root", action="store", - default=spack.paths.prefix, + default=paths.prefix, help="scan a different prefix for license issues", ) diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 9d23aa6d9427e9..48f919e3b10a0c 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -9,7 +9,7 @@ import spack.cmd import spack.environment as ev import spack.llnl.util.tty as tty -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.stage from spack.cmd.common import arguments @@ -93,11 +93,11 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def location(parser, args): if args.module_dir: - print(spack.paths.module_path) + print(paths.module_path) return if args.spack_root: - print(spack.paths.prefix) + print(paths.prefix) return # no -e corresponds to False, -e without arg to None, -e name to the string name. diff --git a/lib/spack/spack/cmd/make_installer.py b/lib/spack/spack/cmd/make_installer.py index 4b6a9bf81be989..1a523bd6d102b4 100644 --- a/lib/spack/spack/cmd/make_installer.py +++ b/lib/spack/spack/cmd/make_installer.py @@ -7,7 +7,7 @@ import sys import spack.concretize -import spack.paths +from spack.paths import locations as paths import spack.util.executable from spack.llnl.path import convert_to_posix_path @@ -86,7 +86,7 @@ def make_installer(parser, args): here = os.path.dirname(os.path.abspath(__file__)) source_dir = os.path.join(here, "installer") - posix_root = convert_to_posix_path(spack.paths.spack_root) + posix_root = convert_to_posix_path(paths.spack_root) spack_license = posixpath.join(posix_root, "LICENSE-APACHE") rtf_spack_license = txt_to_rtf(spack_license) spack_license = posixpath.join(source_dir, "LICENSE.rtf") diff --git a/lib/spack/spack/cmd/style.py b/lib/spack/spack/cmd/style.py index 183a551273e5db..4d9a59696ac5f2 100644 --- a/lib/spack/spack/cmd/style.py +++ b/lib/spack/spack/cmd/style.py @@ -12,7 +12,7 @@ import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.util.git import spack.util.spack_yaml @@ -35,7 +35,7 @@ def grouper(iterable, n, fillvalue=None): #: List of paths to exclude from checks -- relative to spack root -exclude_paths = [os.path.relpath(spack.paths.vendor_path, spack.paths.prefix)] +exclude_paths = [os.path.relpath(paths.vendor_path, paths.prefix)] #: Order in which tools should be run. flake8 is last so that it can #: double-check the results of other tools (if, e.g., ``--fix`` was provided) @@ -96,7 +96,7 @@ def changed_files(base="develop", untracked=True, all_files=False, root=None): root (str): use this directory instead of the Spack prefix. """ if root is None: - root = spack.paths.prefix + root = paths.prefix git = spack.util.git.git(required=True) @@ -274,7 +274,7 @@ def run_flake8(flake8_cmd, file_list, args): for chunk in grouper(file_list, 100): output = flake8_cmd( # always run with config from running spack prefix - "--config=%s" % os.path.join(spack.paths.prefix, ".flake8"), + "--config=%s" % os.path.join(paths.prefix, ".flake8"), *chunk, fail_on_error=False, output=str, @@ -292,7 +292,7 @@ def run_mypy(mypy_cmd, file_list, args): # always run with config from running spack prefix common_mypy_args = [ "--config-file", - os.path.join(spack.paths.prefix, "pyproject.toml"), + os.path.join(paths.prefix, "pyproject.toml"), "--show-error-codes", ] mypy_arg_sets = [common_mypy_args + ["--package", "spack", "--package", "llnl"]] @@ -315,7 +315,7 @@ def run_mypy(mypy_cmd, file_list, args): @tool("isort") def run_isort(isort_cmd, file_list, args): # always run with config from running spack prefix - isort_args = ("--settings-path", os.path.join(spack.paths.prefix, "pyproject.toml")) + isort_args = ("--settings-path", os.path.join(paths.prefix, "pyproject.toml")) if not args.fix: isort_args += ("--check", "--diff") @@ -343,7 +343,7 @@ def process_files(file_list, is_args): @tool("black") def run_black(black_cmd, file_list, args): # always run with config from running spack prefix - black_args = ("--config", os.path.join(spack.paths.prefix, "pyproject.toml")) + black_args = ("--config", os.path.join(paths.prefix, "pyproject.toml")) if not args.fix: black_args += ("--check", "--diff") if color.get_color_when(): # only show color when spack would @@ -391,8 +391,8 @@ def _run_import_check( *, fix: bool, root_relative: bool, - root=spack.paths.prefix, - working_dir=spack.paths.prefix, + root=paths.prefix, + working_dir=paths.prefix, out=sys.stdout, ): if sys.version_info < (3, 9): @@ -738,11 +738,11 @@ def style(parser, args): # ensure that the config files we need actually exist in the spack prefix. # assertions b/c users should not ever see these errors -- they're checked in CI. - assert os.path.isfile(os.path.join(spack.paths.prefix, "pyproject.toml")) - assert os.path.isfile(os.path.join(spack.paths.prefix, ".flake8")) + assert os.path.isfile(os.path.join(paths.prefix, "pyproject.toml")) + assert os.path.isfile(os.path.join(paths.prefix, ".flake8")) # validate spack root if the user provided one - args.root = os.path.realpath(args.root) if args.root else spack.paths.prefix + args.root = os.path.realpath(args.root) if args.root else paths.prefix spack_script = os.path.join(args.root, "bin", "spack") if not os.path.exists(spack_script): tty.die("This does not look like a valid spack root.", "No such file: '%s'" % spack_script) diff --git a/lib/spack/spack/cmd/tutorial.py b/lib/spack/spack/cmd/tutorial.py index afa2463ad63285..1f0b528fbae054 100644 --- a/lib/spack/spack/cmd/tutorial.py +++ b/lib/spack/spack/cmd/tutorial.py @@ -10,7 +10,7 @@ import spack.cmd import spack.config import spack.llnl.util.tty as tty -import spack.paths +from spack.paths import locations as paths import spack.util.git import spack.util.gpg from spack.cmd.common import arguments @@ -25,7 +25,7 @@ # tutorial configuration parameters tutorial_branch = "releases/v1.1" tutorial_mirror = "file:///mirror" -tutorial_key = os.path.join(spack.paths.share_path, "keys", "tutorial.pub") +tutorial_key = os.path.join(paths.share_path, "keys", "tutorial.pub") # configs to remove rm_configs = [ @@ -83,6 +83,6 @@ def tutorial(parser, args): # that follows (exacerbated by the various lazy singletons we use) tty.msg(f"Ensuring we're on the {tutorial_branch} branch") git = spack.util.git.git(required=True) - with working_dir(spack.paths.prefix): + with working_dir(paths.prefix): git("checkout", tutorial_branch) # NO CODE BEYOND HERE diff --git a/lib/spack/spack/cmd/unit_test.py b/lib/spack/spack/cmd/unit_test.py index 271da82c4918e6..3bf14f1d54dc3c 100644 --- a/lib/spack/spack/cmd/unit_test.py +++ b/lib/spack/spack/cmd/unit_test.py @@ -19,7 +19,7 @@ import spack.llnl.util.filesystem import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color -import spack.paths +from spack.paths import locations as paths from spack.llnl.util.tty.colify import colify description = "run spack's unit tests (wrapper around pytest)" @@ -121,10 +121,10 @@ def colorize(c, prefix): # which doesn't need to wait for pytest collection and doesn't # require parsing pytest output files = spack.llnl.util.filesystem.find( - root=spack.paths.test_path, files="*.py", recursive=True + root=paths.test_path, files="*.py", recursive=True ) files = [ - os.path.relpath(f, start=spack.paths.spack_root) + os.path.relpath(f, start=paths.spack_root) for f in files if not f.endswith(("conftest.py", "__init__.py")) ] @@ -154,7 +154,7 @@ def colorize(c, prefix): len_indent = len(indent) if os.path.isabs(name): - name = os.path.relpath(name, start=spack.paths.spack_root) + name = os.path.relpath(name, start=paths.spack_root) item = (len_indent, name, nodetype) @@ -233,7 +233,7 @@ def unit_test(parser, args, unknown_args): # The default is to test the core of Spack. If the option `--extension` # has been used, then test that extension. - pytest_root = spack.paths.spack_root + pytest_root = paths.spack_root if args.extension: pytest_root = spack.extensions.load_extension(args.extension) diff --git a/lib/spack/spack/environment/depfile.py b/lib/spack/spack/environment/depfile.py index 46389b0c24ffa0..4f1be2bc1e595b 100644 --- a/lib/spack/spack/environment/depfile.py +++ b/lib/spack/spack/environment/depfile.py @@ -14,7 +14,7 @@ import spack.deptypes as dt import spack.environment.environment as ev -import spack.paths +from spack.paths import locations as paths import spack.spec import spack.traverse as traverse @@ -229,7 +229,7 @@ def to_dict(self): "install_deps_target": self._target("install-deps"), "any_hash_target": self._target("%"), "jobserver_support": self.jobserver_support, - "spack_script": shlex.quote(spack.paths.spack_script), + "spack_script": shlex.quote(paths.spack_script), "adjacency_list": self.make_adjacency_list, "phony_convenience_targets": " ".join(self.phony_convenience_targets), "pkg_ids_variable": self.pkg_identifier_variable, diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index ac31a5c64835aa..f41e0f11100b0c 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -24,7 +24,7 @@ import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as clr -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.schema.env import spack.spec @@ -64,7 +64,7 @@ _active_environment_error: Optional[spack.config.ConfigFormatError] = None #: default path where environments are stored in the spack tree -default_env_path = spack.paths.default_envs_path +default_env_path = paths.default_envs_path #: Name of the input yaml file for an environment diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 0ba85100e0c760..6750ba7d88e8b7 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -14,7 +14,7 @@ import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.package_prefs -import spack.paths +from spack.paths import locations as paths import spack.spec import spack.store from spack.util.socket import _gethostname @@ -186,7 +186,7 @@ def install_sbang(): """ # copy in a new version of sbang if it differs from what's in spack sbang_path = sbang_install_path() - if os.path.exists(sbang_path) and filecmp.cmp(spack.paths.sbang_script, sbang_path): + if os.path.exists(sbang_path) and filecmp.cmp(paths.sbang_script, sbang_path): return # make $install_tree/bin @@ -210,7 +210,7 @@ def install_sbang(): # copy over the fresh copy of `sbang` sbang_tmp_path = os.path.join(sbang_bin_dir, f".sbang.{_gethostname()}.{os.getpid()}.tmp") - shutil.copy(spack.paths.sbang_script, sbang_tmp_path) + shutil.copy(paths.sbang_script, sbang_tmp_path) # set permissions on `sbang` (including group if set in configuration) os.chmod(sbang_tmp_path, config_mode) diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py index c38f218a6b829b..0f3f0c1be2c302 100644 --- a/lib/spack/spack/install_test.py +++ b/lib/spack/spack/install_test.py @@ -20,7 +20,7 @@ import spack.llnl.util.tty as tty import spack.llnl.util.tty.log import spack.package_base -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.report import spack.spec @@ -97,7 +97,7 @@ def get_test_stage_dir() -> str: absolute path to the configured test stage root or, if none, the default test stage path """ return spack.util.path.canonicalize_path( - spack.config.get("config:test_stage", spack.paths.default_test_path) + spack.config.get("config:test_stage", paths.default_test_path) ) diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py index d5da06235ecf9e..afdc14a5cf43ff 100644 --- a/lib/spack/spack/modules/common.py +++ b/lib/spack/spack/modules/common.py @@ -42,7 +42,7 @@ import spack.error import spack.llnl.util.filesystem import spack.llnl.util.tty as tty -import spack.paths +from spack.paths import locations as paths import spack.projections as proj import spack.schema import spack.schema.environment @@ -212,7 +212,7 @@ def root_path(name, module_set_name): # Root folders where the various module files should be written roots = spack.config.get(f"modules:{module_set_name}:roots", {}) - path = roots.get(name, os.path.join(spack.paths.modules_base, name)) + path = roots.get(name, os.path.join(paths.modules_base, name)) return spack.util.path.canonicalize_path(path) diff --git a/lib/spack/spack/new_installer.py b/lib/spack/spack/new_installer.py index b83dfb7cc799bf..f36d3ffd43ffa7 100644 --- a/lib/spack/spack/new_installer.py +++ b/lib/spack/spack/new_installer.py @@ -52,7 +52,7 @@ import spack.hooks import spack.llnl.util.lock import spack.llnl.util.tty -import spack.paths +import spack.paths_base import spack.report import spack.spec import spack.store @@ -379,7 +379,7 @@ def worker_function( os.environ["MAKEFLAGS"] = makeflags spack.store.STORE = store spack.config.CONFIG = config - spack.paths.set_working_dir() + spack.paths_base.set_working_dir() # Use closedfd=false because of the connection objects. Use line buffering. state_stream = os.fdopen(state.fileno(), "w", buffering=1, closefd=False) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 947c7f5e0c2927..33d946126df603 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -95,55 +95,103 @@ def dir_is_occupied(x, except_for=None): class SpackPaths: def __init__(self, base): + self.base = base + #: Not a location itself, but used for when Spack instances #: share the same cache base directory for caches that should #: not be shared between those instances. self.spack_instance_id = hash.b32_hash(base.prefix)[:7] - self.state_home = self.resolve_a_home(["SPACK_USER_CACHE_PATH", "SPACK_STATE_HOME"], "XDG_STATE_HOME", "state", ".local/state/spack") - self.cache_home = self.resolve_a_home("SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", ".cache/spack") - self.data_home = self.resolve_a_home("SPACK_DATA_HOME", "XDG_DATA_HOME", "data", ".local/share/spack") - self.user_cache_path = self.state_home + @property + def state_home(self): + return self.resolve_a_home(["SPACK_USER_CACHE_PATH", "SPACK_STATE_HOME"], "XDG_STATE_HOME", "state", ".local/state/spack") + + @property + def cache_home(self): + return self.resolve_a_home("SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", ".cache/spack") + + @property + def data_home(self): + return self.resolve_a_home("SPACK_DATA_HOME", "XDG_DATA_HOME", "data", ".local/share/spack") + + @property + def user_cache_path(self): + return self.state_home + + @property + def default_install_location(self): + return self.prefer_old_location(self.base.old_install_path, os.path.join(self.data_home, "installs")) - self.default_install_location = self.prefer_old_location(base.old_install_path, os.path.join(self.data_home, "installs")) - self.default_envs_path = self.prefer_old_location(base.old_envs_path, os.path.join(self.data_home, "envs")) - self.default_fetch_cache_path = self.prefer_old_location(base.old_fetch_cache_path, os.path.join(self.data_home, "downloads")) + @property + def default_envs_path(self): + return self.prefer_old_location(self.base.old_envs_path, os.path.join(self.data_home, "envs")) + @property + def default_fetch_cache_path(self): + return self.prefer_old_location(self.base.old_fetch_cache_path, os.path.join(self.data_home, "downloads")) + + @property + def reports_path(self): #: junit, cdash, etc. reports about builds - self.reports_path = os.path.join(self.state_home, "reports") + return os.path.join(self.state_home, "reports") + @property + def default_test_path(self): #: installation test (spack test) output - self.default_test_path = os.path.join(self.state_home, "test") + return os.path.join(self.state_home, "test") + @property + def default_monitor_path(self): #: spack monitor analysis directories - self.default_monitor_path = os.path.join(self.reports_path, "monitor") + return os.path.join(self.reports_path, "monitor") + @property + def user_repos_cache_path(self): #: git repositories fetched to compare commits to versions - self.user_repos_cache_path = os.path.join(self.state_home, "git_repos") + return os.path.join(self.state_home, "git_repos") + @property + def package_repos_path(self): #: default location where remote package repositories are cloned - self.package_repos_path = os.path.join(self.state_home, "package_repos") + return os.path.join(self.state_home, "package_repos") + @property + def default_user_bootstrap_path(self): #: bootstrap store for bootstrapping clingo and other tools #: overridden by `bootstrap:root` - self.default_user_bootstrap_path = os.path.join(self.state_home, "bootstrap") + return os.path.join(self.state_home, "bootstrap") - self.gpg_path = self.prefer_old_location(base.old_gpg_path, os.path.join(self.data_home, "gpg")) - self.gpg_keys_path = self.prefer_old_location(base.old_gpg_keys_path, os.path.join(self.data_home, "gpg-keys")) + @property + def gpg_path(self): + return self.prefer_old_location(self.base.old_gpg_path, os.path.join(self.data_home, "gpg")) - self.modules_base = None - for module_dir in ["lmod", "modules"]: - if dir_is_occupied(os.path.join(base.share_path, module_dir)): - self.modules_base = base.share_path - if not self.modules_base: - self.modules_base = self.data_home + @property + def gpg_keys_path(self): + return self.prefer_old_location(self.base.old_gpg_keys_path, os.path.join(self.data_home, "gpg-keys")) + @property + def modules_base(self): + modules_base = None + for module_dir in ["lmod", "modules"]: + if dir_is_occupied(os.path.join(self.base.share_path, module_dir)): + modules_base = self.base.share_path + if not modules_base: + modules_base = self.data_home + return modules_base + + @property + def default_misc_cache_path(self): #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) #: overridden by `config:misc_cache` - self.default_misc_cache_path = os.path.join( + return os.path.join( self.state_home, self.spack_instance_id, "cache" ) + def __getattr__(self, name): + # Things that aren't sensitive to import cycles can import the + # paths module and access all items from it + return getattr(self.base, name) + def resolve_a_home(self, env_vars, xdg_var, config_var, home_rel): disable_env = config.get("config:locations:disable_env", False) @@ -196,54 +244,4 @@ def prefer_old_location(self, old_location, new_location): return new_location -locations = SpackPaths(paths_base.locations) - -spack_instance_id = locations.spack_instance_id -spack_state_home = locations.state_home -#spack_config_home = locations.spack_config_home -spack_cache_home = locations.cache_home -spack_data_home = locations.data_home -default_install_location = locations.default_install_location -default_envs_path = locations.default_envs_path -default_fetch_cache_path = locations.default_fetch_cache_path -user_cache_path = locations.user_cache_path -reports_path = locations.reports_path -default_test_path = locations.default_test_path -default_monitor_path = locations.default_monitor_path -user_repos_cache_path = locations.user_repos_cache_path -package_repos_path = locations.package_repos_path -default_user_bootstrap_path = locations.default_user_bootstrap_path -gpg_path = locations.gpg_path -gpg_keys_path = locations.gpg_keys_path -modules_base = locations.modules_base -user_config_path = locations.user_config_path -default_misc_cache_path = locations.default_misc_cache_path - -# Copy from paths_base -prefix = paths_base.prefix -spack_root = paths_base.spack_root -bin_path = paths_base.bin_path -spack_script = paths_base.spack_script -sbang_script = paths_base.sbang_script -lib_path = paths_base.lib_path -external_path = paths_base.external_path -module_path = paths_base.module_path -vendor_path = paths_base.vendor_path -command_path = paths_base.command_path -analyzers_path = paths_base.analyzers_path -platform_path = paths_base.platform_path -compilers_path = paths_base.compilers_path -operating_system_path = paths_base.operating_system_path -test_path = paths_base.test_path -hooks_path = paths_base.hooks_path -share_path = paths_base.share_path -etc_path = paths_base.etc_path -default_license_dir = paths_base.default_license_dir -var_path = paths_base.var_path -repos_path = paths_base.repos_path -test_repos_path = paths_base.test_repos_path -mock_packages_path = paths_base.mock_packages_path -mock_gpg_data_path = paths_base.mock_gpg_data_path -mock_gpg_keys_path = paths_base.mock_gpg_keys_path -default_xdg_cache_home = paths_base.default_xdg_cache_home -system_config_path = paths_base.system_config_path \ No newline at end of file +locations = SpackPaths(paths_base.locations) \ No newline at end of file diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py index 0c5ef9d52c28d7..fd0ee5b25b1402 100644 --- a/lib/spack/spack/repo.py +++ b/lib/spack/spack/repo.py @@ -45,7 +45,7 @@ import spack.llnl.util.lang import spack.llnl.util.tty as tty import spack.patch -import spack.paths +from spack.paths import locations as paths import spack.provider_index import spack.tag import spack.util.executable @@ -73,7 +73,7 @@ def package_repository_lock() -> spack.util.lock.Lock: """Lock for process safety when cloning remote package repositories""" return spack.util.lock.Lock( - os.path.join(spack.paths.user_cache_path, "package-repository.lock") + os.path.join(paths.user_cache_path, "package-repository.lock") ) @@ -1983,7 +1983,7 @@ def parse_config_descriptor( if destination is None: # use a default destination dir_name = spack.util.hash.b32_hash(repository)[-7:] - destination = os.path.join(spack.paths.package_repos_path, dir_name) + destination = os.path.join(paths.package_repos_path, dir_name) else: destination = spack.util.path.canonicalize_path(destination) diff --git a/lib/spack/spack/reporters/cdash.py b/lib/spack/spack/reporters/cdash.py index 57f47c05818093..dc979b045d1905 100644 --- a/lib/spack/spack/reporters/cdash.py +++ b/lib/spack/spack/reporters/cdash.py @@ -18,7 +18,7 @@ import spack import spack.llnl.util.tty as tty -import spack.paths +from spack.paths import locations as paths import spack.platforms import spack.spec import spack.tengine @@ -117,7 +117,7 @@ def __init__(self, configuration: CDashConfiguration): self.buildIds: Dict[str, str] = {} self.revision = "" git = spack.util.git.git(required=True) - with working_dir(spack.paths.spack_root): + with working_dir(paths.spack_root): self.revision = git("rev-parse", "HEAD", output=str).strip() self.generator = "spack-{0}".format(spack.get_version()) self.multiple_packages = False diff --git a/lib/spack/spack/solver/core.py b/lib/spack/spack/solver/core.py index 67f2e435a7fed1..430317e1ae2438 100644 --- a/lib/spack/spack/solver/core.py +++ b/lib/spack/spack/solver/core.py @@ -145,9 +145,11 @@ def _ensure_clingo_or_raise(clingo_mod: ModuleType) -> None: # These are imports that may be problematic at top level (circular imports). They are used # only to provide exhaustive details when erroring due to a broken clingo module. import spack.config - import spack.paths as sp + import spack.paths import spack.util.path as sup + paths = spack.paths.locations + try: clingo_mod.Symbol except AttributeError: @@ -165,7 +167,7 @@ def _ensure_clingo_or_raise(clingo_mod: ModuleType) -> None: if ( pathlib.Path( sup.canonicalize_path( - spack.config.CONFIG.get("bootstrap:root", sp.default_user_bootstrap_path) + spack.config.CONFIG.get("bootstrap:root", paths.default_user_bootstrap_path) ) ) in pathlib.Path(clingo_mod.__file__).parents diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 38819691e83ff2..ce5d7461ea7639 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -89,7 +89,7 @@ import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as clr import spack.patch -import spack.paths +from spack.paths import locations as paths import spack.platforms import spack.provider_index import spack.repo @@ -4022,7 +4022,7 @@ def namespace_if_anonymous(self): @property def spack_root(self): """Special field for using ``{spack_root}`` in :meth:`format`.""" - return spack.paths.spack_root + return paths.spack_root @property def spack_install(self): diff --git a/lib/spack/spack/spec_parser.py b/lib/spack/spack/spec_parser.py index bfdabe17f006d6..67d5ec488f0bd0 100644 --- a/lib/spack/spack/spec_parser.py +++ b/lib/spack/spack/spec_parser.py @@ -67,7 +67,7 @@ import spack.config import spack.deptypes import spack.error -import spack.paths +from spack.paths import locations as paths import spack.util.spack_yaml import spack.version from spack.aliases import LEGACY_COMPILER_TO_BUILTIN @@ -253,7 +253,7 @@ def _warn_about_variant_after_compiler(literal_str: str, issues: List[str]): """Issue a warning if variant or other token is preceded by a compiler token. The warning is only issued if it's actionable: either we know the config file it originates from, or we have call site that's not internal to Spack.""" - ignore = [spack.paths.lib_path, spack.paths.bin_path] + ignore = [paths.lib_path, paths.bin_path] mark = spack.util.spack_yaml.get_mark_from_yaml_data(literal_str) issue_str = ", ".join(issues) error = f"{issue_str} in `{literal_str}`" diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index cf023b2645a16f..110b82e034e05a 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -27,7 +27,7 @@ import spack.directory_layout import spack.error import spack.llnl.util.lang -import spack.paths +from spack.paths import locations as paths import spack.spec import spack.util.path from spack.llnl.util import tty @@ -75,7 +75,7 @@ def parse_install_tree(config_dict: dict) -> Tuple[str, str, Dict[str, str]]: projections = {"all": all_projection} else: - unpadded_root = install_tree.get("root", spack.paths.default_install_location) + unpadded_root = install_tree.get("root", paths.default_install_location) unpadded_root = spack.util.path.canonicalize_path(unpadded_root) padded_length = install_tree.get("padded_length", False) diff --git a/lib/spack/spack/test/builder.py b/lib/spack/spack/test/builder.py index b38eb85505da5c..5d161c61d72c75 100644 --- a/lib/spack/spack/test/builder.py +++ b/lib/spack/spack/test/builder.py @@ -8,14 +8,14 @@ import spack.builder import spack.concretize -import spack.paths +from spack.paths import locations as paths import spack.repo from spack.llnl.util.filesystem import touch @pytest.fixture() def builder_test_repository(config): - builder_test_path = os.path.join(spack.paths.test_repos_path, "spack_repo", "builder_test") + builder_test_path = os.path.join(paths.test_repos_path, "spack_repo", "builder_test") with spack.repo.use_repositories(builder_test_path) as mock_repo: yield mock_repo diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index 4aec672c37b13d..0193122e5cb45c 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -14,7 +14,7 @@ import spack.environment as ev import spack.error import spack.llnl.util.filesystem as fs -import spack.paths as spack_paths +from spack.paths import locations as paths import spack.repo as repo import spack.util.git from spack.spec import Spec @@ -190,7 +190,7 @@ def test_pipeline_dag(config, repo_builder: RepoBuilder): @pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_import_signing_key(mock_gnupghome): - signing_key_dir = spack_paths.mock_gpg_keys_path + signing_key_dir = paths.mock_gpg_keys_path signing_key_path = os.path.join(signing_key_dir, "package-signing-key") with open(signing_key_path, encoding="utf-8") as fd: signing_key = fd.read() @@ -205,7 +205,7 @@ def test_download_and_extract_artifacts(tmp_path: pathlib.Path, monkeypatch): url = "https://www.nosuchurlexists.itsfake/artifacts.zip" working_dir = tmp_path / "repro" test_artifacts_path = os.path.join( - spack_paths.test_path, "data", "ci", "gitlab", "artifacts.zip" + paths.test_path, "data", "ci", "gitlab", "artifacts.zip" ) def _urlopen_OK(*args, **kwargs): diff --git a/lib/spack/spack/test/cmd/blame.py b/lib/spack/spack/test/cmd/blame.py index f9b8d6345a2d5e..4a56874d5a4b8d 100644 --- a/lib/spack/spack/test/cmd/blame.py +++ b/lib/spack/spack/test/cmd/blame.py @@ -9,7 +9,7 @@ import pytest import spack.cmd.blame -import spack.paths +from spack.paths import locations as paths import spack.util.spack_json as sjson from spack.cmd.blame import ensure_full_history, git_prefix, package_repo_root from spack.llnl.util.filesystem import mkdirp, working_dir @@ -40,7 +40,7 @@ def test_blame_by_percent(mock_packages): def test_blame_file(): """Sanity check the blame command to make sure it works.""" - with working_dir(spack.paths.prefix): + with working_dir(paths.prefix): out = blame(os.path.join("bin", "spack")) assert "LAST_COMMIT" in out assert "AUTHOR" in out @@ -72,8 +72,8 @@ def test_blame_file_outside_spack_repo(tmp_path: Path): def test_blame_spack_not_git_clone(monkeypatch): """Ensure attempt to get blame when spack not a git clone fails.""" - non_git_dir = os.path.join(spack.paths.prefix, "..") - monkeypatch.setattr(spack.paths, "prefix", non_git_dir) + non_git_dir = os.path.join(paths.prefix, "..") + monkeypatch.setattr(paths, "prefix", non_git_dir) with pytest.raises(SpackCommandError): out = blame(".") @@ -82,7 +82,7 @@ def test_blame_spack_not_git_clone(monkeypatch): def test_blame_json(mock_packages): """Ensure that we can output json as a blame.""" - with working_dir(spack.paths.prefix): + with working_dir(paths.prefix): out = blame("--json", "mpich") # Test loading the json, and top level keys diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index fe31a884e93e40..5dabbde8a56818 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -19,7 +19,7 @@ import spack.environment as ev import spack.hash_types as ht import spack.main -import spack.paths as spack_paths +from spack.paths import locations as paths import spack.repo import spack.spec import spack.stage @@ -474,7 +474,7 @@ def test_ci_rebuild_missing_config(tmp_path: pathlib.Path, working_env, mutable_ def _signing_key(): - signing_key_path = pathlib.Path(spack_paths.mock_gpg_keys_path) / "package-signing-key" + signing_key_path = pathlib.Path(paths.mock_gpg_keys_path) / "package-signing-key" return signing_key_path.read_text() @@ -1093,9 +1093,9 @@ def test_ci_rebuild_index( def test_ci_get_stack_changed(mock_git_repo, monkeypatch): """Test that we can detect the change to .gitlab-ci.yml in a mock spack git repo.""" - monkeypatch.setattr(spack.paths, "prefix", mock_git_repo) + monkeypatch.setattr(paths, "prefix", mock_git_repo) fake_env_path = os.path.join( - spack.paths.prefix, os.path.sep.join(("no", "such", "env", "path")) + paths.prefix, os.path.sep.join(("no", "such", "env", "path")) ) assert ci.stack_changed(fake_env_path) is True diff --git a/lib/spack/spack/test/cmd/clean.py b/lib/spack/spack/test/cmd/clean.py index 145187917afe3b..4606f6a27298be 100644 --- a/lib/spack/spack/test/cmd/clean.py +++ b/lib/spack/spack/test/cmd/clean.py @@ -12,7 +12,7 @@ import spack.llnl.util.filesystem as fs import spack.main import spack.package_base -import spack.paths +from spack.paths import locations as paths import spack.stage import spack.store @@ -101,8 +101,8 @@ def _check_files(directory): # spack.cmd.clean references paths from spack.paths: we want to # update them for the duration of this test. - monkeypatch.setattr(spack.paths, "lib_path", source_dir) - monkeypatch.setattr(spack.paths, "repos_path", repos_dir) + monkeypatch.setattr(paths, "lib_path", source_dir) + monkeypatch.setattr(paths, "repos_path", repos_dir) spack.cmd.clean.remove_python_cache() diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index de46a972cbe3c4..dfbe546bcab725 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -15,7 +15,7 @@ import spack.cmd.commands import spack.config import spack.main -import spack.paths +from spack.paths import locations as paths from spack.cmd.commands import _dest_to_fish_complete, _positional_to_subroutine from spack.util.executable import Executable @@ -24,7 +24,7 @@ def commands(*args: str) -> str: """Run `spack commands args...` and return output as a string. It's a separate process so that we run through the main Spack command logic and avoid caching issues.""" python = Executable(sys.executable) - return python(spack.paths.spack_script, "commands", *args, output=str) + return python(paths.spack_script, "commands", *args, output=str) def test_names(): @@ -290,9 +290,9 @@ def test_updated_completion_scripts(shell, tmp_path: pathlib.Path, mutable_confi ) msg = "\n".join(lines) - header = os.path.join(spack.paths.share_path, shell, f"spack-completion.{shell}") + header = os.path.join(paths.share_path, shell, f"spack-completion.{shell}") script = f"spack-completion.{shell}" - old_script = os.path.join(spack.paths.share_path, script) + old_script = os.path.join(paths.share_path, script) new_script = str(tmp_path / script) commands("--aliases", "--format", shell, "--header", header, "--update", new_script) diff --git a/lib/spack/spack/test/cmd/diff.py b/lib/spack/spack/test/cmd/diff.py index 1e703c4849f109..98e3a8dc840476 100644 --- a/lib/spack/spack/test/cmd/diff.py +++ b/lib/spack/spack/test/cmd/diff.py @@ -9,7 +9,7 @@ import spack.cmd.diff import spack.concretize import spack.main -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.util.spack_json as sjson import spack.version @@ -33,7 +33,7 @@ @pytest.fixture def test_repo(config): - builder_test_path = os.path.join(spack.paths.test_repos_path, "spack_repo", "diff") + builder_test_path = os.path.join(paths.test_repos_path, "spack_repo", "diff") with spack.repo.use_repositories(builder_test_path) as mock_repo: yield mock_repo diff --git a/lib/spack/spack/test/cmd/edit.py b/lib/spack/spack/test/cmd/edit.py index 47f52fc6d66f87..02f68bdf9a33e7 100644 --- a/lib/spack/spack/test/cmd/edit.py +++ b/lib/spack/spack/test/cmd/edit.py @@ -5,7 +5,7 @@ import os import pathlib -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.util.editor from spack.main import SpackCommand @@ -61,7 +61,7 @@ def editor(*args: str, **kwargs): monkeypatch.setattr(spack.util.editor, "editor", editor) # set up an additional repo - extra_repo_dir = pathlib.Path(spack.paths.test_repos_path) / "spack_repo" / "requirements_test" + extra_repo_dir = pathlib.Path(paths.test_repos_path) / "spack_repo" / "requirements_test" with spack.repo.use_repositories(str(extra_repo_dir), override=False): edit("--build-system", "builtin_mock.autotools", "builtin_mock.cmake") assert called diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index c3cf1f4f5c55b5..d5928858ce7ddc 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -28,7 +28,7 @@ import spack.modules import spack.modules.tcl import spack.package_base -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.solver.asp import spack.spec @@ -2060,7 +2060,7 @@ def test_env_include_concrete_env_yaml(env_name): @pytest.mark.regression("45766") @pytest.mark.parametrize("format", ["v1", "v2", "v3"]) def test_env_include_concrete_old_env(format): - lockfile = os.path.join(spack.paths.test_path, "data", "legacy_env", f"{format}.lock") + lockfile = os.path.join(paths.test_path, "data", "legacy_env", f"{format}.lock") # create an env from old .lock file -- this does not update the format env("create", "old-env", lockfile) env("create", "--include-concrete", "old-env", "test") @@ -3941,7 +3941,7 @@ def test_read_old_lock_and_write_new(tmp_path: pathlib.Path, lockfile): # the environment, anyway. # # This test ensures the behavior described above. - lockfile_path = os.path.join(spack.paths.test_path, "data", "legacy_env", "%s.lock" % lockfile) + lockfile_path = os.path.join(paths.test_path, "data", "legacy_env", "%s.lock" % lockfile) # read in the JSON from a legacy lockfile with open(lockfile_path, encoding="utf-8") as f: @@ -3992,7 +3992,7 @@ def test_read_v1_lock_creates_backup(tmp_path: pathlib.Path): """When reading a version-1 lockfile, make sure that a backup of that file is created. """ - v1_lockfile_path = pathlib.Path(spack.paths.test_path) / "data" / "legacy_env" / "v1.lock" + v1_lockfile_path = pathlib.Path(paths.test_path) / "data" / "legacy_env" / "v1.lock" test_lockfile_path = tmp_path / "init" / ev.lockfile_name test_lockfile_path.parent.mkdir(parents=True, exist_ok=False) shutil.copy(v1_lockfile_path, test_lockfile_path) @@ -4015,7 +4015,7 @@ def test_read_legacy_lockfile_and_reconcretize( # After reconcretization with the *new*, finer-grained DAG hash, there should no # longer be conflicts, and the previously conflicting specs can coexist in the # same environment. - test_path = pathlib.Path(spack.paths.test_path) + test_path = pathlib.Path(paths.test_path) lockfile_content = test_path / "data" / "legacy_env" / f"{lockfile}.lock" legacy_lockfile_path = tmp_path / ev.lockfile_name shutil.copy(lockfile_content, legacy_lockfile_path) diff --git a/lib/spack/spack/test/cmd/find.py b/lib/spack/spack/test/cmd/find.py index b16106858c4a84..5fe00130df7fa3 100644 --- a/lib/spack/spack/test/cmd/find.py +++ b/lib/spack/spack/test/cmd/find.py @@ -14,7 +14,7 @@ import spack.concretize import spack.environment as ev import spack.package_base -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.store import spack.user_environment as uenv @@ -460,7 +460,7 @@ def test_environment_with_version_range_in_compiler_doesnt_fail( @pytest.fixture def test_repo(mock_stage): with spack.repo.use_repositories( - os.path.join(spack.paths.test_repos_path, "spack_repo", "find") + os.path.join(paths.test_repos_path, "spack_repo", "find") ) as mock_packages_repo: yield mock_packages_repo diff --git a/lib/spack/spack/test/cmd/license.py b/lib/spack/spack/test/cmd/license.py index 4ab5188e08cf4f..42d47d2f98e75e 100644 --- a/lib/spack/spack/test/cmd/license.py +++ b/lib/spack/spack/test/cmd/license.py @@ -8,7 +8,7 @@ import pytest -import spack.paths +from spack.paths import locations as paths from spack.llnl.util.filesystem import mkdirp, touch from spack.main import SpackCommand @@ -19,8 +19,8 @@ def test_list_files(): files = license("list-files").strip().split("\n") - assert all(f.startswith(spack.paths.prefix) for f in files) - assert os.path.join(spack.paths.bin_path, "spack") in files + assert all(f.startswith(paths.prefix) for f in files) + assert os.path.join(paths.bin_path, "spack") in files assert os.path.abspath(__file__) in files diff --git a/lib/spack/spack/test/cmd/list.py b/lib/spack/spack/test/cmd/list.py index ee45f163a91e5d..a4f91f8062e77c 100644 --- a/lib/spack/spack/test/cmd/list.py +++ b/lib/spack/spack/test/cmd/list.py @@ -7,7 +7,7 @@ import pytest -import spack.paths +from spack.paths import locations as paths import spack.repo from spack.main import SpackCommand @@ -130,8 +130,8 @@ def test_list_count(): def test_list_repos(): with spack.repo.use_repositories( - os.path.join(spack.paths.test_repos_path, "spack_repo", "builtin_mock"), - os.path.join(spack.paths.test_repos_path, "spack_repo", "builder_test"), + os.path.join(paths.test_repos_path, "spack_repo", "builtin_mock"), + os.path.join(paths.test_repos_path, "spack_repo", "builder_test"), ): total_pkgs = len(list().strip().split()) mock_pkgs = len(list("-r", "builtin_mock").strip().split()) diff --git a/lib/spack/spack/test/cmd/location.py b/lib/spack/spack/test/cmd/location.py index 4a9f5830a39c16..c5206a5b63a9b8 100644 --- a/lib/spack/spack/test/cmd/location.py +++ b/lib/spack/spack/test/cmd/location.py @@ -10,7 +10,7 @@ import spack.concretize import spack.environment as ev import spack.main -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.stage from spack.llnl.util.filesystem import mkdirp @@ -123,9 +123,9 @@ def test_location_package_dir(mock_spec): @pytest.mark.parametrize( "option,expected", [ - ("--module-dir", spack.paths.module_path), - ("--packages", spack.paths.mock_packages_path), - ("--spack-root", spack.paths.prefix), + ("--module-dir", paths.module_path), + ("--packages", paths.mock_packages_path), + ("--spack-root", paths.prefix), ], ) def test_location_paths_options(option, expected): @@ -159,8 +159,8 @@ def test_location_stages(mock_spec): def test_location_specified_repo(): """Tests spack location --repo .""" with spack.repo.use_repositories( - os.path.join(spack.paths.test_repos_path, "spack_repo", "builtin_mock"), - os.path.join(spack.paths.test_repos_path, "spack_repo", "builder_test"), + os.path.join(paths.test_repos_path, "spack_repo", "builtin_mock"), + os.path.join(paths.test_repos_path, "spack_repo", "builder_test"), ): assert location("--repo").strip() == spack.repo.PATH.get_repo("builtin_mock").root assert ( diff --git a/lib/spack/spack/test/cmd/pkg.py b/lib/spack/spack/test/cmd/pkg.py index 6a1f6dfd1244c1..1bf85f003cef33 100644 --- a/lib/spack/spack/test/cmd/pkg.py +++ b/lib/spack/spack/test/cmd/pkg.py @@ -10,7 +10,7 @@ import spack.cmd import spack.cmd.pkg import spack.main -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.util.executable import spack.util.file_cache @@ -43,7 +43,7 @@ def _builtin_mock_copy( # create spack_repo subdir (root_dir / "spack_repo").mkdir() repo_dir = root_dir / "spack_repo" / "builtin_mock" - shutil.copytree(spack.paths.mock_packages_path, str(repo_dir)) + shutil.copytree(paths.mock_packages_path, str(repo_dir)) repo_cache = spack.util.file_cache.FileCache(root_dir / "cache") mock_repo = spack.repo.Repo(str(repo_dir), cache=repo_cache) diff --git a/lib/spack/spack/test/cmd/style.py b/lib/spack/spack/test/cmd/style.py index a8dbe772dd55ee..178c1e3099e0b7 100644 --- a/lib/spack/spack/test/cmd/style.py +++ b/lib/spack/spack/test/cmd/style.py @@ -13,14 +13,14 @@ import spack.cmd.style import spack.main -import spack.paths +from spack.paths import locations as paths import spack.repo from spack.cmd.style import _run_import_check, changed_files from spack.llnl.util.filesystem import FileFilter, working_dir from spack.util.executable import which #: directory with sample style files -style_data = os.path.join(spack.paths.test_path, "data", "style") +style_data = os.path.join(paths.test_path, "data", "style") style = spack.main.SpackCommand("style") @@ -46,9 +46,9 @@ def flake8_package(tmp_path: pathlib.Path): change to the ``flake8`` mock package, yields the filename, then undoes the change on cleanup. """ - repo = spack.repo.from_path(spack.paths.mock_packages_path) + repo = spack.repo.from_path(paths.mock_packages_path) filename = repo.filename_for_package_name("flake8") - rel_path = os.path.dirname(os.path.relpath(filename, spack.paths.prefix)) + rel_path = os.path.dirname(os.path.relpath(filename, paths.prefix)) tmp = tmp_path / rel_path / "flake8-ci-package.py" tmp.parent.mkdir(parents=True, exist_ok=True) tmp.touch() @@ -63,7 +63,7 @@ def flake8_package(tmp_path: pathlib.Path): @pytest.fixture def flake8_package_with_errors(scope="function"): """A flake8 package with errors.""" - repo = spack.repo.from_path(spack.paths.mock_packages_path) + repo = spack.repo.from_path(paths.mock_packages_path) filename = repo.filename_for_package_name("flake8") tmp = filename + ".tmp" @@ -121,7 +121,7 @@ def test_changed_no_base(git, tmp_path: pathlib.Path, capfd): def test_changed_files_all_files(mock_packages): # it's hard to guarantee "all files", so do some sanity checks. files = { - os.path.join(spack.paths.prefix, os.path.normpath(path)) + os.path.join(paths.prefix, os.path.normpath(path)) for path in changed_files(all_files=True) } @@ -136,10 +136,10 @@ def test_changed_files_all_files(mock_packages): assert zlib_file in files # a core spack file - assert os.path.join(spack.paths.module_path, "spec.py") in files + assert os.path.join(paths.module_path, "spec.py") in files # a mock package - repo = spack.repo.from_path(spack.paths.mock_packages_path) + repo = spack.repo.from_path(paths.mock_packages_path) filename = repo.filename_for_package_name("flake8") assert filename in files @@ -147,7 +147,7 @@ def test_changed_files_all_files(mock_packages): assert __file__ in files # ensure externals are excluded - assert not any(f.startswith(spack.paths.vendor_path) for f in files) + assert not any(f.startswith(paths.vendor_path) for f in files) def test_bad_root(tmp_path: pathlib.Path): @@ -256,7 +256,7 @@ def test_external_root(external_style_root): @pytest.mark.skipif(not FLAKE8, reason="flake8 is not installed.") def test_style(flake8_package, tmp_path: pathlib.Path): - root_relative = os.path.relpath(flake8_package, spack.paths.prefix) + root_relative = os.path.relpath(flake8_package, paths.prefix) # use a working directory to test cwd-relative paths, as tests run in # the spack prefix by default @@ -282,7 +282,7 @@ def test_style(flake8_package, tmp_path: pathlib.Path): @pytest.mark.skipif(not FLAKE8, reason="flake8 is not installed.") def test_style_with_errors(flake8_package_with_errors): - root_relative = os.path.relpath(flake8_package_with_errors, spack.paths.prefix) + root_relative = os.path.relpath(flake8_package_with_errors, paths.prefix) output = style( "--tool", "flake8", "--root-relative", flake8_package_with_errors, fail_on_error=False ) @@ -335,7 +335,7 @@ def foo(config: "spack.error.SpackError"): fix=False, out=output_buf, root_relative=False, - root=spack.paths.prefix, + root=paths.prefix, working_dir=root, ) output = output_buf.getvalue() @@ -356,7 +356,7 @@ def foo(config: "spack.error.SpackError"): fix=True, out=output_buf, root_relative=False, - root=spack.paths.prefix, + root=paths.prefix, working_dir=root, ) output = output_buf.getvalue() @@ -372,7 +372,7 @@ def foo(config: "spack.error.SpackError"): fix=True, out=output_buf, root_relative=False, - root=spack.paths.prefix, + root=paths.prefix, working_dir=root, ) output = output_buf.getvalue() @@ -413,8 +413,8 @@ def test_case_sensitive_imports(tmp_path: pathlib.Path): def test_pkg_imports(): - assert spack.cmd.style._module_part(spack.paths.prefix, "spack.pkg.builtin.boost") is None - assert spack.cmd.style._module_part(spack.paths.prefix, "spack.pkg") is None + assert spack.cmd.style._module_part(paths.prefix, "spack.pkg.builtin.boost") is None + assert spack.cmd.style._module_part(paths.prefix, "spack.pkg") is None def test_spec_strings(tmp_path: pathlib.Path): diff --git a/lib/spack/spack/test/cmd/test.py b/lib/spack/spack/test/cmd/test.py index b78a32738d7898..7fff1cc7ad7283 100644 --- a/lib/spack/spack/test/cmd/test.py +++ b/lib/spack/spack/test/cmd/test.py @@ -14,7 +14,7 @@ import spack.config import spack.install_test import spack.main -import spack.paths +from spack.paths import locations as paths from spack.install_test import TestStatus from spack.llnl.util.filesystem import copy_tree, working_dir from spack.main import SpackCommand @@ -228,7 +228,7 @@ def test_read_old_results(mock_packages, mock_test_stage): # spack install printing-package # spack test run --alias printpkg printing-package - test_data_src = os.path.join(spack.paths.test_path, "data", "test", "test_stage") + test_data_src = os.path.join(paths.test_path, "data", "test", "test_stage") # Copy the old test data into the mock stage directory copy_tree(test_data_src, mock_test_stage) diff --git a/lib/spack/spack/test/concretization/compiler_runtimes.py b/lib/spack/spack/test/concretization/compiler_runtimes.py index 21912c4033c640..ec1b4b73fee7ed 100644 --- a/lib/spack/spack/test/concretization/compiler_runtimes.py +++ b/lib/spack/spack/test/concretization/compiler_runtimes.py @@ -11,7 +11,7 @@ import spack.concretize import spack.config -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.solver.asp import spack.spec @@ -34,7 +34,7 @@ def _concretize_with_reuse(*, root_str, reused_str, config): @pytest.fixture def runtime_repo(mutable_config): - repo = os.path.join(spack.paths.test_repos_path, "spack_repo", "compiler_runtime_test") + repo = os.path.join(paths.test_repos_path, "spack_repo", "compiler_runtime_test") with spack.repo.use_repositories(repo) as mock_repo: yield mock_repo diff --git a/lib/spack/spack/test/concretization/core.py b/lib/spack/spack/test/concretization/core.py index 4574612d9e2a7e..d531117127b249 100644 --- a/lib/spack/spack/test/concretization/core.py +++ b/lib/spack/spack/test/concretization/core.py @@ -27,7 +27,7 @@ import spack.hash_types as ht import spack.llnl.util.lang import spack.package_base -import spack.paths +from spack.paths import locations as paths import spack.platforms import spack.platforms.test import spack.repo @@ -2493,7 +2493,7 @@ def test_select_lower_priority_package_from_repository_stack( """ # 'builtin_mock" and "duplicates_test" share a 'gmake' package additional_repo = os.path.join( - spack.paths.test_repos_path, "spack_repo", "duplicates_test" + paths.test_repos_path, "spack_repo", "duplicates_test" ) with spack.repo.use_repositories(additional_repo, override=False): s = spack.concretize.concretize_one(spec_str) @@ -2745,7 +2745,7 @@ def test_git_based_version_must_exist_to_use_ref(self): @pytest.fixture() def duplicates_test_repository(): - repository_path = os.path.join(spack.paths.test_repos_path, "spack_repo", "duplicates_test") + repository_path = os.path.join(paths.test_repos_path, "spack_repo", "duplicates_test") with spack.repo.use_repositories(repository_path) as mock_repo: yield mock_repo @@ -2980,7 +2980,7 @@ def test_adding_specs(self, input_specs, default_mock_concretization): @pytest.fixture() def edges_test_repository(): - repository_path = os.path.join(spack.paths.test_repos_path, "spack_repo", "edges_test") + repository_path = os.path.join(paths.test_repos_path, "spack_repo", "edges_test") with spack.repo.use_repositories(repository_path) as mock_repo: yield mock_repo diff --git a/lib/spack/spack/test/concretization/flag_mixing.py b/lib/spack/spack/test/concretization/flag_mixing.py index 59e1ee6e583957..18993926afb912 100644 --- a/lib/spack/spack/test/concretization/flag_mixing.py +++ b/lib/spack/spack/test/concretization/flag_mixing.py @@ -38,7 +38,7 @@ import spack.concretize import spack.config import spack.environment as ev -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.spec import spack.util.spack_yaml as syaml @@ -46,7 +46,7 @@ @pytest.fixture def test_repo(mutable_config, monkeypatch, mock_stage): - repo_dir = pathlib.Path(spack.paths.test_repos_path) / "spack_repo" / "flags_test" + repo_dir = pathlib.Path(paths.test_repos_path) / "spack_repo" / "flags_test" with spack.repo.use_repositories(str(repo_dir)) as mock_packages_repo: yield mock_packages_repo diff --git a/lib/spack/spack/test/concretization/requirements.py b/lib/spack/spack/test/concretization/requirements.py index 1a3ab4f43b1fa7..18835d87890b92 100644 --- a/lib/spack/spack/test/concretization/requirements.py +++ b/lib/spack/spack/test/concretization/requirements.py @@ -10,7 +10,7 @@ import spack.error import spack.installer import spack.package_base -import spack.paths +from spack.paths import locations as paths import spack.platforms import spack.repo import spack.solver.asp @@ -32,7 +32,7 @@ def update_packages_config(conf_str): @pytest.fixture def test_repo(mutable_config, monkeypatch, mock_stage): - repo_dir = pathlib.Path(spack.paths.test_repos_path) / "spack_repo" / "requirements_test" + repo_dir = pathlib.Path(paths.test_repos_path) / "spack_repo" / "requirements_test" with spack.repo.use_repositories(str(repo_dir)) as mock_packages_repo: yield mock_packages_repo diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 4d663dd7a0f067..0dc4a1fc96c72f 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -20,7 +20,7 @@ import spack.error import spack.llnl.util.filesystem as fs import spack.package_base -import spack.paths +from spack.paths import locations as paths import spack.platforms import spack.repo import spack.schema.compilers @@ -340,13 +340,13 @@ def __init__(self, path): def test_substitute_config_variables(mock_low_high_config, monkeypatch): - prefix = spack.paths.prefix.lstrip("/") + prefix = paths.prefix.lstrip("/") assert cross_plat_join( os.sep + os.path.join("foo", "bar", "baz"), prefix ) == spack_path.canonicalize_path("/foo/bar/baz/$spack") assert cross_plat_join( - spack.paths.prefix, os.path.join("foo", "bar", "baz") + paths.prefix, os.path.join("foo", "bar", "baz") ) == spack_path.canonicalize_path("$spack/foo/bar/baz/") assert cross_plat_join( @@ -358,7 +358,7 @@ def test_substitute_config_variables(mock_low_high_config, monkeypatch): ) == spack_path.canonicalize_path("/foo/bar/baz/${spack}") assert cross_plat_join( - spack.paths.prefix, os.path.join("foo", "bar", "baz") + paths.prefix, os.path.join("foo", "bar", "baz") ) == spack_path.canonicalize_path("${spack}/foo/bar/baz/") assert cross_plat_join( @@ -446,7 +446,7 @@ def test_substitute_user(mock_low_high_config): def test_substitute_user_cache(mock_low_high_config): - user_cache_path = spack.paths.user_cache_path + user_cache_path = paths.user_cache_path assert user_cache_path + os.sep + "baz" == spack_path.canonicalize_path( os.path.join("$user_cache_path", "baz") ) @@ -1149,7 +1149,7 @@ def test_bad_path_double_override(config): def test_license_dir_config(mutable_config, mock_packages): """Ensure license directory is customizable""" - expected_dir = spack.paths.default_license_dir + expected_dir = paths.default_license_dir assert spack.config.get("config:license_dir") == expected_dir assert spack.package_base.PackageBase.global_license_dir == expected_dir assert spack.repo.PATH.get_pkg_class("pkg-a").global_license_dir == expected_dir @@ -1720,7 +1720,7 @@ def test_included_path_git_unsat( def test_included_path_git( tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, key, value, capfd ): - monkeypatch.setattr(spack.paths, "user_cache_path", str(tmp_path)) + monkeypatch.setattr(paths, "user_cache_path", str(tmp_path)) class MockIncludeGit(spack.util.executable.Executable): def __init__(self, required: bool): @@ -1788,7 +1788,7 @@ def _checkout(*args, **kwargs): def test_included_path_git_errs(tmp_path: pathlib.Path, mock_low_high_config, monkeypatch): - monkeypatch.setattr(spack.paths, "user_cache_path", str(tmp_path)) + monkeypatch.setattr(paths, "user_cache_path", str(tmp_path)) paths = ["concretizer.yaml"] entry = { diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 51740093b66c5b..e9026b3a6255f8 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -45,6 +45,7 @@ import spack.modules.common import spack.package_base import spack.paths +from spack.paths import locations as paths import spack.platforms import spack.repo import spack.solver.asp @@ -132,7 +133,7 @@ def git(): # @pytest.fixture(scope="session") def last_two_git_commits(git): - spack_git_path = spack.paths.prefix + spack_git_path = paths.prefix with working_dir(spack_git_path): git_log_out = git("log", "-n", "2", output=str, error=os.devnull) @@ -150,12 +151,12 @@ def write_file(filename, contents): @pytest.fixture def override_git_repos_cache_path(tmp_path: Path): - saved = spack.paths.user_repos_cache_path + saved = paths.user_repos_cache_path tmp_git_path = tmp_path / "git-repo-cache-path-for-tests" tmp_git_path.mkdir() - spack.paths.user_repos_cache_path = str(tmp_git_path) + paths.user_repos_cache_path = str(tmp_git_path) yield - spack.paths.user_repos_cache_path = saved + paths.user_repos_cache_path = saved @pytest.fixture @@ -316,19 +317,19 @@ def latest_commit(): os.makedirs(os.path.dirname(filename)) # add pkg-a as a new package to the repository - shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-0.txt", filename) + shutil.copy2(f"{paths.test_path}/data/conftest/diff-test/package-0.txt", filename) git("add", filename) commit("diff-test: new package") commits.append(latest_commit()) # add v2.1.5 to pkg-a - shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-1.txt", filename) + shutil.copy2(f"{paths.test_path}/data/conftest/diff-test/package-1.txt", filename) git("add", filename) commit("diff-test: add v2.1.5") commits.append(latest_commit()) # add v2.1.6 to pkg-a - shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-2.txt", filename) + shutil.copy2(f"{paths.test_path}/data/conftest/diff-test/package-2.txt", filename) git("add", filename) commit("diff-test: add v2.1.6") commits.append(latest_commit()) @@ -693,7 +694,7 @@ def _use_test_platform(test_platform): # @pytest.fixture(scope="session") def mock_packages_repo(): - yield spack.repo.from_path(spack.paths.mock_packages_path) + yield spack.repo.from_path(paths.mock_packages_path) def _pkg_install_fn(pkg, spec, prefix): @@ -742,7 +743,7 @@ def mock_packages(mock_packages_repo, mock_pkg_install, request): def mutable_mock_repo(mock_packages_repo, request): """Function-scoped mock packages, for tests that need to modify them.""" ensure_configuration_fixture_run_before(request) - mock_repo = spack.repo.from_path(spack.paths.mock_packages_path) + mock_repo = spack.repo.from_path(paths.mock_packages_path) with spack.repo.use_repositories(mock_repo) as mock_packages_repo: yield mock_packages_repo @@ -757,7 +758,7 @@ def __init__(self, root_directory: str) -> None: namespace = f"test_namespace_{RepoBuilder._counter}" repo_root = os.path.join(root_directory, namespace) os.makedirs(repo_root, exist_ok=True) - self.template_dirs = (os.path.join(spack.paths.share_path, "templates"),) + self.template_dirs = (os.path.join(paths.share_path, "templates"),) self.root, self.namespace = spack.repo.create_repo(repo_root, namespace) self.build_system_name = f"test_build_system_{self.namespace}" self._add_build_system() @@ -856,7 +857,7 @@ def default_config(): This ensures we can test the real default configuration without having tests fail when the user overrides the defaults that we test against.""" - defaults_path = os.path.join(spack.paths.etc_path, "defaults") + defaults_path = os.path.join(paths.etc_path, "defaults") if sys.platform == "win32": defaults_path = os.path.join(defaults_path, "windows") with spack.config.use_configuration(defaults_path) as defaults_config: @@ -869,7 +870,7 @@ def mock_uarch_json(tmp_path_factory: pytest.TempPathFactory): tmpdir = tmp_path_factory.mktemp("microarchitectures") uarch_json_source = ( - Path(spack.paths.test_path) / "data" / "microarchitectures" / "microarchitectures.json" + Path(paths.test_path) / "data" / "microarchitectures" / "microarchitectures.json" ) uarch_json_dest = tmpdir / "microarchitectures.json" shutil.copy2(uarch_json_source, uarch_json_dest) @@ -913,7 +914,7 @@ def configuration_dir(tmp_path_factory: pytest.TempPathFactory, linux_os): # /data/config has mock config yaml files in it # copy these to the site config. - test_config = Path(spack.paths.test_path) / "data" / "config" + test_config = Path(paths.test_path) / "data" / "config" shutil.copytree(test_config, tmp_path / "site") # Create temporary 'defaults', 'site' and 'user' folders @@ -1372,7 +1373,7 @@ def module_configuration(monkeypatch, request, mutable_config): # Key for specific settings relative to this module type writer_key = str(writer_mod.__name__).split(".")[-1] # Root folder for configuration - root_for_conf = os.path.join(spack.paths.test_path, "data", "modules", writer_key) + root_for_conf = os.path.join(paths.test_path, "data", "modules", writer_key) # ConfigUpdate, when called, will modify configuration, so we need to use # the mutable_config fixture @@ -1975,7 +1976,7 @@ def mock_clone_repo(tmp_path_factory: pytest.TempPathFactory): ) shutil.copytree( - os.path.join(spack.paths.mock_packages_path, spack.repo.packages_dir_name), + os.path.join(paths.mock_packages_path, spack.repo.packages_dir_name), os.path.join(str(repodir), spack.repo.packages_dir_name), ) @@ -2109,7 +2110,7 @@ def noncyclical_dir_structure(tmp_path: Path): @pytest.fixture(scope="function") def mock_config_data(): - config_data_dir = os.path.join(spack.paths.test_path, "data", "config") + config_data_dir = os.path.join(paths.test_path, "data", "config") return config_data_dir, os.listdir(config_data_dir) diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 7191f1225593f7..f894139809d7c9 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -34,7 +34,7 @@ import spack.llnl.util.filesystem as fs import spack.llnl.util.lock as lk import spack.package_base -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.spec import spack.store @@ -1331,7 +1331,7 @@ def test_querying_reindexed_database_specfilev5(tmp_path: pathlib.Path): """Tests that we can query a reindexed database from before compilers as dependencies, and get appropriate results for % and similar selections. """ - test_path = pathlib.Path(spack.paths.test_path) + test_path = pathlib.Path(paths.test_path) zipfile = test_path / "data" / "database" / "index.json.v7_v8.json.gz" with gzip.open(str(zipfile), "rt", encoding="utf-8") as f: data = json.load(f) diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 63e60500c4c526..266c7a1ab98997 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -13,7 +13,7 @@ import spack.concretize import spack.hash_types -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.spec import spack.util.file_cache @@ -158,7 +158,7 @@ def test_handle_unknown_package(temporary_store, config, mock_packages, tmp_path layout = temporary_store.layout repo_cache = spack.util.file_cache.FileCache(tmp_path / "cache") - mock_db = spack.repo.Repo(spack.paths.mock_packages_path, cache=repo_cache) + mock_db = spack.repo.Repo(paths.mock_packages_path, cache=repo_cache) not_in_mock = set.difference( set(spack.repo.all_package_names()), set(mock_db.all_package_names()) @@ -178,7 +178,7 @@ def test_handle_unknown_package(temporary_store, config, mock_packages, tmp_path layout.create_install_directory(spec) installed_specs[spec] = layout.path_for_spec(spec) - with spack.repo.use_repositories(spack.paths.mock_packages_path): + with spack.repo.use_repositories(paths.mock_packages_path): # Now check that even without the package files, we know # enough to read a spec from the spec file. for spec, path in installed_specs.items(): diff --git a/lib/spack/spack/test/link_paths.py b/lib/spack/spack/test/link_paths.py index 5c07c8e987700f..05c0f8d63b9ef2 100644 --- a/lib/spack/spack/test/link_paths.py +++ b/lib/spack/spack/test/link_paths.py @@ -8,18 +8,18 @@ import pytest import spack.compilers.libraries -import spack.paths +from spack.paths import locations as paths from spack.compilers.libraries import parse_non_system_link_dirs drive = "" if sys.platform == "win32": - match = re.search(r"[A-Za-z]:", spack.paths.test_path) + match = re.search(r"[A-Za-z]:", paths.test_path) if match: drive = match.group() root = drive + os.sep #: directory with sample compiler data -datadir = os.path.join(spack.paths.test_path, "data", "compiler_verbose_output") +datadir = os.path.join(paths.test_path, "data", "compiler_verbose_output") @pytest.fixture(autouse=True) diff --git a/lib/spack/spack/test/llnl/util/file_list.py b/lib/spack/spack/test/llnl/util/file_list.py index 6c5ddf4618d83d..4c18dae06b1625 100644 --- a/lib/spack/spack/test/llnl/util/file_list.py +++ b/lib/spack/spack/test/llnl/util/file_list.py @@ -8,7 +8,7 @@ import pytest -import spack.paths +from spack.paths import locations as paths from spack.llnl.util.filesystem import HeaderList, LibraryList, find_headers, find_libraries @@ -240,7 +240,7 @@ def test_add(self, header_list): #: Directory where the data for the test below is stored -search_dir = os.path.join(spack.paths.test_path, "data", "directory_search") +search_dir = os.path.join(paths.test_path, "data", "directory_search") @pytest.mark.parametrize( diff --git a/lib/spack/spack/test/llnl/util/filesystem.py b/lib/spack/spack/test/llnl/util/filesystem.py index 408303cda1b8af..96a2211ec1378b 100644 --- a/lib/spack/spack/test/llnl/util/filesystem.py +++ b/lib/spack/spack/test/llnl/util/filesystem.py @@ -14,7 +14,7 @@ import pytest import spack.llnl.util.filesystem as fs -import spack.paths +from spack.paths import locations as paths @pytest.fixture() @@ -504,7 +504,7 @@ def test_filter_files_with_different_encodings( # All files given as input to this test must satisfy the pre-requisite # that the 'replacement' string is not present in the file initially and # that there's at least one match for the regex - original_file = os.path.join(spack.paths.test_path, "data", "filter_file", filename) + original_file = os.path.join(paths.test_path, "data", "filter_file", filename) target_file = os.path.join(str(tmp_path), filename) shutil.copy(original_file, target_file) # This should not raise exceptions @@ -552,7 +552,7 @@ def test_filter_files_multiple(tmp_path: pathlib.Path): # All files given as input to this test must satisfy the pre-requisite # that the 'replacement' string is not present in the file initially and # that there's at least one match for the regex - original_file = os.path.join(spack.paths.test_path, "data", "filter_file", "x86_cpuid_info.c") + original_file = os.path.join(paths.test_path, "data", "filter_file", "x86_cpuid_info.c") target_file = os.path.join(str(tmp_path), "x86_cpuid_info.c") shutil.copy(original_file, target_file) # This should not raise exceptions @@ -567,7 +567,7 @@ def test_filter_files_multiple(tmp_path: pathlib.Path): def test_filter_files_start_stop(tmp_path: pathlib.Path): - original_file = os.path.join(spack.paths.test_path, "data", "filter_file", "start_stop.txt") + original_file = os.path.join(paths.test_path, "data", "filter_file", "start_stop.txt") target_file = os.path.join(str(tmp_path), "start_stop.txt") shutil.copy(original_file, target_file) # None of the following should happen: diff --git a/lib/spack/spack/test/main.py b/lib/spack/spack/test/main.py index d31a8a442b2cae..1a7948b8039f5f 100644 --- a/lib/spack/spack/test/main.py +++ b/lib/spack/spack/test/main.py @@ -15,7 +15,7 @@ import spack.error import spack.llnl.util.filesystem as fs import spack.main -import spack.paths +from spack.paths import locations as paths import spack.platforms import spack.util.executable as exe import spack.util.git @@ -74,7 +74,7 @@ def test_git_sha_output(tmp_path: pathlib.Path, working_env, monkeypatch): def test_get_version_no_repo(tmp_path: pathlib.Path, monkeypatch): - monkeypatch.setattr(spack.paths, "prefix", str(tmp_path)) + monkeypatch.setattr(paths, "prefix", str(tmp_path)) assert spack.spack_version == spack.get_version() diff --git a/lib/spack/spack/test/patch.py b/lib/spack/spack/test/patch.py index 3a8b61b29b43a7..fb0631fd5d9f9b 100644 --- a/lib/spack/spack/test/patch.py +++ b/lib/spack/spack/test/patch.py @@ -16,7 +16,7 @@ import spack.error import spack.fetch_strategy import spack.patch -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.spec import spack.stage @@ -73,7 +73,7 @@ def mock_patch_stage(tmp_path_factory: pytest.TempPathFactory, monkeypatch): return mock_path -data_path = os.path.join(spack.paths.test_path, "data", "patch") +data_path = os.path.join(paths.test_path, "data", "patch") @pytest.mark.not_on_windows("Line ending conflict on Windows") diff --git a/lib/spack/spack/test/repo.py b/lib/spack/spack/test/repo.py index 06a90559af9a3f..e386d540a650ed 100644 --- a/lib/spack/spack/test/repo.py +++ b/lib/spack/spack/test/repo.py @@ -8,7 +8,7 @@ import spack.environment import spack.package_base -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.schema.repos import spack.spec @@ -178,7 +178,7 @@ def _repo_descriptors(repos): for entry in repos: if entry == "mock": descriptors["builtin_mock"] = spack.repo.LocalRepoDescriptor( - "builtin_mock", spack.paths.mock_packages_path + "builtin_mock", paths.mock_packages_path ) if entry == "extra": repo_dir = tmp_path / "extra_mock" @@ -210,9 +210,7 @@ def test_path_computation_with_names(method_name, mock_packages_repo): def test_use_repositories_and_import(): """Tests that use_repositories changes the import search too""" - import spack.paths - - repo_dir = pathlib.Path(spack.paths.test_repos_path) + repo_dir = pathlib.Path(paths.test_repos_path) with spack.repo.use_repositories(str(repo_dir / "spack_repo" / "compiler_runtime_test")): import spack_repo.compiler_runtime_test.packages.gcc_runtime.package # type: ignore[import] # noqa: E501 @@ -227,7 +225,7 @@ class TestRepo: """ def test_creation(self, mock_test_cache): - repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) + repo = spack.repo.Repo(paths.mock_packages_path, cache=mock_test_cache) assert repo.config_file.endswith("repo.yaml") assert repo.namespace == "builtin_mock" @@ -235,7 +233,7 @@ def test_creation(self, mock_test_cache): "name,expected", [("mpi", True), ("mpich", False), ("mpileaks", False)] ) def test_is_virtual(self, name, expected, mock_test_cache): - repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) + repo = spack.repo.Repo(paths.mock_packages_path, cache=mock_test_cache) assert repo.is_virtual(name) is expected assert repo.is_virtual_safe(name) is expected @@ -268,7 +266,7 @@ def test_real_name(self, module_name, pkg_name, mock_test_cache, tmp_path: pathl @pytest.mark.parametrize("name", ["mpileaks", "7zip", "dla-future"]) def test_get(self, name, mock_test_cache): - repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) + repo = spack.repo.Repo(paths.mock_packages_path, cache=mock_test_cache) mock_spec = spack.spec.Spec(name) mock_spec._mark_concrete() pkg = repo.get(mock_spec) @@ -276,7 +274,7 @@ def test_get(self, name, mock_test_cache): @pytest.mark.parametrize("virtual_name,expected", [("mpi", ["mpich", "zmpi"])]) def test_providers(self, virtual_name, expected, mock_test_cache): - repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) + repo = spack.repo.Repo(paths.mock_packages_path, cache=mock_test_cache) provider_names = {x.name for x in repo.providers_for(virtual_name)} assert provider_names.issuperset(expected) @@ -285,14 +283,14 @@ def test_providers(self, virtual_name, expected, mock_test_cache): [("python", ["py-extension1", "python-venv"]), ("perl", ["perl-extension"])], ) def test_extensions(self, extended, expected, mock_test_cache): - repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) + repo = spack.repo.Repo(paths.mock_packages_path, cache=mock_test_cache) repo_path = spack.repo.RepoPath(repo) for instance in (repo, repo_path): provider_names = {x.name for x in instance.extensions_for(extended)} assert provider_names.issuperset(expected) def test_all_package_names(self, mock_test_cache): - repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) + repo = spack.repo.Repo(paths.mock_packages_path, cache=mock_test_cache) repo_path = spack.repo.RepoPath(repo) for instance in (repo, repo_path): @@ -304,7 +302,7 @@ def test_all_package_names(self, mock_test_cache): assert instance.is_virtual_safe(name) def test_packages_with_tags(self, mock_test_cache): - repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) + repo = spack.repo.Repo(paths.mock_packages_path, cache=mock_test_cache) repo_path = spack.repo.RepoPath(repo) for instance in (repo, repo_path): @@ -322,7 +320,7 @@ def test_creation_from_string(self, mock_test_cache): spack.repo.RepoDescriptors( { "builtin_mock": spack.repo.LocalRepoDescriptor( - "builtin_mock", spack.paths.mock_packages_path + "builtin_mock", paths.mock_packages_path ) } ), @@ -336,7 +334,7 @@ def test_get_repo(self, mock_test_cache): spack.repo.RepoDescriptors( { "builtin_mock": spack.repo.LocalRepoDescriptor( - "builtin_mock", spack.paths.mock_packages_path + "builtin_mock", paths.mock_packages_path ) } ), diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index bd3e23bf84913e..568a45d9abc3b1 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -11,7 +11,7 @@ import spack.directives import spack.error import spack.llnl.util.lang -import spack.paths +from spack.paths import locations as paths import spack.solver.asp import spack.spec import spack.spec_parser @@ -926,7 +926,7 @@ def test_spec_formatting(self, default_mock_concretization): ] other_segments = [ - ("{spack_root}", spack.paths.spack_root), + ("{spack_root}", paths.spack_root), ("{spack_install}", spack.store.STORE.layout.root), ] diff --git a/lib/spack/spack/test/spec_yaml.py b/lib/spack/spack/test/spec_yaml.py index 6ff98f15d63563..4b7640cc86922a 100644 --- a/lib/spack/spack/test/spec_yaml.py +++ b/lib/spack/spack/test/spec_yaml.py @@ -23,7 +23,7 @@ import spack.concretize import spack.config import spack.hash_types as ht -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.spec import spack.test.conftest @@ -47,7 +47,7 @@ def check_json_round_trip(spec): def test_read_spec_from_signed_json(): - spec_dir = os.path.join(spack.paths.test_path, "data", "mirrors", "signed_json") + spec_dir = os.path.join(paths.test_path, "data", "mirrors", "signed_json") file_name = ( "linux-ubuntu18.04-haswell-gcc-8.4.0-" "zlib-1.2.12-g7otk5dra3hifqxej36m5qzm7uyghqgb.spec.json.sig" @@ -411,7 +411,7 @@ def test_legacy_yaml(install_mockery, mock_packages): ], ) def test_load_json_specfiles(specfile, expected_hash, reader_cls): - fullpath = os.path.join(spack.paths.test_path, "data", specfile) + fullpath = os.path.join(paths.test_path, "data", specfile) with gzip.open(fullpath, "rt", encoding="utf-8") as f: data = json.load(f) diff --git a/lib/spack/spack/test/util/package_hash.py b/lib/spack/spack/test/util/package_hash.py index 58a6a3b0c85a5d..208957793dc742 100644 --- a/lib/spack/spack/test/util/package_hash.py +++ b/lib/spack/spack/test/util/package_hash.py @@ -9,13 +9,13 @@ import spack.concretize import spack.directives_meta -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.util.package_hash as ph from spack.spec import Spec from spack.util.unparse import unparse -datadir = os.path.join(spack.paths.test_path, "data", "unparse") +datadir = os.path.join(paths.test_path, "data", "unparse") def compare_sans_name(eq, spec1, spec2): diff --git a/lib/spack/spack/test/util/path.py b/lib/spack/spack/test/util/path.py index 3c821b1ce3e710..fa82dbff6efb6b 100644 --- a/lib/spack/spack/test/util/path.py +++ b/lib/spack/spack/test/util/path.py @@ -9,7 +9,7 @@ import spack.config import spack.llnl.util.tty as tty -import spack.paths +from spack.paths import locations as paths import spack.util.path as sup #: Some lines with lots of placeholders @@ -141,7 +141,7 @@ def test_path_debug_padded_filter(debug, monkeypatch): [ ("/home/spack/path/to/file.txt", "/home/spack/path/to/file.txt"), ("file:///home/another/config.yaml", "/home/another/config.yaml"), - ("path/to.txt", os.path.join(spack.paths.spack_root, "path", "to.txt")), + ("path/to.txt", os.path.join(paths.spack_root, "path", "to.txt")), (r"C:\Files (x86)\Windows\10", r"C:\Files (x86)\Windows\10"), (r"E:/spack stage", "E:\\spack stage"), ], diff --git a/lib/spack/spack/test/util/remote_file_cache.py b/lib/spack/spack/test/util/remote_file_cache.py index 0f1f44c2144284..37032228bf37b2 100644 --- a/lib/spack/spack/test/util/remote_file_cache.py +++ b/lib/spack/spack/test/util/remote_file_cache.py @@ -9,7 +9,7 @@ import spack.config import spack.llnl.util.tty as tty -import spack.paths +from spack.paths import locations as paths import spack.util.remote_file_cache as rfc_util from spack.llnl.util.filesystem import join_path @@ -36,7 +36,7 @@ def test_rfc_local_path_bad_scheme(path, err): ("file:///this/is/a/file/url/include.yaml", "/this/is/a/file/url/include.yaml"), ( "relative/packages.txt", - os.path.join(spack.paths.spack_root, "relative", "packages.txt"), + os.path.join(paths.spack_root, "relative", "packages.txt"), ), (r"C:\Files (x86)\Windows\10", r"C:\Files (x86)\Windows\10"), (r"D:/spack stage", "D:\\spack stage"), diff --git a/lib/spack/spack/test/web.py b/lib/spack/spack/test/web.py index 3a5d91b6f9a2ea..bbd836b630ccc9 100644 --- a/lib/spack/spack/test/web.py +++ b/lib/spack/spack/test/web.py @@ -15,7 +15,7 @@ import spack.config import spack.llnl.util.tty as tty import spack.mirrors.mirror -import spack.paths +from spack.paths import locations as paths import spack.url import spack.util.s3 import spack.util.url as url_util @@ -25,7 +25,7 @@ def _create_url(relative_url): - web_data_path = os.path.join(spack.paths.test_path, "data", "web") + web_data_path = os.path.join(paths.test_path, "data", "web") return url_util.path_to_file_url(os.path.join(web_data_path, relative_url)) diff --git a/lib/spack/spack/util/gpg.py b/lib/spack/spack/util/gpg.py index 406c60b77fd695..68bffb4d79c778 100644 --- a/lib/spack/spack/util/gpg.py +++ b/lib/spack/spack/util/gpg.py @@ -10,7 +10,7 @@ import spack.error import spack.llnl.util.filesystem -import spack.paths +from spack.paths import locations as paths import spack.util.executable import spack.version @@ -57,7 +57,7 @@ def init(gnupghome=None, force=False): return # Set the value of GNUPGHOME to be used in this module - GNUPGHOME = gnupghome or os.getenv("SPACK_GNUPGHOME") or spack.paths.gpg_path + GNUPGHOME = gnupghome or os.getenv("SPACK_GNUPGHOME") or paths.gpg_path # Set the executable objects for "gpg" and "gpgconf" with spack.bootstrap.ensure_bootstrap_configuration(): diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 2c274c641c25ee..eb3a3f276c7cf8 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -62,19 +62,21 @@ def replacements(): arch = architecture() + paths = spack.paths.locations + return { "spack": lambda: spack.paths_base.prefix, "user": lambda: get_user(), "tempdir": lambda: tempfile.gettempdir(), - "user_cache_path": lambda: spack.paths.user_cache_path, - "default_install_root": lambda: spack.paths.default_install_location, - "default_envs_root": lambda: spack.paths.default_envs_path, - "default_download_root": lambda: spack.paths.default_fetch_cache_path, - "spack_data_home": lambda: spack.paths.spack_data_home, - "spack_cache_home": lambda: spack.paths.spack_cache_home, - "spack_state_home": lambda: spack.paths.spack_state_home, - "spack_config_home": lambda: spack.paths.spack_config_home, - "spack_instance_id": lambda: spack.paths.spack_instance_id, + "user_cache_path": lambda: paths.user_cache_path, + "default_install_root": lambda: paths.default_install_location, + "default_envs_root": lambda: paths.default_envs_path, + "default_download_root": lambda: paths.default_fetch_cache_path, + "data_home": lambda: paths.spack_data_home, + "cache_home": lambda: paths.spack_cache_home, + "state_home": lambda: paths.spack_state_home, + #"spack_config_home": lambda: paths.spack_config_home, + "spack_instance_id": lambda: paths.spack_instance_id, "architecture": lambda: arch, "arch": lambda: arch, "platform": lambda: arch.platform, diff --git a/lib/spack/spack/version/git_ref_lookup.py b/lib/spack/spack/version/git_ref_lookup.py index 607a97e3a16b7c..82dbb2a0821e22 100644 --- a/lib/spack/spack/version/git_ref_lookup.py +++ b/lib/spack/spack/version/git_ref_lookup.py @@ -9,7 +9,7 @@ import spack.caches import spack.fetch_strategy -import spack.paths +from spack.paths import locations as paths import spack.repo import spack.util.executable import spack.util.hash @@ -125,7 +125,7 @@ def lookup_ref(self, ref) -> Tuple[Optional[str], int]: known version prior to the commit, as well as the distance from that version to the commit in the git repo. Those values are used to compare Version objects. """ - pathlib_dest = Path(spack.paths.user_repos_cache_path) / self.repository_uri + pathlib_dest = Path(paths.user_repos_cache_path) / self.repository_uri dest = str(pathlib_dest) # prepare a cache for the repository From 17be46d07d4d68030c2b8a57f22a214253314325 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 12 Dec 2025 17:55:19 -0800 Subject: [PATCH 191/506] fix default home --- lib/spack/spack/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 33d946126df603..5782677bfd9646 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -230,7 +230,7 @@ def spack_home_cfg_check(): if possible_resolution: return possible_resolution - return os.path.join("~", home_rel) + return os.path.join(os.path.expanduser("~"), home_rel) def prefer_old_location(self, old_location, new_location): # TODO: perhaps it should be configurable whether old locations From 908d7f86e5822b0cf2ecbade4e1c6e54b4d57d10 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 12 Dec 2025 17:58:56 -0800 Subject: [PATCH 192/506] no-longer used mapping --- lib/spack/spack/paths.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 5782677bfd9646..4b8274b73148c0 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -34,36 +34,6 @@ class XDG_overrides(Enum): cache_home = "SPACK_CACHE_HOME" -xdg_mapping = namedtuple("xdg_mapping", ["spack", "xdg", "xdg_default"]) - - -class XDG_mappings(Enum): - config_home = xdg_mapping( - spack=XDG_overrides.config_home.value, - xdg=XDG_vars.config_home.value, - xdg_default=os.path.join("~", ".config"), - ) - state_home = xdg_mapping( - spack=XDG_overrides.state_home.value, - xdg=XDG_vars.state_home.value, - xdg_default=os.path.join("~", ".local", "state"), - ) - data_home = xdg_mapping( - spack=XDG_overrides.data_home.value, - xdg=XDG_vars.data_home.value, - xdg_default=os.path.join("~", ".local", "share"), - ) - cache_home = xdg_mapping( - spack=XDG_overrides.cache_home.value, - xdg=XDG_vars.cache_home.value, - xdg_default=os.path.join("~", ".cache"), - ) - - -class Location_vars(Enum): - user_cache_path = "USER_CACHE_PATH" - - # This is for tests that want to clean the environment of XDG_ variables that # affect spack behavior (and the corresponding SPACK_ overrides). Note that # these vars will affect .default_test_path for the running instance, but From 9cd8ace906f67cb5fcd2eb3e9f2e86e7f97f36ca Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 15 Dec 2025 11:17:35 -0800 Subject: [PATCH 193/506] update some references --- lib/spack/spack/build_environment.py | 4 ++-- lib/spack/spack/cmd/__init__.py | 1 + lib/spack/spack/paths.py | 4 +++- lib/spack/spack/test/ci.py | 8 ++++---- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index afccbd21c01b2d..0b27335d4db96d 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -71,7 +71,7 @@ import spack.llnl.util.tty as tty import spack.multimethod import spack.package_base -from spack.paths import locations as paths +import spack.paths_base import spack.platforms import spack.schema.environment import spack.spec @@ -487,7 +487,7 @@ def set_wrapper_variables(pkg, env): env.set(SPACK_DEBUG, "TRUE") env.set(SPACK_SHORT_SPEC, pkg.spec.short_spec) env.set(SPACK_DEBUG_LOG_ID, pkg.spec.format("{name}-{hash:7}")) - env.set(SPACK_DEBUG_LOG_DIR, paths.spack_working_dir) + env.set(SPACK_DEBUG_LOG_DIR, spack.paths_base.spack_working_dir) if spack.config.get("config:ccache"): # Enable ccache in the compiler wrapper diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index bbd65466f8366e..97bfc524a4da1e 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse +import spack.paths import difflib import importlib import os diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 4b8274b73148c0..ad5570e36f0a55 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -214,4 +214,6 @@ def prefer_old_location(self, old_location, new_location): return new_location -locations = SpackPaths(paths_base.locations) \ No newline at end of file +locations = SpackPaths(paths_base.locations) + +spack_script = locations.spack_script \ No newline at end of file diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index 0193122e5cb45c..172620506053b4 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -14,7 +14,7 @@ import spack.environment as ev import spack.error import spack.llnl.util.filesystem as fs -from spack.paths import locations as paths +import spack.paths_base import spack.repo as repo import spack.util.git from spack.spec import Spec @@ -269,8 +269,8 @@ def test_setup_spack_repro_version( spack_dir = repro_dir / "spack" spack_dir.mkdir(parents=True) - prefix_save = spack.paths.prefix - monkeypatch.setattr(spack.paths, "prefix", "/garbage") + prefix_save = spack.paths_base.prefix + monkeypatch.setattr(spack.paths_base, "prefix", "/garbage") ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1) _, err = capfd.readouterr() @@ -278,7 +278,7 @@ def test_setup_spack_repro_version( assert not ret assert "Unable to find the path" in err - monkeypatch.setattr(spack.paths, "prefix", prefix_save) + monkeypatch.setattr(spack.paths_base, "prefix", prefix_save) monkeypatch.setattr(spack.util.git, "git", lambda: None) ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1) From 6df445c295b56b8cb0b918066ddc121cb2ce6621 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 15 Dec 2025 11:27:07 -0800 Subject: [PATCH 194/506] auto style --- lib/spack/spack/caches.py | 1 + lib/spack/spack/cmd/unit_test.py | 4 +- lib/spack/spack/llnl/util/lang.py | 3 -- lib/spack/spack/paths.py | 45 ++++++++++++++----- lib/spack/spack/repo.py | 4 +- lib/spack/spack/test/ci.py | 4 +- lib/spack/spack/test/cmd/ci.py | 4 +- lib/spack/spack/test/concretization/core.py | 4 +- .../spack/test/util/remote_file_cache.py | 5 +-- lib/spack/spack/util/path.py | 2 +- 10 files changed, 41 insertions(+), 35 deletions(-) diff --git a/lib/spack/spack/caches.py b/lib/spack/spack/caches.py index e0ef6c5d9933ac..3ab71da2727b56 100644 --- a/lib/spack/spack/caches.py +++ b/lib/spack/spack/caches.py @@ -13,6 +13,7 @@ from spack.llnl.util.filesystem import mkdirp from spack.paths import locations as paths + def misc_cache_location(): """The ``MISC_CACHE`` is Spack's cache for small data. diff --git a/lib/spack/spack/cmd/unit_test.py b/lib/spack/spack/cmd/unit_test.py index 3bf14f1d54dc3c..b7eb1e71ce85c2 100644 --- a/lib/spack/spack/cmd/unit_test.py +++ b/lib/spack/spack/cmd/unit_test.py @@ -120,9 +120,7 @@ def colorize(c, prefix): # To list the files we just need to inspect the filesystem, # which doesn't need to wait for pytest collection and doesn't # require parsing pytest output - files = spack.llnl.util.filesystem.find( - root=paths.test_path, files="*.py", recursive=True - ) + files = spack.llnl.util.filesystem.find(root=paths.test_path, files="*.py", recursive=True) files = [ os.path.relpath(f, start=paths.spack_root) for f in files diff --git a/lib/spack/spack/llnl/util/lang.py b/lib/spack/spack/llnl/util/lang.py index 76efb1c1e94fc7..b1d854b3611a12 100644 --- a/lib/spack/spack/llnl/util/lang.py +++ b/lib/spack/spack/llnl/util/lang.py @@ -734,9 +734,6 @@ def instance(self): # if not, just assign the result like a normal singleton self._instance = instance except AttributeError as e: - import traceback - import pdb; pdb.set_trace() - traceback.print_exc() raise Exception("AttrbuteError during creation of Singleton instance") from e return self._instance diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index ad5570e36f0a55..b6a6d799809eff 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -74,7 +74,12 @@ def __init__(self, base): @property def state_home(self): - return self.resolve_a_home(["SPACK_USER_CACHE_PATH", "SPACK_STATE_HOME"], "XDG_STATE_HOME", "state", ".local/state/spack") + return self.resolve_a_home( + ["SPACK_USER_CACHE_PATH", "SPACK_STATE_HOME"], + "XDG_STATE_HOME", + "state", + ".local/state/spack", + ) @property def cache_home(self): @@ -82,7 +87,9 @@ def cache_home(self): @property def data_home(self): - return self.resolve_a_home("SPACK_DATA_HOME", "XDG_DATA_HOME", "data", ".local/share/spack") + return self.resolve_a_home( + "SPACK_DATA_HOME", "XDG_DATA_HOME", "data", ".local/share/spack" + ) @property def user_cache_path(self): @@ -90,15 +97,21 @@ def user_cache_path(self): @property def default_install_location(self): - return self.prefer_old_location(self.base.old_install_path, os.path.join(self.data_home, "installs")) + return self.prefer_old_location( + self.base.old_install_path, os.path.join(self.data_home, "installs") + ) @property def default_envs_path(self): - return self.prefer_old_location(self.base.old_envs_path, os.path.join(self.data_home, "envs")) + return self.prefer_old_location( + self.base.old_envs_path, os.path.join(self.data_home, "envs") + ) @property def default_fetch_cache_path(self): - return self.prefer_old_location(self.base.old_fetch_cache_path, os.path.join(self.data_home, "downloads")) + return self.prefer_old_location( + self.base.old_fetch_cache_path, os.path.join(self.data_home, "downloads") + ) @property def reports_path(self): @@ -133,11 +146,15 @@ def default_user_bootstrap_path(self): @property def gpg_path(self): - return self.prefer_old_location(self.base.old_gpg_path, os.path.join(self.data_home, "gpg")) + return self.prefer_old_location( + self.base.old_gpg_path, os.path.join(self.data_home, "gpg") + ) @property def gpg_keys_path(self): - return self.prefer_old_location(self.base.old_gpg_keys_path, os.path.join(self.data_home, "gpg-keys")) + return self.prefer_old_location( + self.base.old_gpg_keys_path, os.path.join(self.data_home, "gpg-keys") + ) @property def modules_base(self): @@ -153,9 +170,7 @@ def modules_base(self): def default_misc_cache_path(self): #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) #: overridden by `config:misc_cache` - return os.path.join( - self.state_home, self.spack_instance_id, "cache" - ) + return os.path.join(self.state_home, self.spack_instance_id, "cache") def __getattr__(self, name): # Things that aren't sensitive to import cycles can import the @@ -195,7 +210,13 @@ def spack_home_cfg_check(): if h: return os.path.join(h, home_rel) - for check in [spack_env_check, xdg_env_check, spack_home_env_check, cfg_check, spack_home_cfg_check]: + for check in [ + spack_env_check, + xdg_env_check, + spack_home_env_check, + cfg_check, + spack_home_cfg_check, + ]: possible_resolution = check() if possible_resolution: return possible_resolution @@ -216,4 +237,4 @@ def prefer_old_location(self, old_location, new_location): locations = SpackPaths(paths_base.locations) -spack_script = locations.spack_script \ No newline at end of file +spack_script = locations.spack_script diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py index fd0ee5b25b1402..064eab7ef4aa72 100644 --- a/lib/spack/spack/repo.py +++ b/lib/spack/spack/repo.py @@ -72,9 +72,7 @@ def package_repository_lock() -> spack.util.lock.Lock: """Lock for process safety when cloning remote package repositories""" - return spack.util.lock.Lock( - os.path.join(paths.user_cache_path, "package-repository.lock") - ) + return spack.util.lock.Lock(os.path.join(paths.user_cache_path, "package-repository.lock")) def is_package_module(fullname: str) -> bool: diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index 172620506053b4..62c086b3e1143e 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -204,9 +204,7 @@ def test_download_and_extract_artifacts(tmp_path: pathlib.Path, monkeypatch): url = "https://www.nosuchurlexists.itsfake/artifacts.zip" working_dir = tmp_path / "repro" - test_artifacts_path = os.path.join( - paths.test_path, "data", "ci", "gitlab", "artifacts.zip" - ) + test_artifacts_path = os.path.join(paths.test_path, "data", "ci", "gitlab", "artifacts.zip") def _urlopen_OK(*args, **kwargs): with open(test_artifacts_path, "rb") as f: diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 5dabbde8a56818..23b656deb1aa09 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -1094,9 +1094,7 @@ def test_ci_get_stack_changed(mock_git_repo, monkeypatch): """Test that we can detect the change to .gitlab-ci.yml in a mock spack git repo.""" monkeypatch.setattr(paths, "prefix", mock_git_repo) - fake_env_path = os.path.join( - paths.prefix, os.path.sep.join(("no", "such", "env", "path")) - ) + fake_env_path = os.path.join(paths.prefix, os.path.sep.join(("no", "such", "env", "path"))) assert ci.stack_changed(fake_env_path) is True diff --git a/lib/spack/spack/test/concretization/core.py b/lib/spack/spack/test/concretization/core.py index d531117127b249..2db0862c25569d 100644 --- a/lib/spack/spack/test/concretization/core.py +++ b/lib/spack/spack/test/concretization/core.py @@ -2492,9 +2492,7 @@ def test_select_lower_priority_package_from_repository_stack( from cli. """ # 'builtin_mock" and "duplicates_test" share a 'gmake' package - additional_repo = os.path.join( - paths.test_repos_path, "spack_repo", "duplicates_test" - ) + additional_repo = os.path.join(paths.test_repos_path, "spack_repo", "duplicates_test") with spack.repo.use_repositories(additional_repo, override=False): s = spack.concretize.concretize_one(spec_str) diff --git a/lib/spack/spack/test/util/remote_file_cache.py b/lib/spack/spack/test/util/remote_file_cache.py index 37032228bf37b2..78c3ac6d34aa1e 100644 --- a/lib/spack/spack/test/util/remote_file_cache.py +++ b/lib/spack/spack/test/util/remote_file_cache.py @@ -34,10 +34,7 @@ def test_rfc_local_path_bad_scheme(path, err): [ ("/a/b/c/d/e/config.py", "/a/b/c/d/e/config.py"), ("file:///this/is/a/file/url/include.yaml", "/this/is/a/file/url/include.yaml"), - ( - "relative/packages.txt", - os.path.join(paths.spack_root, "relative", "packages.txt"), - ), + ("relative/packages.txt", os.path.join(paths.spack_root, "relative", "packages.txt")), (r"C:\Files (x86)\Windows\10", r"C:\Files (x86)\Windows\10"), (r"D:/spack stage", "D:\\spack stage"), ], diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index eb3a3f276c7cf8..986cdf0b912a4b 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -75,7 +75,7 @@ def replacements(): "data_home": lambda: paths.spack_data_home, "cache_home": lambda: paths.spack_cache_home, "state_home": lambda: paths.spack_state_home, - #"spack_config_home": lambda: paths.spack_config_home, + # "spack_config_home": lambda: paths.spack_config_home, "spack_instance_id": lambda: paths.spack_instance_id, "architecture": lambda: arch, "arch": lambda: arch, From e002d7610a505814c4dd00a78efb0c1a022bb4e2 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 15 Dec 2025 11:27:38 -0800 Subject: [PATCH 195/506] auto isort --- lib/spack/spack/binary_distribution.py | 2 +- lib/spack/spack/bootstrap/config.py | 2 +- lib/spack/spack/ci/__init__.py | 2 +- lib/spack/spack/cmd/__init__.py | 2 +- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/cmd/commands.py | 2 +- lib/spack/spack/cmd/edit.py | 2 +- lib/spack/spack/cmd/gpg.py | 2 +- lib/spack/spack/cmd/install.py | 2 +- lib/spack/spack/cmd/location.py | 2 +- lib/spack/spack/cmd/make_installer.py | 2 +- lib/spack/spack/cmd/style.py | 2 +- lib/spack/spack/cmd/tutorial.py | 2 +- lib/spack/spack/cmd/unit_test.py | 2 +- lib/spack/spack/environment/depfile.py | 2 +- lib/spack/spack/environment/environment.py | 2 +- lib/spack/spack/hooks/sbang.py | 2 +- lib/spack/spack/install_test.py | 2 +- lib/spack/spack/modules/common.py | 2 +- lib/spack/spack/paths.py | 4 ++-- lib/spack/spack/paths_base.py | 2 +- lib/spack/spack/repo.py | 2 +- lib/spack/spack/reporters/cdash.py | 2 +- lib/spack/spack/spec.py | 2 +- lib/spack/spack/spec_parser.py | 2 +- lib/spack/spack/store.py | 2 +- lib/spack/spack/test/builder.py | 2 +- lib/spack/spack/test/cmd/blame.py | 2 +- lib/spack/spack/test/cmd/ci.py | 2 +- lib/spack/spack/test/cmd/clean.py | 2 +- lib/spack/spack/test/cmd/commands.py | 2 +- lib/spack/spack/test/cmd/diff.py | 2 +- lib/spack/spack/test/cmd/edit.py | 2 +- lib/spack/spack/test/cmd/env.py | 2 +- lib/spack/spack/test/cmd/find.py | 2 +- lib/spack/spack/test/cmd/license.py | 2 +- lib/spack/spack/test/cmd/list.py | 2 +- lib/spack/spack/test/cmd/location.py | 2 +- lib/spack/spack/test/cmd/pkg.py | 2 +- lib/spack/spack/test/cmd/style.py | 2 +- lib/spack/spack/test/cmd/test.py | 2 +- lib/spack/spack/test/concretization/compiler_runtimes.py | 2 +- lib/spack/spack/test/concretization/core.py | 2 +- lib/spack/spack/test/concretization/flag_mixing.py | 2 +- lib/spack/spack/test/concretization/requirements.py | 2 +- lib/spack/spack/test/config.py | 2 +- lib/spack/spack/test/conftest.py | 2 +- lib/spack/spack/test/database.py | 2 +- lib/spack/spack/test/directory_layout.py | 2 +- lib/spack/spack/test/link_paths.py | 2 +- lib/spack/spack/test/llnl/util/file_list.py | 2 +- lib/spack/spack/test/main.py | 2 +- lib/spack/spack/test/patch.py | 2 +- lib/spack/spack/test/repo.py | 2 +- lib/spack/spack/test/spec_semantics.py | 2 +- lib/spack/spack/test/spec_yaml.py | 2 +- lib/spack/spack/test/util/package_hash.py | 2 +- lib/spack/spack/test/util/path.py | 2 +- lib/spack/spack/test/util/remote_file_cache.py | 2 +- lib/spack/spack/test/web.py | 2 +- lib/spack/spack/util/gpg.py | 2 +- lib/spack/spack/version/git_ref_lookup.py | 2 +- 62 files changed, 63 insertions(+), 63 deletions(-) diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index 0170f59651934b..334e04ef228e7f 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -43,7 +43,6 @@ import spack.oci.image import spack.oci.oci import spack.oci.opener -from spack.paths import locations as paths import spack.platforms import spack.relocate as relocate import spack.spec @@ -78,6 +77,7 @@ upload_manifest_with_retry, ) from spack.package_prefs import get_package_dir_permissions, get_package_group +from spack.paths import locations as paths from spack.relocate_text import utf8_paths_to_single_binary_regex from spack.stage import Stage from spack.util.executable import which diff --git a/lib/spack/spack/bootstrap/config.py b/lib/spack/spack/bootstrap/config.py index 3fb59a6a1e1993..8638a753dfd000 100644 --- a/lib/spack/spack/bootstrap/config.py +++ b/lib/spack/spack/bootstrap/config.py @@ -11,12 +11,12 @@ import spack.config import spack.environment import spack.modules -from spack.paths import locations as paths import spack.platforms import spack.repo import spack.store import spack.util.path from spack.llnl.util import tty +from spack.paths import locations as paths #: Reference counter for the bootstrapping configuration context manager _REF_COUNT = 0 diff --git a/lib/spack/spack/ci/__init__.py b/lib/spack/spack/ci/__init__.py index 2c461716176541..83effc4fb3bb8e 100644 --- a/lib/spack/spack/ci/__init__.py +++ b/lib/spack/spack/ci/__init__.py @@ -27,7 +27,6 @@ import spack.llnl.util.tty as tty import spack.main import spack.mirrors.mirror -from spack.paths import locations as paths import spack.repo import spack.spec import spack.stage @@ -40,6 +39,7 @@ from spack import traverse from spack.error import SpackError from spack.llnl.util.tty.color import cescape, colorize +from spack.paths import locations as paths from spack.reporters.cdash import SPACK_CDASH_TIMEOUT from .common import ( diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 97bfc524a4da1e..85776c448b9d8d 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -3,7 +3,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse -import spack.paths import difflib import importlib import os @@ -21,6 +20,7 @@ import spack.extensions import spack.llnl.string import spack.llnl.util.tty as tty +import spack.paths import spack.paths_base import spack.repo import spack.spec diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 2e6d1f9cef63f5..b2cd436ee34041 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -11,11 +11,11 @@ import spack.config import spack.llnl.util.filesystem import spack.llnl.util.tty as tty -from spack.paths import locations as paths import spack.stage import spack.store import spack.util.path from spack.cmd.common import arguments +from spack.paths import locations as paths description = "remove temporary build files and/or downloaded archives" section = "build" diff --git a/lib/spack/spack/cmd/commands.py b/lib/spack/spack/cmd/commands.py index a51eee8726b184..7bae7d66a4ed98 100644 --- a/lib/spack/spack/cmd/commands.py +++ b/lib/spack/spack/cmd/commands.py @@ -15,11 +15,11 @@ import spack.config import spack.llnl.util.tty as tty import spack.main -from spack.paths import locations as paths import spack.platforms from spack.llnl.util.argparsewriter import ArgparseRstWriter, ArgparseWriter, Command from spack.llnl.util.tty.colify import colify from spack.main import section_descriptions +from spack.paths import locations as paths description = "list available spack commands" section = "config" diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index e26b58292e7e17..5ae74581129029 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -10,9 +10,9 @@ import spack.cmd import spack.llnl.util.tty as tty -from spack.paths import locations as paths import spack.repo import spack.util.editor +from spack.paths import locations as paths description = "open package files in ``$EDITOR``" section = "packaging" diff --git a/lib/spack/spack/cmd/gpg.py b/lib/spack/spack/cmd/gpg.py index 4aa8a8bd8b3528..ec10e1645cd88e 100644 --- a/lib/spack/spack/cmd/gpg.py +++ b/lib/spack/spack/cmd/gpg.py @@ -8,11 +8,11 @@ import spack.binary_distribution import spack.mirrors.mirror -from spack.paths import locations as paths import spack.stage import spack.util.gpg import spack.util.url from spack.cmd.common import arguments +from spack.paths import locations as paths description = "handle GPG actions for spack" section = "packaging" diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 19e8c641e2e895..8b55b0ec914e65 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -12,7 +12,6 @@ import spack.config import spack.environment as ev import spack.llnl.util.filesystem as fs -from spack.paths import locations as paths import spack.spec import spack.store from spack.cmd.common import arguments @@ -20,6 +19,7 @@ from spack.installer import InstallPolicy from spack.llnl.string import plural from spack.llnl.util import tty +from spack.paths import locations as paths description = "build and install packages" section = "build" diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 48f919e3b10a0c..7b495160397bf2 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -9,10 +9,10 @@ import spack.cmd import spack.environment as ev import spack.llnl.util.tty as tty -from spack.paths import locations as paths import spack.repo import spack.stage from spack.cmd.common import arguments +from spack.paths import locations as paths description = "print out locations of packages and spack directories" section = "query" diff --git a/lib/spack/spack/cmd/make_installer.py b/lib/spack/spack/cmd/make_installer.py index 1a523bd6d102b4..b3f526f813d6dd 100644 --- a/lib/spack/spack/cmd/make_installer.py +++ b/lib/spack/spack/cmd/make_installer.py @@ -7,9 +7,9 @@ import sys import spack.concretize -from spack.paths import locations as paths import spack.util.executable from spack.llnl.path import convert_to_posix_path +from spack.paths import locations as paths description = "generate Windows installer" section = "admin" diff --git a/lib/spack/spack/cmd/style.py b/lib/spack/spack/cmd/style.py index 4d9a59696ac5f2..dc795a71d4e862 100644 --- a/lib/spack/spack/cmd/style.py +++ b/lib/spack/spack/cmd/style.py @@ -12,11 +12,11 @@ import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color -from spack.paths import locations as paths import spack.repo import spack.util.git import spack.util.spack_yaml from spack.llnl.util.filesystem import working_dir +from spack.paths import locations as paths from spack.spec_parser import NAME, VERSION_LIST, SpecTokens from spack.tokenize import Token, TokenBase, Tokenizer from spack.util.executable import Executable, which diff --git a/lib/spack/spack/cmd/tutorial.py b/lib/spack/spack/cmd/tutorial.py index 1f0b528fbae054..dc93e9e9844b4f 100644 --- a/lib/spack/spack/cmd/tutorial.py +++ b/lib/spack/spack/cmd/tutorial.py @@ -10,11 +10,11 @@ import spack.cmd import spack.config import spack.llnl.util.tty as tty -from spack.paths import locations as paths import spack.util.git import spack.util.gpg from spack.cmd.common import arguments from spack.llnl.util.filesystem import working_dir +from spack.paths import locations as paths from spack.util.spack_yaml import syaml_dict description = "set up spack for our tutorial (WARNING: modifies config!)" diff --git a/lib/spack/spack/cmd/unit_test.py b/lib/spack/spack/cmd/unit_test.py index b7eb1e71ce85c2..0546d2e5ca9e43 100644 --- a/lib/spack/spack/cmd/unit_test.py +++ b/lib/spack/spack/cmd/unit_test.py @@ -19,8 +19,8 @@ import spack.llnl.util.filesystem import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color -from spack.paths import locations as paths from spack.llnl.util.tty.colify import colify +from spack.paths import locations as paths description = "run spack's unit tests (wrapper around pytest)" section = "developer" diff --git a/lib/spack/spack/environment/depfile.py b/lib/spack/spack/environment/depfile.py index 4f1be2bc1e595b..71c2dbf5147a80 100644 --- a/lib/spack/spack/environment/depfile.py +++ b/lib/spack/spack/environment/depfile.py @@ -14,9 +14,9 @@ import spack.deptypes as dt import spack.environment.environment as ev -from spack.paths import locations as paths import spack.spec import spack.traverse as traverse +from spack.paths import locations as paths class UseBuildCache(Enum): diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index f41e0f11100b0c..1cb95a0989d787 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -24,7 +24,6 @@ import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as clr -from spack.paths import locations as paths import spack.repo import spack.schema.env import spack.spec @@ -40,6 +39,7 @@ from spack import traverse from spack.llnl.util.filesystem import copy_tree, islink, readlink, symlink from spack.llnl.util.link_tree import ConflictingSpecsError +from spack.paths import locations as paths from spack.schema.env import TOP_LEVEL_KEY from spack.spec import Spec from spack.util.path import substitute_path_variables diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 6750ba7d88e8b7..655501e0be6c4d 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -14,9 +14,9 @@ import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.package_prefs -from spack.paths import locations as paths import spack.spec import spack.store +from spack.paths import locations as paths from spack.util.socket import _gethostname #: OS-imposed character limit for shebang line: 127 for Linux; 511 for Mac. diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py index 0f3f0c1be2c302..a54a68d9d2df7a 100644 --- a/lib/spack/spack/install_test.py +++ b/lib/spack/spack/install_test.py @@ -20,7 +20,6 @@ import spack.llnl.util.tty as tty import spack.llnl.util.tty.log import spack.package_base -from spack.paths import locations as paths import spack.repo import spack.report import spack.spec @@ -31,6 +30,7 @@ from spack.llnl.string import plural from spack.llnl.util.lang import nullcontext from spack.llnl.util.tty.color import colorize +from spack.paths import locations as paths from spack.spec import Spec from spack.util.prefix import Prefix diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py index afdc14a5cf43ff..c4ad62dd1a432f 100644 --- a/lib/spack/spack/modules/common.py +++ b/lib/spack/spack/modules/common.py @@ -42,7 +42,6 @@ import spack.error import spack.llnl.util.filesystem import spack.llnl.util.tty as tty -from spack.paths import locations as paths import spack.projections as proj import spack.schema import spack.schema.environment @@ -56,6 +55,7 @@ import spack.util.spack_yaml as syaml from spack.context import Context from spack.llnl.util.lang import Singleton, dedupe, memoized +from spack.paths import locations as paths #: config section for this file diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index b6a6d799809eff..04d144e02b20ef 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -15,9 +15,9 @@ from enum import Enum from pathlib import PurePath -import spack.util.hash as hash -import spack.paths_base as paths_base import spack.config as config +import spack.paths_base as paths_base +import spack.util.hash as hash class XDG_vars(Enum): diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index f2c85a93d51afd..18a72f4a44f31b 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -2,8 +2,8 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from pathlib import PurePath import os +from pathlib import PurePath import spack.llnl.util.filesystem diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py index 064eab7ef4aa72..1af155fef471c7 100644 --- a/lib/spack/spack/repo.py +++ b/lib/spack/spack/repo.py @@ -45,7 +45,6 @@ import spack.llnl.util.lang import spack.llnl.util.tty as tty import spack.patch -from spack.paths import locations as paths import spack.provider_index import spack.tag import spack.util.executable @@ -57,6 +56,7 @@ import spack.util.path import spack.util.spack_yaml as syaml from spack.llnl.util.filesystem import working_dir +from spack.paths import locations as paths if TYPE_CHECKING: import spack.package_base diff --git a/lib/spack/spack/reporters/cdash.py b/lib/spack/spack/reporters/cdash.py index dc979b045d1905..93fe194bc751ad 100644 --- a/lib/spack/spack/reporters/cdash.py +++ b/lib/spack/spack/reporters/cdash.py @@ -18,7 +18,6 @@ import spack import spack.llnl.util.tty as tty -from spack.paths import locations as paths import spack.platforms import spack.spec import spack.tengine @@ -26,6 +25,7 @@ import spack.util.web as web_util from spack.error import SpackError from spack.llnl.util.filesystem import working_dir +from spack.paths import locations as paths from spack.util.crypto import checksum from spack.util.log_parse import parse_log_events diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index ce5d7461ea7639..a91ec0fa892f5f 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -89,7 +89,6 @@ import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as clr import spack.patch -from spack.paths import locations as paths import spack.platforms import spack.provider_index import spack.repo @@ -102,6 +101,7 @@ import spack.variant as vt import spack.version as vn import spack.version.git_ref_lookup +from spack.paths import locations as paths from .enums import InstallRecordStatus, PropagationPolicy diff --git a/lib/spack/spack/spec_parser.py b/lib/spack/spack/spec_parser.py index 67d5ec488f0bd0..ae7da757635063 100644 --- a/lib/spack/spack/spec_parser.py +++ b/lib/spack/spack/spec_parser.py @@ -67,12 +67,12 @@ import spack.config import spack.deptypes import spack.error -from spack.paths import locations as paths import spack.util.spack_yaml import spack.version from spack.aliases import LEGACY_COMPILER_TO_BUILTIN from spack.enums import PropagationPolicy from spack.llnl.util.tty import color +from spack.paths import locations as paths from spack.tokenize import Token, TokenBase, Tokenizer if TYPE_CHECKING: diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 110b82e034e05a..c1baa5a377772a 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -27,10 +27,10 @@ import spack.directory_layout import spack.error import spack.llnl.util.lang -from spack.paths import locations as paths import spack.spec import spack.util.path from spack.llnl.util import tty +from spack.paths import locations as paths def parse_install_tree(config_dict: dict) -> Tuple[str, str, Dict[str, str]]: diff --git a/lib/spack/spack/test/builder.py b/lib/spack/spack/test/builder.py index 5d161c61d72c75..3419cd8ff86844 100644 --- a/lib/spack/spack/test/builder.py +++ b/lib/spack/spack/test/builder.py @@ -8,9 +8,9 @@ import spack.builder import spack.concretize -from spack.paths import locations as paths import spack.repo from spack.llnl.util.filesystem import touch +from spack.paths import locations as paths @pytest.fixture() diff --git a/lib/spack/spack/test/cmd/blame.py b/lib/spack/spack/test/cmd/blame.py index 4a56874d5a4b8d..ee75c96e9c54b5 100644 --- a/lib/spack/spack/test/cmd/blame.py +++ b/lib/spack/spack/test/cmd/blame.py @@ -9,11 +9,11 @@ import pytest import spack.cmd.blame -from spack.paths import locations as paths import spack.util.spack_json as sjson from spack.cmd.blame import ensure_full_history, git_prefix, package_repo_root from spack.llnl.util.filesystem import mkdirp, working_dir from spack.main import SpackCommand, SpackCommandError +from spack.paths import locations as paths from spack.repo import RepoDescriptors from spack.util.executable import ProcessError diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 23b656deb1aa09..4dc05bf6e83661 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -19,7 +19,6 @@ import spack.environment as ev import spack.hash_types as ht import spack.main -from spack.paths import locations as paths import spack.repo import spack.spec import spack.stage @@ -32,6 +31,7 @@ from spack.cmd.ci import FAILED_CREATE_BUILDCACHE_CODE from spack.error import SpackError from spack.llnl.util.filesystem import mkdirp, working_dir +from spack.paths import locations as paths from spack.schema.database_index import schema as db_idx_schema from spack.test.conftest import MockHTTPResponse, RepoBuilder diff --git a/lib/spack/spack/test/cmd/clean.py b/lib/spack/spack/test/cmd/clean.py index 4606f6a27298be..53d986a848d38b 100644 --- a/lib/spack/spack/test/cmd/clean.py +++ b/lib/spack/spack/test/cmd/clean.py @@ -12,9 +12,9 @@ import spack.llnl.util.filesystem as fs import spack.main import spack.package_base -from spack.paths import locations as paths import spack.stage import spack.store +from spack.paths import locations as paths clean = spack.main.SpackCommand("clean") diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index dfbe546bcab725..d6ba50d13a2bce 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -15,8 +15,8 @@ import spack.cmd.commands import spack.config import spack.main -from spack.paths import locations as paths from spack.cmd.commands import _dest_to_fish_complete, _positional_to_subroutine +from spack.paths import locations as paths from spack.util.executable import Executable diff --git a/lib/spack/spack/test/cmd/diff.py b/lib/spack/spack/test/cmd/diff.py index 98e3a8dc840476..2456ed327bd46c 100644 --- a/lib/spack/spack/test/cmd/diff.py +++ b/lib/spack/spack/test/cmd/diff.py @@ -9,10 +9,10 @@ import spack.cmd.diff import spack.concretize import spack.main -from spack.paths import locations as paths import spack.repo import spack.util.spack_json as sjson import spack.version +from spack.paths import locations as paths install_cmd = spack.main.SpackCommand("install") diff_cmd = spack.main.SpackCommand("diff") diff --git a/lib/spack/spack/test/cmd/edit.py b/lib/spack/spack/test/cmd/edit.py index 02f68bdf9a33e7..a5b30f3df0babe 100644 --- a/lib/spack/spack/test/cmd/edit.py +++ b/lib/spack/spack/test/cmd/edit.py @@ -5,10 +5,10 @@ import os import pathlib -from spack.paths import locations as paths import spack.repo import spack.util.editor from spack.main import SpackCommand +from spack.paths import locations as paths edit = SpackCommand("edit") diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index d5928858ce7ddc..71bac2832faafd 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -28,7 +28,6 @@ import spack.modules import spack.modules.tcl import spack.package_base -from spack.paths import locations as paths import spack.repo import spack.solver.asp import spack.spec @@ -42,6 +41,7 @@ from spack.installer import PackageInstaller from spack.llnl.util.filesystem import readlink from spack.main import SpackCommand, SpackCommandError +from spack.paths import locations as paths from spack.spec import Spec from spack.stage import stage_prefix from spack.test.conftest import RepoBuilder diff --git a/lib/spack/spack/test/cmd/find.py b/lib/spack/spack/test/cmd/find.py index 5fe00130df7fa3..57a333229af46d 100644 --- a/lib/spack/spack/test/cmd/find.py +++ b/lib/spack/spack/test/cmd/find.py @@ -14,13 +14,13 @@ import spack.concretize import spack.environment as ev import spack.package_base -from spack.paths import locations as paths import spack.repo import spack.store import spack.user_environment as uenv from spack.enums import InstallRecordStatus from spack.llnl.util.filesystem import working_dir from spack.main import SpackCommand +from spack.paths import locations as paths from spack.test.utilities import SpackCommandArgs from spack.util.pattern import Bunch diff --git a/lib/spack/spack/test/cmd/license.py b/lib/spack/spack/test/cmd/license.py index 42d47d2f98e75e..8d1ee210f4433c 100644 --- a/lib/spack/spack/test/cmd/license.py +++ b/lib/spack/spack/test/cmd/license.py @@ -8,9 +8,9 @@ import pytest -from spack.paths import locations as paths from spack.llnl.util.filesystem import mkdirp, touch from spack.main import SpackCommand +from spack.paths import locations as paths license = SpackCommand("license") diff --git a/lib/spack/spack/test/cmd/list.py b/lib/spack/spack/test/cmd/list.py index a4f91f8062e77c..1bcf2c231426c1 100644 --- a/lib/spack/spack/test/cmd/list.py +++ b/lib/spack/spack/test/cmd/list.py @@ -7,9 +7,9 @@ import pytest -from spack.paths import locations as paths import spack.repo from spack.main import SpackCommand +from spack.paths import locations as paths pytestmark = [pytest.mark.usefixtures("mock_packages")] diff --git a/lib/spack/spack/test/cmd/location.py b/lib/spack/spack/test/cmd/location.py index c5206a5b63a9b8..987b8d6202b203 100644 --- a/lib/spack/spack/test/cmd/location.py +++ b/lib/spack/spack/test/cmd/location.py @@ -10,11 +10,11 @@ import spack.concretize import spack.environment as ev import spack.main -from spack.paths import locations as paths import spack.repo import spack.stage from spack.llnl.util.filesystem import mkdirp from spack.main import SpackCommand +from spack.paths import locations as paths # Everything here uses (or can use) the mock config and database. pytestmark = [pytest.mark.usefixtures("mutable_config", "mutable_database")] diff --git a/lib/spack/spack/test/cmd/pkg.py b/lib/spack/spack/test/cmd/pkg.py index 1bf85f003cef33..f4a39f2b51477c 100644 --- a/lib/spack/spack/test/cmd/pkg.py +++ b/lib/spack/spack/test/cmd/pkg.py @@ -10,11 +10,11 @@ import spack.cmd import spack.cmd.pkg import spack.main -from spack.paths import locations as paths import spack.repo import spack.util.executable import spack.util.file_cache from spack.llnl.util.filesystem import mkdirp, working_dir +from spack.paths import locations as paths pkg = spack.main.SpackCommand("pkg") diff --git a/lib/spack/spack/test/cmd/style.py b/lib/spack/spack/test/cmd/style.py index 178c1e3099e0b7..949d0f5561d402 100644 --- a/lib/spack/spack/test/cmd/style.py +++ b/lib/spack/spack/test/cmd/style.py @@ -13,10 +13,10 @@ import spack.cmd.style import spack.main -from spack.paths import locations as paths import spack.repo from spack.cmd.style import _run_import_check, changed_files from spack.llnl.util.filesystem import FileFilter, working_dir +from spack.paths import locations as paths from spack.util.executable import which #: directory with sample style files diff --git a/lib/spack/spack/test/cmd/test.py b/lib/spack/spack/test/cmd/test.py index 7fff1cc7ad7283..a88a844bac5310 100644 --- a/lib/spack/spack/test/cmd/test.py +++ b/lib/spack/spack/test/cmd/test.py @@ -14,10 +14,10 @@ import spack.config import spack.install_test import spack.main -from spack.paths import locations as paths from spack.install_test import TestStatus from spack.llnl.util.filesystem import copy_tree, working_dir from spack.main import SpackCommand +from spack.paths import locations as paths install = SpackCommand("install") spack_test = SpackCommand("test") diff --git a/lib/spack/spack/test/concretization/compiler_runtimes.py b/lib/spack/spack/test/concretization/compiler_runtimes.py index ec1b4b73fee7ed..d85fca819a2724 100644 --- a/lib/spack/spack/test/concretization/compiler_runtimes.py +++ b/lib/spack/spack/test/concretization/compiler_runtimes.py @@ -11,11 +11,11 @@ import spack.concretize import spack.config -from spack.paths import locations as paths import spack.repo import spack.solver.asp import spack.spec from spack.environment.environment import ViewDescriptor +from spack.paths import locations as paths from spack.solver.reuse import SpecFilter from spack.version import Version diff --git a/lib/spack/spack/test/concretization/core.py b/lib/spack/spack/test/concretization/core.py index 2db0862c25569d..dd478c6eceb774 100644 --- a/lib/spack/spack/test/concretization/core.py +++ b/lib/spack/spack/test/concretization/core.py @@ -27,7 +27,6 @@ import spack.hash_types as ht import spack.llnl.util.lang import spack.package_base -from spack.paths import locations as paths import spack.platforms import spack.platforms.test import spack.repo @@ -43,6 +42,7 @@ import spack.variant as vt from spack.externals import ExternalDependencyError from spack.installer import PackageInstaller +from spack.paths import locations as paths from spack.solver.reuse import SpecFilter from spack.spec import Spec from spack.test.conftest import RepoBuilder diff --git a/lib/spack/spack/test/concretization/flag_mixing.py b/lib/spack/spack/test/concretization/flag_mixing.py index 18993926afb912..e5b2433b98f0e9 100644 --- a/lib/spack/spack/test/concretization/flag_mixing.py +++ b/lib/spack/spack/test/concretization/flag_mixing.py @@ -38,10 +38,10 @@ import spack.concretize import spack.config import spack.environment as ev -from spack.paths import locations as paths import spack.repo import spack.spec import spack.util.spack_yaml as syaml +from spack.paths import locations as paths @pytest.fixture diff --git a/lib/spack/spack/test/concretization/requirements.py b/lib/spack/spack/test/concretization/requirements.py index 18835d87890b92..800a58935ecf90 100644 --- a/lib/spack/spack/test/concretization/requirements.py +++ b/lib/spack/spack/test/concretization/requirements.py @@ -10,7 +10,6 @@ import spack.error import spack.installer import spack.package_base -from spack.paths import locations as paths import spack.platforms import spack.repo import spack.solver.asp @@ -19,6 +18,7 @@ import spack.util.spack_yaml as syaml import spack.version from spack.installer import PackageInstaller +from spack.paths import locations as paths from spack.solver.asp import InternalConcretizerError, UnsatisfiableSpecError from spack.solver.reuse import SpecFilter from spack.spec import Spec diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 0dc4a1fc96c72f..281fc55e9469d5 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -20,7 +20,6 @@ import spack.error import spack.llnl.util.filesystem as fs import spack.package_base -from spack.paths import locations as paths import spack.platforms import spack.repo import spack.schema.compilers @@ -37,6 +36,7 @@ import spack.util.spack_yaml as syaml from spack.enums import ConfigScopePriority from spack.llnl.util.filesystem import join_path, touch +from spack.paths import locations as paths from spack.util.spack_yaml import DictWithLineInfo # sample config data diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index e9026b3a6255f8..2d409044aa1d9a 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -45,7 +45,6 @@ import spack.modules.common import spack.package_base import spack.paths -from spack.paths import locations as paths import spack.platforms import spack.repo import spack.solver.asp @@ -78,6 +77,7 @@ working_dir, ) from spack.main import SpackCommand +from spack.paths import locations as paths from spack.util.pattern import Bunch from spack.util.remote_file_cache import raw_github_gitlab_url diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index f894139809d7c9..35e1afc32e261c 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -34,7 +34,6 @@ import spack.llnl.util.filesystem as fs import spack.llnl.util.lock as lk import spack.package_base -from spack.paths import locations as paths import spack.repo import spack.spec import spack.store @@ -43,6 +42,7 @@ from spack.enums import InstallRecordStatus from spack.installer import PackageInstaller from spack.llnl.util.tty.colify import colify +from spack.paths import locations as paths from spack.schema.database_index import schema from spack.test.conftest import RepoBuilder from spack.util.executable import Executable diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 266c7a1ab98997..2c1578e928ecb0 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -13,12 +13,12 @@ import spack.concretize import spack.hash_types -from spack.paths import locations as paths import spack.repo import spack.spec import spack.util.file_cache from spack.directory_layout import DirectoryLayout, InvalidDirectoryLayoutParametersError from spack.llnl.path import path_to_os_path +from spack.paths import locations as paths from spack.spec import Spec # number of packages to test (to reduce test time) diff --git a/lib/spack/spack/test/link_paths.py b/lib/spack/spack/test/link_paths.py index 05c0f8d63b9ef2..aff8326abc1a82 100644 --- a/lib/spack/spack/test/link_paths.py +++ b/lib/spack/spack/test/link_paths.py @@ -8,8 +8,8 @@ import pytest import spack.compilers.libraries -from spack.paths import locations as paths from spack.compilers.libraries import parse_non_system_link_dirs +from spack.paths import locations as paths drive = "" if sys.platform == "win32": diff --git a/lib/spack/spack/test/llnl/util/file_list.py b/lib/spack/spack/test/llnl/util/file_list.py index 4c18dae06b1625..5e06aa98871c83 100644 --- a/lib/spack/spack/test/llnl/util/file_list.py +++ b/lib/spack/spack/test/llnl/util/file_list.py @@ -8,8 +8,8 @@ import pytest -from spack.paths import locations as paths from spack.llnl.util.filesystem import HeaderList, LibraryList, find_headers, find_libraries +from spack.paths import locations as paths @pytest.fixture() diff --git a/lib/spack/spack/test/main.py b/lib/spack/spack/test/main.py index 1a7948b8039f5f..b8351e83fe280a 100644 --- a/lib/spack/spack/test/main.py +++ b/lib/spack/spack/test/main.py @@ -15,11 +15,11 @@ import spack.error import spack.llnl.util.filesystem as fs import spack.main -from spack.paths import locations as paths import spack.platforms import spack.util.executable as exe import spack.util.git import spack.util.spack_yaml as syaml +from spack.paths import locations as paths pytestmark = pytest.mark.not_on_windows( "Test functionality supported but tests are failing on Win" diff --git a/lib/spack/spack/test/patch.py b/lib/spack/spack/test/patch.py index fb0631fd5d9f9b..bc5b8a85365453 100644 --- a/lib/spack/spack/test/patch.py +++ b/lib/spack/spack/test/patch.py @@ -16,12 +16,12 @@ import spack.error import spack.fetch_strategy import spack.patch -from spack.paths import locations as paths import spack.repo import spack.spec import spack.stage import spack.util.url as url_util from spack.llnl.util.filesystem import mkdirp, touch, working_dir +from spack.paths import locations as paths from spack.spec import Spec from spack.stage import Stage from spack.util.executable import Executable diff --git a/lib/spack/spack/test/repo.py b/lib/spack/spack/test/repo.py index e386d540a650ed..c3cd793e9dd680 100644 --- a/lib/spack/spack/test/repo.py +++ b/lib/spack/spack/test/repo.py @@ -8,7 +8,6 @@ import spack.environment import spack.package_base -from spack.paths import locations as paths import spack.repo import spack.schema.repos import spack.spec @@ -16,6 +15,7 @@ import spack.util.file_cache import spack.util.lock import spack.util.naming +from spack.paths import locations as paths from spack.test.conftest import RepoBuilder from spack.util.naming import valid_module_name diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 568a45d9abc3b1..0357a9b43ac036 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -11,7 +11,6 @@ import spack.directives import spack.error import spack.llnl.util.lang -from spack.paths import locations as paths import spack.solver.asp import spack.spec import spack.spec_parser @@ -20,6 +19,7 @@ import spack.version as vn from spack.enums import PropagationPolicy from spack.error import SpecError, UnsatisfiableSpecError +from spack.paths import locations as paths from spack.spec import ArchSpec, DependencySpec, Spec, SpecFormatSigilError, SpecFormatStringError from spack.variant import ( InvalidVariantValueError, diff --git a/lib/spack/spack/test/spec_yaml.py b/lib/spack/spack/test/spec_yaml.py index 4b7640cc86922a..587edef60a8821 100644 --- a/lib/spack/spack/test/spec_yaml.py +++ b/lib/spack/spack/test/spec_yaml.py @@ -23,12 +23,12 @@ import spack.concretize import spack.config import spack.hash_types as ht -from spack.paths import locations as paths import spack.repo import spack.spec import spack.test.conftest import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml +from spack.paths import locations as paths from spack.spec import Spec, save_dependency_specfiles from spack.test.conftest import RepoBuilder from spack.util.spack_yaml import SpackYAMLError, syaml_dict diff --git a/lib/spack/spack/test/util/package_hash.py b/lib/spack/spack/test/util/package_hash.py index 208957793dc742..911136939b3526 100644 --- a/lib/spack/spack/test/util/package_hash.py +++ b/lib/spack/spack/test/util/package_hash.py @@ -9,9 +9,9 @@ import spack.concretize import spack.directives_meta -from spack.paths import locations as paths import spack.repo import spack.util.package_hash as ph +from spack.paths import locations as paths from spack.spec import Spec from spack.util.unparse import unparse diff --git a/lib/spack/spack/test/util/path.py b/lib/spack/spack/test/util/path.py index fa82dbff6efb6b..ce8b72c638f6ed 100644 --- a/lib/spack/spack/test/util/path.py +++ b/lib/spack/spack/test/util/path.py @@ -9,8 +9,8 @@ import spack.config import spack.llnl.util.tty as tty -from spack.paths import locations as paths import spack.util.path as sup +from spack.paths import locations as paths #: Some lines with lots of placeholders padded_lines = [ diff --git a/lib/spack/spack/test/util/remote_file_cache.py b/lib/spack/spack/test/util/remote_file_cache.py index 78c3ac6d34aa1e..9846cea5dc1691 100644 --- a/lib/spack/spack/test/util/remote_file_cache.py +++ b/lib/spack/spack/test/util/remote_file_cache.py @@ -9,9 +9,9 @@ import spack.config import spack.llnl.util.tty as tty -from spack.paths import locations as paths import spack.util.remote_file_cache as rfc_util from spack.llnl.util.filesystem import join_path +from spack.paths import locations as paths github_url = "https://github.com/fake/fake/{0}/develop" gitlab_url = "https://gitlab.fake.io/user/repo/-/blob/config/defaults" diff --git a/lib/spack/spack/test/web.py b/lib/spack/spack/test/web.py index bbd836b630ccc9..c257153fa61f88 100644 --- a/lib/spack/spack/test/web.py +++ b/lib/spack/spack/test/web.py @@ -15,12 +15,12 @@ import spack.config import spack.llnl.util.tty as tty import spack.mirrors.mirror -from spack.paths import locations as paths import spack.url import spack.util.s3 import spack.util.url as url_util import spack.util.web from spack.llnl.util.filesystem import working_dir +from spack.paths import locations as paths from spack.version import Version diff --git a/lib/spack/spack/util/gpg.py b/lib/spack/spack/util/gpg.py index 68bffb4d79c778..90e832797f6cbe 100644 --- a/lib/spack/spack/util/gpg.py +++ b/lib/spack/spack/util/gpg.py @@ -10,9 +10,9 @@ import spack.error import spack.llnl.util.filesystem -from spack.paths import locations as paths import spack.util.executable import spack.version +from spack.paths import locations as paths #: Executable instance for "gpg", initialized lazily GPG = None diff --git a/lib/spack/spack/version/git_ref_lookup.py b/lib/spack/spack/version/git_ref_lookup.py index 82dbb2a0821e22..146e4ce92c892d 100644 --- a/lib/spack/spack/version/git_ref_lookup.py +++ b/lib/spack/spack/version/git_ref_lookup.py @@ -9,12 +9,12 @@ import spack.caches import spack.fetch_strategy -from spack.paths import locations as paths import spack.repo import spack.util.executable import spack.util.hash import spack.util.spack_json as sjson from spack.llnl.util.filesystem import mkdirp, working_dir +from spack.paths import locations as paths from .common import VersionLookupError from .lookup import AbstractRefLookup From 09faf4aaa5ceecc491505e6a3f957725c4d74aee Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 15 Dec 2025 14:01:30 -0800 Subject: [PATCH 196/506] partial fixes --- lib/spack/spack/test/cmd/buildcache.py | 4 ++-- lib/spack/spack/test/cmd/gpg.py | 10 +++++----- lib/spack/spack/test/llnl/util/filesystem.py | 8 ++++---- lib/spack/spack/test/packaging.py | 4 ++-- lib/spack/spack/test/util/compression.py | 4 ++-- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/lib/spack/spack/test/cmd/buildcache.py b/lib/spack/spack/test/cmd/buildcache.py index 744006296bd8ae..75022e21096081 100644 --- a/lib/spack/spack/test/cmd/buildcache.py +++ b/lib/spack/spack/test/cmd/buildcache.py @@ -27,7 +27,7 @@ import spack.util.web as web_util from spack.installer import PackageInstaller from spack.llnl.util.filesystem import copy_tree, find -from spack.paths import test_path +from spack.paths import locations as paths from spack.url_buildcache import ( BuildcacheComponent, URLBuildcacheEntry, @@ -538,7 +538,7 @@ def test_push_without_build_deps( @pytest.fixture(scope="function") def v2_buildcache_layout(tmp_path: pathlib.Path): def _layout(signedness: str = "signed"): - source_path = str(pathlib.Path(test_path) / "data" / "mirrors" / "v2_layout" / signedness) + source_path = str(pathlib.Path(paths.test_path) / "data" / "mirrors" / "v2_layout" / signedness) test_mirror_path = tmp_path / "mirror" copy_tree(source_path, test_mirror_path) return test_mirror_path diff --git a/lib/spack/spack/test/cmd/gpg.py b/lib/spack/spack/test/cmd/gpg.py index 339bedd614b1c0..1c863748e07e98 100644 --- a/lib/spack/spack/test/cmd/gpg.py +++ b/lib/spack/spack/test/cmd/gpg.py @@ -12,7 +12,7 @@ import spack.util.executable import spack.util.gpg from spack.main import SpackCommand -from spack.paths import mock_gpg_data_path, mock_gpg_keys_path +from spack.paths import locations as paths from spack.util.executable import ProcessError #: spack command used by tests below @@ -63,10 +63,10 @@ def test_no_gpg_in_path(tmp_path: pathlib.Path, mock_gnupghome, monkeypatch, mut def test_gpg(tmp_path: pathlib.Path, mutable_config, mock_gnupghome): # Verify a file with an empty keyring. with pytest.raises(ProcessError): - gpg("verify", os.path.join(mock_gpg_data_path, "content.txt")) + gpg("verify", os.path.join(paths.mock_gpg_data_path, "content.txt")) # Import the default key. - gpg("init", "--from", mock_gpg_keys_path) + gpg("init", "--from", paths.mock_gpg_keys_path) # List the keys. # TODO: Test the output here. @@ -74,14 +74,14 @@ def test_gpg(tmp_path: pathlib.Path, mutable_config, mock_gnupghome): gpg("list", "--signing") # Verify the file now that the key has been trusted. - gpg("verify", os.path.join(mock_gpg_data_path, "content.txt")) + gpg("verify", os.path.join(paths.mock_gpg_data_path, "content.txt")) # Untrust the default key. gpg("untrust", "Spack testing") # Now that the key is untrusted, verification should fail. with pytest.raises(ProcessError): - gpg("verify", os.path.join(mock_gpg_data_path, "content.txt")) + gpg("verify", os.path.join(paths.mock_gpg_data_path, "content.txt")) # Create a file to test signing. test_path = tmp_path / "to-sign.txt" diff --git a/lib/spack/spack/test/llnl/util/filesystem.py b/lib/spack/spack/test/llnl/util/filesystem.py index 96a2211ec1378b..34c0428a74f2ad 100644 --- a/lib/spack/spack/test/llnl/util/filesystem.py +++ b/lib/spack/spack/test/llnl/util/filesystem.py @@ -14,7 +14,7 @@ import pytest import spack.llnl.util.filesystem as fs -from spack.paths import locations as paths +from spack.paths import locations as spack_paths @pytest.fixture() @@ -504,7 +504,7 @@ def test_filter_files_with_different_encodings( # All files given as input to this test must satisfy the pre-requisite # that the 'replacement' string is not present in the file initially and # that there's at least one match for the regex - original_file = os.path.join(paths.test_path, "data", "filter_file", filename) + original_file = os.path.join(spack_paths.test_path, "data", "filter_file", filename) target_file = os.path.join(str(tmp_path), filename) shutil.copy(original_file, target_file) # This should not raise exceptions @@ -552,7 +552,7 @@ def test_filter_files_multiple(tmp_path: pathlib.Path): # All files given as input to this test must satisfy the pre-requisite # that the 'replacement' string is not present in the file initially and # that there's at least one match for the regex - original_file = os.path.join(paths.test_path, "data", "filter_file", "x86_cpuid_info.c") + original_file = os.path.join(spack_paths.test_path, "data", "filter_file", "x86_cpuid_info.c") target_file = os.path.join(str(tmp_path), "x86_cpuid_info.c") shutil.copy(original_file, target_file) # This should not raise exceptions @@ -567,7 +567,7 @@ def test_filter_files_multiple(tmp_path: pathlib.Path): def test_filter_files_start_stop(tmp_path: pathlib.Path): - original_file = os.path.join(paths.test_path, "data", "filter_file", "start_stop.txt") + original_file = os.path.join(spack_paths.test_path, "data", "filter_file", "start_stop.txt") target_file = os.path.join(str(tmp_path), "start_stop.txt") shutil.copy(original_file, target_file) # None of the following should happen: diff --git a/lib/spack/spack/test/packaging.py b/lib/spack/spack/test/packaging.py index d38dda9ea80106..ccf13406ffcbee 100644 --- a/lib/spack/spack/test/packaging.py +++ b/lib/spack/spack/test/packaging.py @@ -30,7 +30,7 @@ from spack.installer import PackageInstaller from spack.llnl.util import filesystem as fs from spack.llnl.util.filesystem import readlink, symlink -from spack.paths import mock_gpg_keys_path +from spack.paths import locations as paths from spack.relocate import _macho_find_paths, relocate_links, relocate_text pytestmark = pytest.mark.not_on_windows("does not run on windows") @@ -109,7 +109,7 @@ def test_buildcache(mock_archive, tmp_path: pathlib.Path, monkeypatch, mutable_c buildcache.buildcache(parser, args) # Copy a key to the mirror to have something to download - shutil.copyfile(mock_gpg_keys_path + "/external.key", mirror_path + "/external.key") + shutil.copyfile(paths.mock_gpg_keys_path + "/external.key", mirror_path + "/external.key") args = parser.parse_args(["keys"]) buildcache.buildcache(parser, args) diff --git a/lib/spack/spack/test/util/compression.py b/lib/spack/spack/test/util/compression.py index 020addce8bf3f8..6f8ed53e5768e3 100644 --- a/lib/spack/spack/test/util/compression.py +++ b/lib/spack/spack/test/util/compression.py @@ -13,11 +13,11 @@ import spack.llnl.url from spack.llnl.util.filesystem import working_dir -from spack.paths import spack_root +from spack.paths import locations as paths from spack.util import compression from spack.util.executable import CommandNotFoundError -datadir = os.path.join(spack_root, "lib", "spack", "spack", "test", "data", "compression") +datadir = os.path.join(paths.spack_root, "lib", "spack", "spack", "test", "data", "compression") ext_archive = { ext: f"Foo.{ext}" for ext in spack.llnl.url.ALLOWED_ARCHIVE_TYPES if "TAR" not in ext From 953bd0a338ea6475811c6246d633c28e39b9735b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 15 Dec 2025 14:28:08 -0800 Subject: [PATCH 197/506] name shadowing --- lib/spack/spack/test/config.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 281fc55e9469d5..3922130821a9c2 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -36,7 +36,7 @@ import spack.util.spack_yaml as syaml from spack.enums import ConfigScopePriority from spack.llnl.util.filesystem import join_path, touch -from spack.paths import locations as paths +from spack.paths import locations as spack_paths from spack.util.spack_yaml import DictWithLineInfo # sample config data @@ -340,13 +340,13 @@ def __init__(self, path): def test_substitute_config_variables(mock_low_high_config, monkeypatch): - prefix = paths.prefix.lstrip("/") + prefix = spack_paths.prefix.lstrip("/") assert cross_plat_join( os.sep + os.path.join("foo", "bar", "baz"), prefix ) == spack_path.canonicalize_path("/foo/bar/baz/$spack") assert cross_plat_join( - paths.prefix, os.path.join("foo", "bar", "baz") + spack_paths.prefix, os.path.join("foo", "bar", "baz") ) == spack_path.canonicalize_path("$spack/foo/bar/baz/") assert cross_plat_join( @@ -358,7 +358,7 @@ def test_substitute_config_variables(mock_low_high_config, monkeypatch): ) == spack_path.canonicalize_path("/foo/bar/baz/${spack}") assert cross_plat_join( - paths.prefix, os.path.join("foo", "bar", "baz") + spack_paths.prefix, os.path.join("foo", "bar", "baz") ) == spack_path.canonicalize_path("${spack}/foo/bar/baz/") assert cross_plat_join( @@ -446,7 +446,7 @@ def test_substitute_user(mock_low_high_config): def test_substitute_user_cache(mock_low_high_config): - user_cache_path = paths.user_cache_path + user_cache_path = spack_paths.user_cache_path assert user_cache_path + os.sep + "baz" == spack_path.canonicalize_path( os.path.join("$user_cache_path", "baz") ) @@ -1149,7 +1149,7 @@ def test_bad_path_double_override(config): def test_license_dir_config(mutable_config, mock_packages): """Ensure license directory is customizable""" - expected_dir = paths.default_license_dir + expected_dir = spack_paths.default_license_dir assert spack.config.get("config:license_dir") == expected_dir assert spack.package_base.PackageBase.global_license_dir == expected_dir assert spack.repo.PATH.get_pkg_class("pkg-a").global_license_dir == expected_dir @@ -1720,7 +1720,7 @@ def test_included_path_git_unsat( def test_included_path_git( tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, key, value, capfd ): - monkeypatch.setattr(paths, "user_cache_path", str(tmp_path)) + monkeypatch.setattr(spack_paths, "user_cache_path", str(tmp_path)) class MockIncludeGit(spack.util.executable.Executable): def __init__(self, required: bool): @@ -1788,7 +1788,7 @@ def _checkout(*args, **kwargs): def test_included_path_git_errs(tmp_path: pathlib.Path, mock_low_high_config, monkeypatch): - monkeypatch.setattr(paths, "user_cache_path", str(tmp_path)) + monkeypatch.setattr(spack_paths, "user_cache_path", str(tmp_path)) paths = ["concretizer.yaml"] entry = { From 3baee0f107375bdd717350881d2c00830006512c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 15 Dec 2025 14:43:01 -0800 Subject: [PATCH 198/506] guard test patching of paths attributes --- lib/spack/spack/test/ci.py | 11 ++++++----- lib/spack/spack/test/cmd/blame.py | 4 ++-- lib/spack/spack/test/cmd/ci.py | 4 ++-- lib/spack/spack/test/cmd/clean.py | 6 +++--- lib/spack/spack/test/config.py | 9 +++++---- lib/spack/spack/test/conftest.py | 22 +++++++++++++++++----- lib/spack/spack/test/main.py | 4 ++-- 7 files changed, 37 insertions(+), 23 deletions(-) diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index 62c086b3e1143e..9a753de096c161 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -14,7 +14,7 @@ import spack.environment as ev import spack.error import spack.llnl.util.filesystem as fs -import spack.paths_base +import spack.paths import spack.repo as repo import spack.util.git from spack.spec import Spec @@ -260,15 +260,16 @@ def test_ci_copy_test_logs_to_artifacts_fail(tmp_path: pathlib.Path, capfd): def test_setup_spack_repro_version( - tmp_path: pathlib.Path, capfd, last_two_git_commits, monkeypatch + tmp_path: pathlib.Path, capfd, last_two_git_commits, monkeypatch, + override_path ): c1, c2 = last_two_git_commits repro_dir = tmp_path / "repro" spack_dir = repro_dir / "spack" spack_dir.mkdir(parents=True) - prefix_save = spack.paths_base.prefix - monkeypatch.setattr(spack.paths_base, "prefix", "/garbage") + prefix_save = spack.paths.locations.prefix + override_path("prefix", "/garbage") ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1) _, err = capfd.readouterr() @@ -276,7 +277,7 @@ def test_setup_spack_repro_version( assert not ret assert "Unable to find the path" in err - monkeypatch.setattr(spack.paths_base, "prefix", prefix_save) + override_path("prefix", prefix_save) monkeypatch.setattr(spack.util.git, "git", lambda: None) ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1) diff --git a/lib/spack/spack/test/cmd/blame.py b/lib/spack/spack/test/cmd/blame.py index ee75c96e9c54b5..df7c24b43ff3f2 100644 --- a/lib/spack/spack/test/cmd/blame.py +++ b/lib/spack/spack/test/cmd/blame.py @@ -70,10 +70,10 @@ def test_blame_file_outside_spack_repo(tmp_path: Path): assert "not within a spack repo" in out -def test_blame_spack_not_git_clone(monkeypatch): +def test_blame_spack_not_git_clone(override_path): """Ensure attempt to get blame when spack not a git clone fails.""" non_git_dir = os.path.join(paths.prefix, "..") - monkeypatch.setattr(paths, "prefix", non_git_dir) + override_path("prefix", non_git_dir) with pytest.raises(SpackCommandError): out = blame(".") diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 4dc05bf6e83661..4175870ca52e43 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -1090,10 +1090,10 @@ def test_ci_rebuild_index( assert concrete_spec.dag_hash() + " callpath" in output -def test_ci_get_stack_changed(mock_git_repo, monkeypatch): +def test_ci_get_stack_changed(mock_git_repo, monkeypatch, override_path): """Test that we can detect the change to .gitlab-ci.yml in a mock spack git repo.""" - monkeypatch.setattr(paths, "prefix", mock_git_repo) + override_path("prefix", mock_git_repo) fake_env_path = os.path.join(paths.prefix, os.path.sep.join(("no", "such", "env", "path"))) assert ci.stack_changed(fake_env_path) is True diff --git a/lib/spack/spack/test/cmd/clean.py b/lib/spack/spack/test/cmd/clean.py index 53d986a848d38b..0ce973f118a81f 100644 --- a/lib/spack/spack/test/cmd/clean.py +++ b/lib/spack/spack/test/cmd/clean.py @@ -72,7 +72,7 @@ def test_function_calls(command_line, effects, mock_calls_for_clean, mutable_con assert mock_calls_for_clean[name] == (1 if name in effects else 0) -def test_remove_python_cache(tmp_path: pathlib.Path, monkeypatch): +def test_remove_python_cache(tmp_path: pathlib.Path, monkeypatch, override_path): cache_files = ["file1.pyo", "file2.pyc"] source_file = "file1.py" @@ -101,8 +101,8 @@ def _check_files(directory): # spack.cmd.clean references paths from spack.paths: we want to # update them for the duration of this test. - monkeypatch.setattr(paths, "lib_path", source_dir) - monkeypatch.setattr(paths, "repos_path", repos_dir) + override_path("lib_path", source_dir) + override_path("repos_path", repos_dir) spack.cmd.clean.remove_python_cache() diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 3922130821a9c2..b08e9945da2732 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1718,9 +1718,10 @@ def test_included_path_git_unsat( "key,value", [("branch", "main"), ("commit", "abcdef123456"), ("tag", "v1.0")] ) def test_included_path_git( - tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, key, value, capfd + tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, key, value, capfd, + override_path ): - monkeypatch.setattr(spack_paths, "user_cache_path", str(tmp_path)) + override_path("user_cache_path", str(tmp_path)) class MockIncludeGit(spack.util.executable.Executable): def __init__(self, required: bool): @@ -1787,8 +1788,8 @@ def _checkout(*args, **kwargs): assert "already cloned" in captured -def test_included_path_git_errs(tmp_path: pathlib.Path, mock_low_high_config, monkeypatch): - monkeypatch.setattr(spack_paths, "user_cache_path", str(tmp_path)) +def test_included_path_git_errs(tmp_path: pathlib.Path, mock_low_high_config, monkeypatch, override_path): + override_path("user_cache_path", str(tmp_path)) paths = ["concretizer.yaml"] entry = { diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 2d409044aa1d9a..93b0c8c69a2e8c 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -45,6 +45,7 @@ import spack.modules.common import spack.package_base import spack.paths +import spack.paths_base import spack.platforms import spack.repo import spack.solver.asp @@ -150,13 +151,24 @@ def write_file(filename, contents): @pytest.fixture -def override_git_repos_cache_path(tmp_path: Path): - saved = paths.user_repos_cache_path +def override_path(monkeypatch): + def _override(path_attr, new_path): + if hasattr(spack.paths_base, path_attr): + monkeypatch.setattr(spack.paths_base, path_attr, new_path) + + if hasattr(spack.paths, path_attr): + monkeypatch.setattr(spack.paths, path_attr, new_path) + + monkeypatch.setattr(spack.paths.locations, path_attr, new_path) + + return _override + + +@pytest.fixture +def override_git_repos_cache_path(tmp_path: Path, monkeypatch, override_path): tmp_git_path = tmp_path / "git-repo-cache-path-for-tests" tmp_git_path.mkdir() - paths.user_repos_cache_path = str(tmp_git_path) - yield - paths.user_repos_cache_path = saved + override_path("user_repos_cache_path", str(tmp_git_path)) @pytest.fixture diff --git a/lib/spack/spack/test/main.py b/lib/spack/spack/test/main.py index b8351e83fe280a..8b97b83b295eb4 100644 --- a/lib/spack/spack/test/main.py +++ b/lib/spack/spack/test/main.py @@ -73,8 +73,8 @@ def test_git_sha_output(tmp_path: pathlib.Path, working_env, monkeypatch): assert expected == spack.get_version() -def test_get_version_no_repo(tmp_path: pathlib.Path, monkeypatch): - monkeypatch.setattr(paths, "prefix", str(tmp_path)) +def test_get_version_no_repo(tmp_path: pathlib.Path, override_path): + override_path("prefix", str(tmp_path)) assert spack.spack_version == spack.get_version() From cea6d839b12e787cd78ea3775eb72c8ac5a21061 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 15 Dec 2025 14:44:01 -0800 Subject: [PATCH 199/506] add explanatory comment --- lib/spack/spack/paths.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 04d144e02b20ef..2e689c34846f03 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -237,4 +237,6 @@ def prefer_old_location(self, old_location, new_location): locations = SpackPaths(paths_base.locations) +# At least one builtin spack package expects that spack.paths is +# importable and that it has this module-level attribute spack_script = locations.spack_script From d6391802f8af17af7c93ea70694ee3e6c6319b30 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 15 Dec 2025 14:51:27 -0800 Subject: [PATCH 200/506] auto style --- lib/spack/spack/test/ci.py | 3 +-- lib/spack/spack/test/cmd/buildcache.py | 4 +++- lib/spack/spack/test/config.py | 14 +++++++++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index 9a753de096c161..a9dcef18ec278b 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -260,8 +260,7 @@ def test_ci_copy_test_logs_to_artifacts_fail(tmp_path: pathlib.Path, capfd): def test_setup_spack_repro_version( - tmp_path: pathlib.Path, capfd, last_two_git_commits, monkeypatch, - override_path + tmp_path: pathlib.Path, capfd, last_two_git_commits, monkeypatch, override_path ): c1, c2 = last_two_git_commits repro_dir = tmp_path / "repro" diff --git a/lib/spack/spack/test/cmd/buildcache.py b/lib/spack/spack/test/cmd/buildcache.py index 75022e21096081..7bfb655fd34746 100644 --- a/lib/spack/spack/test/cmd/buildcache.py +++ b/lib/spack/spack/test/cmd/buildcache.py @@ -538,7 +538,9 @@ def test_push_without_build_deps( @pytest.fixture(scope="function") def v2_buildcache_layout(tmp_path: pathlib.Path): def _layout(signedness: str = "signed"): - source_path = str(pathlib.Path(paths.test_path) / "data" / "mirrors" / "v2_layout" / signedness) + source_path = str( + pathlib.Path(paths.test_path) / "data" / "mirrors" / "v2_layout" / signedness + ) test_mirror_path = tmp_path / "mirror" copy_tree(source_path, test_mirror_path) return test_mirror_path diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index b08e9945da2732..ce95e0a3771483 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1718,8 +1718,14 @@ def test_included_path_git_unsat( "key,value", [("branch", "main"), ("commit", "abcdef123456"), ("tag", "v1.0")] ) def test_included_path_git( - tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, key, value, capfd, - override_path + tmp_path: pathlib.Path, + mock_low_high_config, + ensure_debug, + monkeypatch, + key, + value, + capfd, + override_path, ): override_path("user_cache_path", str(tmp_path)) @@ -1788,7 +1794,9 @@ def _checkout(*args, **kwargs): assert "already cloned" in captured -def test_included_path_git_errs(tmp_path: pathlib.Path, mock_low_high_config, monkeypatch, override_path): +def test_included_path_git_errs( + tmp_path: pathlib.Path, mock_low_high_config, monkeypatch, override_path +): override_path("user_cache_path", str(tmp_path)) paths = ["concretizer.yaml"] From 4599b95484dc2f69e993c26b8dc10b97670ae749 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 15 Dec 2025 14:54:39 -0800 Subject: [PATCH 201/506] more reference issues --- lib/spack/spack/paths.py | 2 -- lib/spack/spack/test/ci.py | 4 ++-- lib/spack/spack/test/cmd/clean.py | 1 - lib/spack/spack/test/main.py | 1 - 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 2e689c34846f03..cfca9dd8364602 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -11,9 +11,7 @@ import itertools import os import pathlib -from collections import namedtuple from enum import Enum -from pathlib import PurePath import spack.config as config import spack.paths_base as paths_base diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index a9dcef18ec278b..bc9954bdad2d92 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -14,7 +14,7 @@ import spack.environment as ev import spack.error import spack.llnl.util.filesystem as fs -import spack.paths +from spack.paths import locations as paths import spack.repo as repo import spack.util.git from spack.spec import Spec @@ -267,7 +267,7 @@ def test_setup_spack_repro_version( spack_dir = repro_dir / "spack" spack_dir.mkdir(parents=True) - prefix_save = spack.paths.locations.prefix + prefix_save = paths.prefix override_path("prefix", "/garbage") ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1) diff --git a/lib/spack/spack/test/cmd/clean.py b/lib/spack/spack/test/cmd/clean.py index 0ce973f118a81f..7462507dcdb249 100644 --- a/lib/spack/spack/test/cmd/clean.py +++ b/lib/spack/spack/test/cmd/clean.py @@ -14,7 +14,6 @@ import spack.package_base import spack.stage import spack.store -from spack.paths import locations as paths clean = spack.main.SpackCommand("clean") diff --git a/lib/spack/spack/test/main.py b/lib/spack/spack/test/main.py index 8b97b83b295eb4..e43d069b9dcc14 100644 --- a/lib/spack/spack/test/main.py +++ b/lib/spack/spack/test/main.py @@ -19,7 +19,6 @@ import spack.util.executable as exe import spack.util.git import spack.util.spack_yaml as syaml -from spack.paths import locations as paths pytestmark = pytest.mark.not_on_windows( "Test functionality supported but tests are failing on Win" From 653f100d366b830645c636238313b8df939820e6 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 15 Dec 2025 14:55:05 -0800 Subject: [PATCH 202/506] style edit --- lib/spack/spack/test/ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index bc9954bdad2d92..49f2bc2701fd5d 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -14,9 +14,9 @@ import spack.environment as ev import spack.error import spack.llnl.util.filesystem as fs -from spack.paths import locations as paths import spack.repo as repo import spack.util.git +from spack.paths import locations as paths from spack.spec import Spec from spack.test.conftest import MockHTTPResponse, RepoBuilder from spack.version import Version From 0f22f97e50851ac00e1b77bd390400939b19fc59 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 15 Dec 2025 14:59:17 -0800 Subject: [PATCH 203/506] other wrong import --- lib/spack/spack/test/environment_modifications.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/environment_modifications.py b/lib/spack/spack/test/environment_modifications.py index d562a72a95a637..c0993072d5b117 100644 --- a/lib/spack/spack/test/environment_modifications.py +++ b/lib/spack/spack/test/environment_modifications.py @@ -9,7 +9,7 @@ import pytest import spack.util.environment as environment -from spack.paths import spack_root +from spack.paths import locations as paths from spack.util.environment import ( AppendPath, EnvironmentModifications, @@ -21,7 +21,7 @@ is_system_path, ) -datadir = os.path.join(spack_root, "lib", "spack", "spack", "test", "data") +datadir = os.path.join(paths.spack_root, "lib", "spack", "spack", "test", "data") shell_extension = ".bat" if sys.platform == "win32" else ".sh" From 7c67dc698cc4198d5f40c7c381b7509d97e6a1e4 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 15 Dec 2025 15:04:21 -0800 Subject: [PATCH 204/506] correction of import --- lib/spack/spack/test/packages.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 75378a0e974473..afead09b65cae3 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -16,7 +16,7 @@ import spack.package import spack.package_base import spack.repo -from spack.paths import mock_packages_path +from spack.paths import locations as paths from spack.spec import Spec from spack.util.naming import pkg_name_to_class_name from spack.version import VersionChecksumError @@ -46,15 +46,17 @@ def test_package_name(self): assert pkg_cls.name == "mpich" def test_package_filename(self): - repo = spack.repo.from_path(mock_packages_path) + repo = spack.repo.from_path(paths.mock_packages_path) filename = repo.filename_for_package_name("mpich") - assert filename == os.path.join(mock_packages_path, "packages", "mpich", "package.py") + assert filename == os.path.join( + paths.mock_packages_path, "packages", "mpich", "package.py" + ) def test_nonexisting_package_filename(self): - repo = spack.repo.from_path(mock_packages_path) + repo = spack.repo.from_path(paths.mock_packages_path) filename = repo.filename_for_package_name("some-nonexisting-package") assert filename == os.path.join( - mock_packages_path, "packages", "some_nonexisting_package", "package.py" + paths.mock_packages_path, "packages", "some_nonexisting_package", "package.py" ) def test_package_class_names(self): From bfa90ff52a8aa5ddee9c7c5de35fac136a779d2c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 15 Dec 2025 15:09:05 -0800 Subject: [PATCH 205/506] re-export all path_base module attributes as paths module attributes --- lib/spack/spack/paths.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index cfca9dd8364602..05c8084d0e01a8 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -236,5 +236,33 @@ def prefer_old_location(self, old_location, new_location): locations = SpackPaths(paths_base.locations) # At least one builtin spack package expects that spack.paths is -# importable and that it has this module-level attribute +# importable and that it has spack_script as a module-level attribute. +# Some test packages expect other paths (like test_path) +prefix = locations.prefix +spack_root = locations.spack_root +bin_path = locations.bin_path spack_script = locations.spack_script +sbang_script = locations.sbang_script +lib_path = locations.lib_path +external_path = locations.external_path +module_path = locations.module_path +vendor_path = locations.vendor_path +command_path = locations.command_path +analyzers_path = locations.analyzers_path +platform_path = locations.platform_path +compilers_path = locations.compilers_path +operating_system_path = locations.operating_system_path +test_path = locations.test_path +hooks_path = locations.hooks_path +share_path = locations.share_path +etc_path = locations.etc_path +default_license_dir = locations.default_license_dir +var_path = locations.var_path +repos_path = locations.repos_path +test_repos_path = locations.test_repos_path +mock_packages_path = locations.mock_packages_path +mock_gpg_data_path = locations.mock_gpg_data_path +mock_gpg_keys_path = locations.mock_gpg_keys_path +default_xdg_cache_home = locations.default_xdg_cache_home +system_config_path = locations.system_config_path +user_config_path = locations.user_config_path From da7cd4813c5f01c7576619f6a68d39d931529281 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 15 Dec 2025 15:31:40 -0800 Subject: [PATCH 206/506] add setter for property updated in tests --- lib/spack/spack/paths.py | 7 +++++++ lib/spack/spack/test/conftest.py | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 05c8084d0e01a8..f2df32554eafab 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -129,8 +129,15 @@ def default_monitor_path(self): @property def user_repos_cache_path(self): #: git repositories fetched to compare commits to versions + if hasattr(self, "_user_repos_cache_path"): + return self._user_repos_cache_path return os.path.join(self.state_home, "git_repos") + @user_repos_cache_path.setter + def user_repos_cache_path(self, val): + # setter for tests + self._user_repos_cache_path = val + @property def package_repos_path(self): #: default location where remote package repositories are cloned diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 93b0c8c69a2e8c..227c8587d67cca 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -159,7 +159,8 @@ def _override(path_attr, new_path): if hasattr(spack.paths, path_attr): monkeypatch.setattr(spack.paths, path_attr, new_path) - monkeypatch.setattr(spack.paths.locations, path_attr, new_path) + if hasattr(spack.paths.locations, path_attr): + monkeypatch.setattr(spack.paths.locations, path_attr, new_path) return _override From eeecfa8a7680189b32e971ad9385c953332a19d1 Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Mon, 15 Dec 2025 16:24:50 -0800 Subject: [PATCH 207/506] update paths tests --- lib/spack/spack/paths.py | 3 +- lib/spack/spack/test/paths.py | 54 ++++++++++++++++++----------------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index f2df32554eafab..da688532d39bed 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -199,7 +199,8 @@ def spack_env_check(): def xdg_env_check(): if disable_env: return - return os.environ.get(xdg_var, None) + if xdg_var in os.environ: + return os.path.join(os.environ[xdg_var], home_rel) def spack_home_env_check(): if disable_env: diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 97912f3bd1d7e1..1c8162472079fd 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -5,7 +5,8 @@ import os import pathlib -import spack.paths as paths +from spack.paths import SpackPaths +from spack.paths_base import SpackPathsBase def _ensure_dir(pathlike): @@ -14,56 +15,56 @@ def _ensure_dir(pathlike): def test_install_location(working_env, tmp_path): - # With no direction from env vars, a fresh clone of Spack - # should default to using $spack/opt/spack (this was the - # default <= 2025) + # If prior default install dir inside spack prefix does not + # exist, place installs in $HOME base_prefix = _ensure_dir(tmp_path / "base-prefix") - p1 = paths.SpackPaths(base_prefix) - assert p1.default_install_location == str(pathlib.Path(base_prefix) / "opt" / "spack") + os.environ["HOME"] = base_prefix + p1 = SpackPaths(SpackPathsBase(base_prefix)) + assert p1.default_install_location == str(pathlib.Path(base_prefix) / ".local" / "share" / "spack" / "installs") # XDG_DATA_HOME overrides all the above xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") os.environ["XDG_DATA_HOME"] = xdg_data_home - p1 = paths.SpackPaths(base_prefix) - assert p1.default_install_location == str(pathlib.Path(xdg_data_home) / "spack" / "installs") + p1 = SpackPaths(SpackPathsBase(base_prefix)) + assert p1.default_install_location == str(pathlib.Path(xdg_data_home) / ".local" / "share" / "spack" / "installs") # Check that SPACK_DATA_HOME overrides all the above spack_data_home = _ensure_dir(tmp_path / "spack_data_home") os.environ["SPACK_DATA_HOME"] = spack_data_home - p2 = paths.SpackPaths(base_prefix) + p2 = SpackPaths(SpackPathsBase(base_prefix)) assert p2.default_install_location == str(pathlib.Path(spack_data_home) / "installs") def test_system_config_path_is_overridable(working_env, tmp_path): redirect_syscfg_path = str(pathlib.Path(tmp_path) / "redirected_syscfg") os.environ["SPACK_SYSTEM_CONFIG_PATH"] = redirect_syscfg_path - p1 = paths.SpackPaths(_ensure_dir(tmp_path / "base-prefix")) + p1 = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "base-prefix"))) assert p1.system_config_path == redirect_syscfg_path def test_system_config_path_is_default_when_env_var_is_empty(working_env, tmp_path): os.environ["SPACK_SYSTEM_CONFIG_PATH"] = "" - p1 = paths.SpackPaths(str(tmp_path)) + p1 = SpackPaths(SpackPathsBase(str(tmp_path))) assert os.sep + os.path.join("etc", "spack") == p1.system_config_path def test_user_config_path_is_overridable(working_env, tmp_path): redirect_usrcfg_path = str(pathlib.Path(tmp_path) / "redirected_usrcfg") os.environ["SPACK_USER_CONFIG_PATH"] = redirect_usrcfg_path - p1 = paths.SpackPaths(_ensure_dir(tmp_path / "base-prefix")) + p1 = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "base-prefix"))) assert p1.user_config_path == redirect_usrcfg_path def test_user_config_path_is_default_when_env_var_is_empty(working_env, tmp_path): os.environ["SPACK_USER_CONFIG_PATH"] = "" - p1 = paths.SpackPaths(str(tmp_path)) + p1 = SpackPaths(SpackPathsBase(str(tmp_path))) assert os.path.expanduser(os.path.join("~", ".config", "spack")) == p1.user_config_path def test_user_cache_path_is_overridable(working_env, tmp_path): redirect_usr_cache = str(pathlib.Path(tmp_path) / "redirected_usr_cache") os.environ["SPACK_USER_CACHE_PATH"] = redirect_usr_cache - p1 = paths.SpackPaths(_ensure_dir(tmp_path / "base-prefix")) + p1 = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "base-prefix"))) assert p1.user_cache_path == redirect_usr_cache # Check that things that are supposed to be bundled inside of @@ -72,34 +73,35 @@ def test_user_cache_path_is_overridable(working_env, tmp_path): def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmp_path): - user_cache_path = _ensure_dir(tmp_path / "user-cache") base_prefix = _ensure_dir(tmp_path / "base-prefix") - os.environ["SPACK_USER_CACHE_PATH"] = user_cache_path + os.environ["HOME"] = base_prefix - p1 = paths.SpackPaths(base_prefix) - assert p1.gpg_path == str(pathlib.Path(user_cache_path) / "gpg") - assert p1.gpg_keys_path == str(pathlib.Path(user_cache_path) / "gpg-keys") + new_default_gpg_base = pathlib.Path(base_prefix) / ".local" / "share" / "spack" + + p1 = SpackPaths(SpackPathsBase(base_prefix)) + assert p1.gpg_path == str(new_default_gpg_base / "gpg") + assert p1.gpg_keys_path == str(new_default_gpg_base / "gpg-keys") old_gpg_dir = pathlib.Path(base_prefix) / "opt" / "spack" / "gpg" (old_gpg_dir).mkdir(parents=True) - p1 = paths.SpackPaths(base_prefix) + p1 = SpackPaths(SpackPathsBase(base_prefix)) # Old dir exists, but is empty, so it should not be used - assert p1.gpg_path == str(pathlib.Path(user_cache_path) / "gpg") + assert p1.gpg_path == str(new_default_gpg_base / "gpg") (old_gpg_dir / "something").touch() - p1 = paths.SpackPaths(base_prefix) + p1 = SpackPaths(SpackPathsBase(base_prefix)) # Now it should redirect assert p1.gpg_path == str(old_gpg_dir) # But the keys are handled separately and should use the new path - assert p1.gpg_keys_path == str(pathlib.Path(user_cache_path) / "gpg-keys") + assert p1.gpg_keys_path == str(new_default_gpg_base / "gpg-keys") old_gpg_keys_dir = pathlib.Path(base_prefix) / "var" / "spack" / "gpg" old_gpg_keys_dir.mkdir(parents=True) (old_gpg_keys_dir / "something").touch() - p1 = paths.SpackPaths(base_prefix) + p1 = SpackPaths(SpackPathsBase(base_prefix)) assert p1.gpg_keys_path == str(old_gpg_keys_dir) def test_user_cache_path_is_default_when_env_var_is_empty(working_env, tmp_path): os.environ["SPACK_USER_CACHE_PATH"] = "" - p1 = paths.SpackPaths(str(tmp_path)) - assert os.path.expanduser(os.path.join("~", ".local", "share", "spack")) == p1.user_cache_path + p1 = SpackPaths(SpackPathsBase(str(tmp_path))) + assert os.path.expanduser(os.path.join("~", ".local", "state", "spack")) == p1.user_cache_path From 5f0f200839830e162766ae1d51ba7c0b197365fb Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Mon, 15 Dec 2025 16:26:23 -0800 Subject: [PATCH 208/506] style auto fix --- lib/spack/spack/test/paths.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 1c8162472079fd..e166bbf6f7ce9d 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -20,13 +20,17 @@ def test_install_location(working_env, tmp_path): base_prefix = _ensure_dir(tmp_path / "base-prefix") os.environ["HOME"] = base_prefix p1 = SpackPaths(SpackPathsBase(base_prefix)) - assert p1.default_install_location == str(pathlib.Path(base_prefix) / ".local" / "share" / "spack" / "installs") + assert p1.default_install_location == str( + pathlib.Path(base_prefix) / ".local" / "share" / "spack" / "installs" + ) # XDG_DATA_HOME overrides all the above xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") os.environ["XDG_DATA_HOME"] = xdg_data_home p1 = SpackPaths(SpackPathsBase(base_prefix)) - assert p1.default_install_location == str(pathlib.Path(xdg_data_home) / ".local" / "share" / "spack" / "installs") + assert p1.default_install_location == str( + pathlib.Path(xdg_data_home) / ".local" / "share" / "spack" / "installs" + ) # Check that SPACK_DATA_HOME overrides all the above spack_data_home = _ensure_dir(tmp_path / "spack_data_home") From 901a444b1a29a8794deb516d59b6cbca20d2d193 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 15 Dec 2025 16:50:33 -0800 Subject: [PATCH 209/506] update test based on new desired behavior --- lib/spack/spack/paths.py | 12 ++++----- lib/spack/spack/test/paths.py | 48 ++++++++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index da688532d39bed..ff35f9bfa5049e 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -76,18 +76,16 @@ def state_home(self): ["SPACK_USER_CACHE_PATH", "SPACK_STATE_HOME"], "XDG_STATE_HOME", "state", - ".local/state/spack", + ".local/state", ) @property def cache_home(self): - return self.resolve_a_home("SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", ".cache/spack") + return self.resolve_a_home("SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", ".cache") @property def data_home(self): - return self.resolve_a_home( - "SPACK_DATA_HOME", "XDG_DATA_HOME", "data", ".local/share/spack" - ) + return self.resolve_a_home("SPACK_DATA_HOME", "XDG_DATA_HOME", "data", ".local/share") @property def user_cache_path(self): @@ -200,13 +198,13 @@ def xdg_env_check(): if disable_env: return if xdg_var in os.environ: - return os.path.join(os.environ[xdg_var], home_rel) + return os.path.join(os.environ[xdg_var], "spack") def spack_home_env_check(): if disable_env: return if "SPACK_HOME" in os.environ: - return os.path.join(os.environ["SPACK_HOME"], home_rel) + return os.path.join(os.environ["SPACK_HOME"], home_rel, "spack") def cfg_check(): return config.get(f"config:locations:{config_var}", None) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index e166bbf6f7ce9d..62eae1008e673e 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -5,6 +5,7 @@ import os import pathlib +import spack.config from spack.paths import SpackPaths from spack.paths_base import SpackPathsBase @@ -14,29 +15,52 @@ def _ensure_dir(pathlike): return str(pathlike) -def test_install_location(working_env, tmp_path): +def test_install_location(working_env, tmp_path, mutable_config): # If prior default install dir inside spack prefix does not # exist, place installs in $HOME - base_prefix = _ensure_dir(tmp_path / "base-prefix") - os.environ["HOME"] = base_prefix + base_prefix = _ensure_dir(tmp_path / "spack-root") + home_prefix = _ensure_dir(tmp_path / "home-prefix") + os.environ["HOME"] = home_prefix p1 = SpackPaths(SpackPathsBase(base_prefix)) assert p1.default_install_location == str( - pathlib.Path(base_prefix) / ".local" / "share" / "spack" / "installs" + pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs" + ) + + # "config:locations:home" variable overrides default + spack_home_prefix = _ensure_dir(tmp_path / "spack-home") + spack.config.set("config:locations:home", spack_home_prefix) + p2 = SpackPaths(SpackPathsBase(base_prefix)) + assert p2.default_install_location == str( + pathlib.Path(spack_home_prefix) / ".local" / "share" / "spack" / "installs" + ) + + # "config:locations:data" overrides the above + spack_data_prefix = _ensure_dir(tmp_path / "spack-data") + spack.config.set("config:locations:data", spack_data_prefix) + p3 = SpackPaths(SpackPathsBase(base_prefix)) + assert p3.default_install_location == str( + pathlib.Path(spack_data_prefix) / "spack" / "installs" ) - # XDG_DATA_HOME overrides all the above + # $XDG_DATA_HOME overrides all the above xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") os.environ["XDG_DATA_HOME"] = xdg_data_home - p1 = SpackPaths(SpackPathsBase(base_prefix)) - assert p1.default_install_location == str( - pathlib.Path(xdg_data_home) / ".local" / "share" / "spack" / "installs" - ) + p4 = SpackPaths(SpackPathsBase(base_prefix)) + assert p4.default_install_location == str(pathlib.Path(xdg_data_home) / "spack" / "installs") - # Check that SPACK_DATA_HOME overrides all the above + # Check that $SPACK_DATA_HOME overrides all the above spack_data_home = _ensure_dir(tmp_path / "spack_data_home") os.environ["SPACK_DATA_HOME"] = spack_data_home - p2 = SpackPaths(SpackPathsBase(base_prefix)) - assert p2.default_install_location == str(pathlib.Path(spack_data_home) / "installs") + p5 = SpackPaths(SpackPathsBase(base_prefix)) + assert p5.default_install_location == str(pathlib.Path(spack_data_home) / "installs") + + # Disable all location-based env vars: this will then defer + # to using "config:locations:data" + spack.config.set("config:locations:disable_env", True) + p6 = SpackPaths(SpackPathsBase(base_prefix)) + assert p6.default_install_location == str( + pathlib.Path(spack_data_prefix) / "spack" / "installs" + ) def test_system_config_path_is_overridable(working_env, tmp_path): From b0150a3e0e588150bb4630bf31e0297e3c086214 Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Mon, 15 Dec 2025 17:08:38 -0800 Subject: [PATCH 210/506] update test/behavior --- lib/spack/spack/paths.py | 4 ++-- lib/spack/spack/test/paths.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index ff35f9bfa5049e..7614381f2cb305 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -212,7 +212,7 @@ def cfg_check(): def spack_home_cfg_check(): h = config.get("config:locations:home", None) if h: - return os.path.join(h, home_rel) + return os.path.join(h, home_rel, "spack") for check in [ spack_env_check, @@ -225,7 +225,7 @@ def spack_home_cfg_check(): if possible_resolution: return possible_resolution - return os.path.join(os.path.expanduser("~"), home_rel) + return os.path.join(os.path.expanduser("~"), home_rel, "spack") def prefer_old_location(self, old_location, new_location): # TODO: perhaps it should be configurable whether old locations diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 62eae1008e673e..a4edba056f34e0 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -26,6 +26,8 @@ def test_install_location(working_env, tmp_path, mutable_config): pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs" ) + spack.config.set("config:locations", {}) + # "config:locations:home" variable overrides default spack_home_prefix = _ensure_dir(tmp_path / "spack-home") spack.config.set("config:locations:home", spack_home_prefix) @@ -39,7 +41,7 @@ def test_install_location(working_env, tmp_path, mutable_config): spack.config.set("config:locations:data", spack_data_prefix) p3 = SpackPaths(SpackPathsBase(base_prefix)) assert p3.default_install_location == str( - pathlib.Path(spack_data_prefix) / "spack" / "installs" + pathlib.Path(spack_data_prefix) / "installs" ) # $XDG_DATA_HOME overrides all the above @@ -59,7 +61,7 @@ def test_install_location(working_env, tmp_path, mutable_config): spack.config.set("config:locations:disable_env", True) p6 = SpackPaths(SpackPathsBase(base_prefix)) assert p6.default_install_location == str( - pathlib.Path(spack_data_prefix) / "spack" / "installs" + pathlib.Path(spack_data_prefix) / "installs" ) From 44ebc83a943540b544ef8a3353de26010b5702ea Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Mon, 15 Dec 2025 17:11:06 -0800 Subject: [PATCH 211/506] style edit --- lib/spack/spack/test/paths.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index a4edba056f34e0..ad711cb3db3749 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -40,9 +40,7 @@ def test_install_location(working_env, tmp_path, mutable_config): spack_data_prefix = _ensure_dir(tmp_path / "spack-data") spack.config.set("config:locations:data", spack_data_prefix) p3 = SpackPaths(SpackPathsBase(base_prefix)) - assert p3.default_install_location == str( - pathlib.Path(spack_data_prefix) / "installs" - ) + assert p3.default_install_location == str(pathlib.Path(spack_data_prefix) / "installs") # $XDG_DATA_HOME overrides all the above xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") @@ -60,9 +58,7 @@ def test_install_location(working_env, tmp_path, mutable_config): # to using "config:locations:data" spack.config.set("config:locations:disable_env", True) p6 = SpackPaths(SpackPathsBase(base_prefix)) - assert p6.default_install_location == str( - pathlib.Path(spack_data_prefix) / "installs" - ) + assert p6.default_install_location == str(pathlib.Path(spack_data_prefix) / "installs") def test_system_config_path_is_overridable(working_env, tmp_path): From e47bbd14f5408a0d3ca913a959adf5150e842905 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 17 Dec 2025 13:47:33 -0800 Subject: [PATCH 212/506] setter for tests --- lib/spack/spack/paths.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 7614381f2cb305..424cd87100f7d8 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -89,8 +89,14 @@ def data_home(self): @property def user_cache_path(self): + if hasattr(self, "_user_cache_path"): + return self._user_cache_path return self.state_home + @user_cache_path.setter + def user_cache_path(self, val): + self._user_cache_path = val + @property def default_install_location(self): return self.prefer_old_location( From 6909a6080d369ccbe1f99d86d13459aa28ecab56 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 17 Dec 2025 15:10:05 -0800 Subject: [PATCH 213/506] override config module cache location vs. paths variable --- lib/spack/spack/paths.py | 6 ------ lib/spack/spack/test/config.py | 3 +-- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 424cd87100f7d8..7614381f2cb305 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -89,14 +89,8 @@ def data_home(self): @property def user_cache_path(self): - if hasattr(self, "_user_cache_path"): - return self._user_cache_path return self.state_home - @user_cache_path.setter - def user_cache_path(self, val): - self._user_cache_path = val - @property def default_install_location(self): return self.prefer_old_location( diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 744444242ed65f..cc87182b355560 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1735,9 +1735,8 @@ def test_included_path_git( key, value, capfd, - override_path, ): - override_path("user_cache_path", str(tmp_path)) + monkeypatch.setattr(spack.config, "_include_cache_location", str(tmp_path)) class MockIncludeGit(spack.util.executable.Executable): def __init__(self, required: bool): From 91cc9b649dfc79b03f66e9cf1d6ca13123e7bda1 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 17 Dec 2025 15:13:51 -0800 Subject: [PATCH 214/506] auto style --- lib/spack/spack/test/config.py | 8 +------- lib/spack/spack/test/util/path.py | 1 - lib/spack/spack/test/util/remote_file_cache.py | 1 - 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index cc87182b355560..a06976ebe8ec2f 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1728,13 +1728,7 @@ def test_included_path_git_unsat( "key,value", [("branch", "main"), ("commit", "abcdef123456"), ("tag", "v1.0")] ) def test_included_path_git( - tmp_path: pathlib.Path, - mock_low_high_config, - ensure_debug, - monkeypatch, - key, - value, - capfd, + tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, key, value, capfd ): monkeypatch.setattr(spack.config, "_include_cache_location", str(tmp_path)) diff --git a/lib/spack/spack/test/util/path.py b/lib/spack/spack/test/util/path.py index 58ed5300d1dcb5..8f4ad0c7122a69 100644 --- a/lib/spack/spack/test/util/path.py +++ b/lib/spack/spack/test/util/path.py @@ -10,7 +10,6 @@ import spack.config import spack.llnl.util.tty as tty import spack.util.path as sup -from spack.paths import locations as paths #: Some lines with lots of placeholders padded_lines = [ diff --git a/lib/spack/spack/test/util/remote_file_cache.py b/lib/spack/spack/test/util/remote_file_cache.py index f99dffe0dbd127..4c5e81332acae9 100644 --- a/lib/spack/spack/test/util/remote_file_cache.py +++ b/lib/spack/spack/test/util/remote_file_cache.py @@ -11,7 +11,6 @@ import spack.llnl.util.tty as tty import spack.util.remote_file_cache as rfc_util from spack.llnl.util.filesystem import join_path -from spack.paths import locations as paths github_url = "https://github.com/fake/fake/{0}/develop" gitlab_url = "https://gitlab.fake.io/user/repo/-/blob/config/defaults" From a4f4fcd1183cb3cb6396273a98177313f7838c5d Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 17 Dec 2025 15:19:21 -0800 Subject: [PATCH 215/506] need to monkeypatch with lambda --- lib/spack/spack/test/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index a06976ebe8ec2f..8387a0961f1bb9 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1730,7 +1730,7 @@ def test_included_path_git_unsat( def test_included_path_git( tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, key, value, capfd ): - monkeypatch.setattr(spack.config, "_include_cache_location", str(tmp_path)) + monkeypatch.setattr(spack.config, "_include_cache_location", lambda: str(tmp_path)) class MockIncludeGit(spack.util.executable.Executable): def __init__(self, required: bool): @@ -1800,7 +1800,7 @@ def _checkout(*args, **kwargs): def test_included_path_git_errs( tmp_path: pathlib.Path, mock_low_high_config, monkeypatch, override_path ): - override_path("user_cache_path", str(tmp_path)) + monkeypatch.setattr(spack.config, "_include_cache_location", lambda: str(tmp_path)) paths = ["concretizer.yaml"] entry = { From ce1693acbfe843c465da4695f0c7e0f09237063c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 17 Dec 2025 15:28:53 -0800 Subject: [PATCH 216/506] importing environment module no longer requires config resolution --- lib/spack/spack/environment/environment.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index b10fac83781186..6b17441eceea8e 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -63,9 +63,6 @@ #: Validation error for a currently activate environment that failed to parse _active_environment_error: Optional[spack.config.ConfigFormatError] = None -#: default path where environments are stored in the spack tree -default_env_path = paths.default_envs_path - #: Name of the input yaml file for an environment manifest_name = "spack.yaml" @@ -82,7 +79,7 @@ def env_root_path() -> str: """Override default root path if the user specified it""" return spack.util.path.canonicalize_path( - spack.config.get("config:environments_root", default=default_env_path) + spack.config.get("config:environments_root", default=paths.default_envs_path) ) From 26bfcaaf1faadc1feef2b627ee7dbb7f9ee256dd Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 17 Dec 2025 15:44:19 -0800 Subject: [PATCH 217/506] amend last commit with something that unit tests can monkeypatch --- lib/spack/spack/environment/environment.py | 5 ++++- lib/spack/spack/test/conftest.py | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index 6b17441eceea8e..b03919dbc22a5b 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -63,6 +63,9 @@ #: Validation error for a currently activate environment that failed to parse _active_environment_error: Optional[spack.config.ConfigFormatError] = None +#: default path where environments are stored in the spack tree +default_env_path = lambda: paths.default_envs_path + #: Name of the input yaml file for an environment manifest_name = "spack.yaml" @@ -79,7 +82,7 @@ def env_root_path() -> str: """Override default root path if the user specified it""" return spack.util.path.canonicalize_path( - spack.config.get("config:environments_root", default=paths.default_envs_path) + spack.config.get("config:environments_root", default=default_env_path()) ) diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 25cad807ff4e9f..926f16321592bc 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -1902,12 +1902,20 @@ def get_rev(): yield t +class LazyValue: + def __init__(self, val): + self.val = val + + def __call__(self): + return self.val + + @pytest.fixture(scope="function") def mutable_mock_env_path(tmp_path: Path, mutable_config, monkeypatch): """Fixture for mocking the internal spack environments directory.""" mock_path = tmp_path / "mock-env-path" mutable_config.set("config:environments_root", str(mock_path)) - monkeypatch.setattr(ev.environment, "default_env_path", str(mock_path)) + monkeypatch.setattr(ev.environment, "default_env_path", LazyValue(str(mock_path))) return mock_path From 71b2711080c595a51799614f0635561feb1d6693 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 17 Dec 2025 16:50:58 -0800 Subject: [PATCH 218/506] proper path formatting cross-system --- lib/spack/spack/paths.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 7614381f2cb305..a95bee0c817a03 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -76,7 +76,7 @@ def state_home(self): ["SPACK_USER_CACHE_PATH", "SPACK_STATE_HOME"], "XDG_STATE_HOME", "state", - ".local/state", + os.path.join(".local", "state"), ) @property @@ -85,7 +85,9 @@ def cache_home(self): @property def data_home(self): - return self.resolve_a_home("SPACK_DATA_HOME", "XDG_DATA_HOME", "data", ".local/share") + return self.resolve_a_home( + "SPACK_DATA_HOME", "XDG_DATA_HOME", "data", os.path.join(".local", "share") + ) @property def user_cache_path(self): From ebcba9d191e9288ff704c370e82a56f41401ee29 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 17 Dec 2025 17:00:02 -0800 Subject: [PATCH 219/506] rm out of date comment --- lib/spack/spack/paths.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index a95bee0c817a03..4f004c4873ba6c 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -232,9 +232,6 @@ def spack_home_cfg_check(): def prefer_old_location(self, old_location, new_location): # TODO: perhaps it should be configurable whether old locations # are used. Other option is to relocate downloads & gpg keys. - # TODO: if user sets SPACK/XDG_DATA_HOME, should we move installs - # there even if old dir is occupied? (right now that's what is - # happening here) if dir_is_occupied(old_location): return old_location else: From c6b66480c18d8c7090a87daed2c00fc2447a5012 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 30 Dec 2025 16:13:22 -0800 Subject: [PATCH 220/506] bootstrap store was moved into xdg_state_home, not xdg_data_home --- .github/workflows/bootstrap.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index 6c7435ef0c324e..2282411dd3ec71 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -47,7 +47,7 @@ jobs: spack bootstrap disable github-actions-v2 spack bootstrap disable github-actions-v0.6 spack -d solve zlib - tree ~/.local/share/spack/bootstrap/store + tree ~/.local/state/spack/bootstrap/store clingo-sources: if: github.repository == 'spack/spack' @@ -73,7 +73,7 @@ jobs: spack bootstrap disable github-actions-v0.6 export PATH="$(brew --prefix bison)/bin:$(brew --prefix cmake)/bin:$PATH" spack -d solve zlib - tree ~/.local/share/spack/bootstrap/store + tree ~/.local/state/spack/bootstrap/store gnupg-sources: if: github.repository == 'spack/spack' runs-on: ${{ matrix.runner }} @@ -100,7 +100,7 @@ jobs: spack bootstrap disable github-actions-v2 spack bootstrap disable github-actions-v0.6 spack -d gpg list - tree ~/.local/share/spack/bootstrap/store + tree ~/.local/state/spack/bootstrap/store from-binaries: if: github.repository == 'spack/spack' @@ -148,12 +148,12 @@ jobs: fi spack solve zlib done - tree ~/.local/share/spack/bootstrap/store + tree ~/.local/state/spack/bootstrap/store - name: Bootstrap GnuPG run: | . share/spack/setup-env.sh spack -d gpg list - tree ~/.local/share/spack/bootstrap/store + tree ~/.local/state/spack/bootstrap/store windows: if: github.repository == 'spack/spack' @@ -177,10 +177,10 @@ jobs: spack bootstrap disable github-actions-v0.6 spack -d solve zlib ./share/spack/qa/validate_last_exit.ps1 - tree $env:userprofile/.local/share/spack/bootstrap/store + tree $env:userprofile/.local/state/spack/bootstrap/store - name: Bootstrap GnuPG run: | ./share/spack/setup-env.ps1 spack -d gpg list ./share/spack/qa/validate_last_exit.ps1 - tree $env:userprofile/.local/share/spack/bootstrap/store + tree $env:userprofile/.local/state/spack/bootstrap/store From 334451eb04072c161397c601fce9a71822c16549 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 30 Dec 2025 16:16:45 -0800 Subject: [PATCH 221/506] style edit --- lib/spack/spack/test/spec_semantics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 3adf0660f4218f..6fc2e51efb7ff7 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -20,8 +20,8 @@ import spack.version as vn from spack.enums import PropagationPolicy from spack.error import SpecError, UnsatisfiableSpecError -from spack.paths import locations as paths from spack.llnl.util.tty.color import colorize +from spack.paths import locations as paths from spack.spec import ArchSpec, DependencySpec, Spec, SpecFormatSigilError, SpecFormatStringError from spack.variant import ( InvalidVariantValueError, From b8c05af7d490a06390cd239867bfbd95faebad58 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 9 Jan 2026 13:44:18 -0800 Subject: [PATCH 222/506] style fix --- lib/spack/spack/test/cmd/buildcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/cmd/buildcache.py b/lib/spack/spack/test/cmd/buildcache.py index 3722b1ff9f81fd..e3b0e0a6fca8e0 100644 --- a/lib/spack/spack/test/cmd/buildcache.py +++ b/lib/spack/spack/test/cmd/buildcache.py @@ -27,8 +27,8 @@ import spack.util.web as web_util from spack.installer import PackageInstaller from spack.llnl.util.filesystem import copy_tree, find, getuid -from spack.paths import locations as paths from spack.llnl.util.lang import nullcontext +from spack.paths import locations as paths from spack.url_buildcache import ( BuildcacheComponent, URLBuildcacheEntry, From c08482a0e40a0782c9955fc3cae426bbcf6fe536 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 9 Jan 2026 13:46:19 -0800 Subject: [PATCH 223/506] rm vestigial method --- lib/spack/spack/paths.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 4f004c4873ba6c..621a4cf28ead99 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -44,17 +44,6 @@ def _unset_xdg_vars(env): return saved -def _spack_xdg_or_backup(xdg_mapping): - if xdg_mapping.spack in os.environ: - val = os.environ[xdg_mapping.spack] - elif xdg_mapping.xdg in os.environ: - val = os.path.join(os.environ[xdg_mapping.xdg], "spack") - else: - val = os.path.join(xdg_mapping.xdg_default, "spack") - - return os.path.expanduser(val) - - def dir_is_occupied(x, except_for=None): x = pathlib.Path(x) except_for = except_for or set() From 0d61e5cdf34496f1e20f61943104860290518f2c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 9 Jan 2026 16:25:20 -0800 Subject: [PATCH 224/506] update error message --- lib/spack/spack/trace.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/trace.py b/lib/spack/spack/trace.py index 980f6c6ac1377f..3c68dc2b208ea5 100644 --- a/lib/spack/spack/trace.py +++ b/lib/spack/spack/trace.py @@ -78,4 +78,6 @@ def warn_writes_into_spack(): if sys.version_info[:2] >= (3, 8): sys.addaudithook(_guard_writes) # novermin else: - raise ValueError(f"Cannot detect writes for Python {sys.version_info[:2]}") + raise ValueError( + f"Cannot detect writes for Python {sys.version_info[:2]} (supported in 3.8 and later)" + ) From 6e3f04d8adada72d8d78677ad2caab94d1cfd96c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 9 Jan 2026 16:36:35 -0800 Subject: [PATCH 225/506] config variables were wrong: replace them with fall-through; use that to redirect download cache (which no longer has a default_ config var because it doesnt matter so much if downloads used to be in the spack root) --- etc/spack/defaults/base/config.yaml | 7 +------ lib/spack/spack/caches.py | 4 +--- lib/spack/spack/paths.py | 6 ------ lib/spack/spack/util/path.py | 6 +++--- 4 files changed, 5 insertions(+), 18 deletions(-) diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index 77e61dcc05be1b..8d34768c654519 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -83,12 +83,7 @@ config: # Cache directory for already downloaded source tarballs and archived # repositories. This can be purged with `spack clean --downloads`. - # If set to a non-default value, this setting has precedence. - # This default value tells spack to check (in order of precedence): - # $SPACK_DATA_HOME/downloads - # $XDG_DATA_HOME/spack/downloads - # $spack/var/spack/cache - source_cache: $default_download_root + source_cache: $data_home/downloads ## Directory where spack managed environments are created and stored diff --git a/lib/spack/spack/caches.py b/lib/spack/spack/caches.py index 3ab71da2727b56..cc0cc86b44a8d1 100644 --- a/lib/spack/spack/caches.py +++ b/lib/spack/spack/caches.py @@ -43,9 +43,7 @@ def fetch_cache_location(): This prevents Spack from repeatedly fetch the same files when building the same package different ways or multiple times. """ - path = spack.config.get("config:source_cache") - if not path: - path = paths.default_fetch_cache_path + path = spack.config.get("config:source_cache", "$data_home/downloads") path = spack.util.path.canonicalize_path(path) return path diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 621a4cf28ead99..f81e395e9c71ee 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -94,12 +94,6 @@ def default_envs_path(self): self.base.old_envs_path, os.path.join(self.data_home, "envs") ) - @property - def default_fetch_cache_path(self): - return self.prefer_old_location( - self.base.old_fetch_cache_path, os.path.join(self.data_home, "downloads") - ) - @property def reports_path(self): #: junit, cdash, etc. reports about builds diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 986cdf0b912a4b..dc62325479493e 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -72,9 +72,9 @@ def replacements(): "default_install_root": lambda: paths.default_install_location, "default_envs_root": lambda: paths.default_envs_path, "default_download_root": lambda: paths.default_fetch_cache_path, - "data_home": lambda: paths.spack_data_home, - "cache_home": lambda: paths.spack_cache_home, - "state_home": lambda: paths.spack_state_home, + "data_home": lambda: paths.data_home, + "cache_home": lambda: paths.cache_home, + "state_home": lambda: paths.state_home, # "spack_config_home": lambda: paths.spack_config_home, "spack_instance_id": lambda: paths.spack_instance_id, "architecture": lambda: arch, From 42b835684845299617ad75e2437002d84cd89c88 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 9 Jan 2026 16:50:31 -0800 Subject: [PATCH 226/506] similar treatment for bootstrap cache --- lib/spack/spack/bootstrap/config.py | 2 +- lib/spack/spack/paths.py | 6 ------ lib/spack/spack/solver/core.py | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/bootstrap/config.py b/lib/spack/spack/bootstrap/config.py index 8638a753dfd000..fc9e2dda5700ff 100644 --- a/lib/spack/spack/bootstrap/config.py +++ b/lib/spack/spack/bootstrap/config.py @@ -43,7 +43,7 @@ def spec_for_current_python() -> str: def root_path() -> str: """Root of all the bootstrap related folders""" return spack.util.path.canonicalize_path( - spack.config.get("bootstrap:root", paths.default_user_bootstrap_path) + spack.config.get("bootstrap:root", "$user_cache_path/bootstrap") ) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index f81e395e9c71ee..fd265f135781b0 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -126,12 +126,6 @@ def package_repos_path(self): #: default location where remote package repositories are cloned return os.path.join(self.state_home, "package_repos") - @property - def default_user_bootstrap_path(self): - #: bootstrap store for bootstrapping clingo and other tools - #: overridden by `bootstrap:root` - return os.path.join(self.state_home, "bootstrap") - @property def gpg_path(self): return self.prefer_old_location( diff --git a/lib/spack/spack/solver/core.py b/lib/spack/spack/solver/core.py index aafed91fc9875c..dda34f10928983 100644 --- a/lib/spack/spack/solver/core.py +++ b/lib/spack/spack/solver/core.py @@ -158,7 +158,7 @@ def _ensure_clingo_or_raise(clingo_mod: ModuleType) -> None: if ( pathlib.Path( sup.canonicalize_path( - spack.config.CONFIG.get("bootstrap:root", paths.default_user_bootstrap_path) + spack.config.CONFIG.get("bootstrap:root", "$user_cache_path/bootstrap") ) ) in pathlib.Path(clingo_mod.__file__).parents From 06d3494521a6441bcb21cb228025c6b7f474ee31 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 9 Jan 2026 17:00:20 -0800 Subject: [PATCH 227/506] update logic that checks for preexisting installs in old location: we should make sure not to use the old location if installs are in the new location; this same change is desirable for all other users of prefer_old_location, which is renamed to _fallback_old_location_if_used --- lib/spack/spack/paths.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index fd265f135781b0..a715ce878e4168 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -84,13 +84,13 @@ def user_cache_path(self): @property def default_install_location(self): - return self.prefer_old_location( + return self._fallback_old_location_if_used( self.base.old_install_path, os.path.join(self.data_home, "installs") ) @property def default_envs_path(self): - return self.prefer_old_location( + return self._fallback_old_location_if_used( self.base.old_envs_path, os.path.join(self.data_home, "envs") ) @@ -128,13 +128,13 @@ def package_repos_path(self): @property def gpg_path(self): - return self.prefer_old_location( + return self._fallback_old_location_if_used( self.base.old_gpg_path, os.path.join(self.data_home, "gpg") ) @property def gpg_keys_path(self): - return self.prefer_old_location( + return self._fallback_old_location_if_used( self.base.old_gpg_keys_path, os.path.join(self.data_home, "gpg-keys") ) @@ -206,10 +206,15 @@ def spack_home_cfg_check(): return os.path.join(os.path.expanduser("~"), home_rel, "spack") - def prefer_old_location(self, old_location, new_location): + def _fallback_old_location_if_used(self, old_location, new_location): # TODO: perhaps it should be configurable whether old locations # are used. Other option is to relocate downloads & gpg keys. - if dir_is_occupied(old_location): + if dir_is_occupied(new_location): + return new_location + elif dir_is_occupied(old_location): + # TODO: should probably raise a deprecation warning here encouraging + # them to set their config explicitly back to the old value that + # will allow us to eventually remove these fallbacks return old_location else: return new_location From 70bc7829e4c791c9ce4e362b3f9fd2cf2b34eea8 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 9 Jan 2026 17:08:32 -0800 Subject: [PATCH 228/506] style edits --- lib/spack/spack/bootstrap/config.py | 1 - lib/spack/spack/solver/core.py | 3 --- lib/spack/spack/spec_parser.py | 1 - 3 files changed, 5 deletions(-) diff --git a/lib/spack/spack/bootstrap/config.py b/lib/spack/spack/bootstrap/config.py index fc9e2dda5700ff..b27d382ea293e6 100644 --- a/lib/spack/spack/bootstrap/config.py +++ b/lib/spack/spack/bootstrap/config.py @@ -16,7 +16,6 @@ import spack.store import spack.util.path from spack.llnl.util import tty -from spack.paths import locations as paths #: Reference counter for the bootstrapping configuration context manager _REF_COUNT = 0 diff --git a/lib/spack/spack/solver/core.py b/lib/spack/spack/solver/core.py index dda34f10928983..6421d570ce0d33 100644 --- a/lib/spack/spack/solver/core.py +++ b/lib/spack/spack/solver/core.py @@ -136,11 +136,8 @@ def _ensure_clingo_or_raise(clingo_mod: ModuleType) -> None: # These are imports that may be problematic at top level (circular imports). They are used # only to provide exhaustive details when erroring due to a broken clingo module. import spack.config - import spack.paths import spack.util.path as sup - paths = spack.paths.locations - try: clingo_mod.Symbol except AttributeError: diff --git a/lib/spack/spack/spec_parser.py b/lib/spack/spack/spec_parser.py index 1ba92ac8b5658f..1d5d66691b46c8 100644 --- a/lib/spack/spack/spec_parser.py +++ b/lib/spack/spack/spec_parser.py @@ -69,7 +69,6 @@ from spack.aliases import LEGACY_COMPILER_TO_BUILTIN from spack.enums import PropagationPolicy from spack.llnl.util.tty import color -from spack.paths import locations as paths from spack.tokenize import Token, TokenBase, Tokenizer if TYPE_CHECKING: From a2e9967bc7c36f766c908799d368beaf487a9220 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 9 Jan 2026 17:35:33 -0800 Subject: [PATCH 229/506] make SPACK_STATE_HOME higher priority than SPACK_USER_CACHE_PATH; add a docstring to resolve_a_home --- lib/spack/spack/paths.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index a715ce878e4168..8eb3cad10d6150 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -62,7 +62,7 @@ def __init__(self, base): @property def state_home(self): return self.resolve_a_home( - ["SPACK_USER_CACHE_PATH", "SPACK_STATE_HOME"], + ["SPACK_STATE_HOME", "SPACK_USER_CACHE_PATH"], "XDG_STATE_HOME", "state", os.path.join(".local", "state"), @@ -160,6 +160,22 @@ def __getattr__(self, name): return getattr(self.base, name) def resolve_a_home(self, env_vars, xdg_var, config_var, home_rel): + """ + Data stored by spack is split into state, data, and cache components. + This function can resolve where each of these components should be + stored. + + Args: + env_vars: spack-specific environment variables that indicate the + component location. Can be a list or a single variable. If this + is a list, earlier elements have precedence. + xdg_var: the XDG-based environment variable that indicates the + component location. + config_var: the spack config variable that indicates the component + location. + home_rel: for $SPACK_HOME and config:locations:home, this relative + path is appended to the result to get the component location. + """ disable_env = config.get("config:locations:disable_env", False) def spack_env_check(): From e2d50dce6b949fb45ff9e85f913f2d541bd5aa6e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 9 Jan 2026 18:07:36 -0800 Subject: [PATCH 230/506] windows env vars affecting test outcome --- lib/spack/spack/test/paths.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index ad711cb3db3749..b064c993257522 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -20,6 +20,9 @@ def test_install_location(working_env, tmp_path, mutable_config): # exist, place installs in $HOME base_prefix = _ensure_dir(tmp_path / "spack-root") home_prefix = _ensure_dir(tmp_path / "home-prefix") + os.environ.pop("USERPROFILE", None) + os.environ.pop("HOMEDRIVE", None) + os.environ.pop("HOMEPATH", None) os.environ["HOME"] = home_prefix p1 = SpackPaths(SpackPathsBase(base_prefix)) assert p1.default_install_location == str( From af494a24d7e882b8d088fd3e735780aff9000395 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 9 Jan 2026 18:15:06 -0800 Subject: [PATCH 231/506] more careful in unit test --- lib/spack/spack/test/paths.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index b064c993257522..2445f9928f4270 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -20,11 +20,21 @@ def test_install_location(working_env, tmp_path, mutable_config): # exist, place installs in $HOME base_prefix = _ensure_dir(tmp_path / "spack-root") home_prefix = _ensure_dir(tmp_path / "home-prefix") + + empty_dir = _ensure_dir(tmp_path / "empty") + + def paths_base_empty_old_install(): + pb = SpackPathsBase(base_prefix) + pb.old_install_path = empty_dir + return pb + + # Clear some env vars that can interfere w/ expanduser(~) on Windows os.environ.pop("USERPROFILE", None) os.environ.pop("HOMEDRIVE", None) os.environ.pop("HOMEPATH", None) + os.environ["HOME"] = home_prefix - p1 = SpackPaths(SpackPathsBase(base_prefix)) + p1 = SpackPaths(paths_base_empty_old_install()) assert p1.default_install_location == str( pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs" ) @@ -34,7 +44,7 @@ def test_install_location(working_env, tmp_path, mutable_config): # "config:locations:home" variable overrides default spack_home_prefix = _ensure_dir(tmp_path / "spack-home") spack.config.set("config:locations:home", spack_home_prefix) - p2 = SpackPaths(SpackPathsBase(base_prefix)) + p2 = SpackPaths(paths_base_empty_old_install()) assert p2.default_install_location == str( pathlib.Path(spack_home_prefix) / ".local" / "share" / "spack" / "installs" ) @@ -42,25 +52,25 @@ def test_install_location(working_env, tmp_path, mutable_config): # "config:locations:data" overrides the above spack_data_prefix = _ensure_dir(tmp_path / "spack-data") spack.config.set("config:locations:data", spack_data_prefix) - p3 = SpackPaths(SpackPathsBase(base_prefix)) + p3 = SpackPaths(paths_base_empty_old_install()) assert p3.default_install_location == str(pathlib.Path(spack_data_prefix) / "installs") # $XDG_DATA_HOME overrides all the above xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") os.environ["XDG_DATA_HOME"] = xdg_data_home - p4 = SpackPaths(SpackPathsBase(base_prefix)) + p4 = SpackPaths(paths_base_empty_old_install()) assert p4.default_install_location == str(pathlib.Path(xdg_data_home) / "spack" / "installs") # Check that $SPACK_DATA_HOME overrides all the above spack_data_home = _ensure_dir(tmp_path / "spack_data_home") os.environ["SPACK_DATA_HOME"] = spack_data_home - p5 = SpackPaths(SpackPathsBase(base_prefix)) + p5 = SpackPaths(paths_base_empty_old_install()) assert p5.default_install_location == str(pathlib.Path(spack_data_home) / "installs") # Disable all location-based env vars: this will then defer # to using "config:locations:data" spack.config.set("config:locations:disable_env", True) - p6 = SpackPaths(SpackPathsBase(base_prefix)) + p6 = SpackPaths(paths_base_empty_old_install()) assert p6.default_install_location == str(pathlib.Path(spack_data_prefix) / "installs") From e23f790375731a15f6b082774f7c5108f871afd4 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 9 Jan 2026 18:20:39 -0800 Subject: [PATCH 232/506] delete duplicate lines --- lib/spack/spack/paths_base.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index 18a72f4a44f31b..8a32ddb47d7ca7 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -13,9 +13,6 @@ def __init__(self, _prefix=None): #: This file lives in $prefix/lib/spack/spack/__file__ self.prefix = _prefix or str(PurePath(spack.llnl.util.filesystem.ancestor(__file__, 4))) - #: This file lives in $prefix/lib/spack/spack/__file__ - self.prefix = _prefix or str(PurePath(spack.llnl.util.filesystem.ancestor(__file__, 4))) - #: synonym for prefix self.spack_root = self.prefix From b43eea2554c63625a1ef94c4564164d8b6f0044b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 9 Jan 2026 18:32:01 -0800 Subject: [PATCH 233/506] tweak TODO --- lib/spack/spack/test/cmd/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index d6ba50d13a2bce..46910541cc6aa5 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -298,7 +298,7 @@ def test_updated_completion_scripts(shell, tmp_path: pathlib.Path, mutable_confi commands("--aliases", "--format", shell, "--header", header, "--update", new_script) # If there is a diff, something is wrong: in that case output what the diff is. - # TODO: if someone runs `spack commands --update-completion` when the site-admin + # TODO: if someone runs `spack commands --update-completion` when a non-default # scope is available, it will be added to the autocompletion scripts, and # cause a discrepancy with the runner; it might be worth changing the update # command to omit that. From 0d5b928e5de519da6be8c13ddb8d9b176e71265a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Jan 2026 10:47:25 -0800 Subject: [PATCH 234/506] no more repos_path --- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/paths.py | 1 - lib/spack/spack/paths_base.py | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index b2cd436ee34041..061f738d61b581 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -67,7 +67,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def remove_python_cache(): - for directory in [paths.lib_path, paths.repos_path]: + for directory in [paths.lib_path]: for root, dirs, files in os.walk(directory): for f in files: if f.endswith(".pyc") or f.endswith(".pyo"): diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 8eb3cad10d6150..d7f11b945d011a 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -261,7 +261,6 @@ def _fallback_old_location_if_used(self, old_location, new_location): etc_path = locations.etc_path default_license_dir = locations.default_license_dir var_path = locations.var_path -repos_path = locations.repos_path test_repos_path = locations.test_repos_path mock_packages_path = locations.mock_packages_path mock_gpg_data_path = locations.mock_gpg_data_path diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index 8a32ddb47d7ca7..561dac93f7194a 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -44,7 +44,6 @@ def __init__(self, _prefix=None): # $spack/var/spack is generally read-only. Older instances may # write gpg keys or environments into ...var/ - self.repos_path = os.path.join(self.var_path, "repos") self.test_repos_path = os.path.join(self.var_path, "test_repos") self.mock_packages_path = os.path.join(self.test_repos_path, "spack_repo", "builtin_mock") @@ -91,7 +90,6 @@ def __init__(self, _prefix=None): etc_path = locations.etc_path default_license_dir = locations.default_license_dir var_path = locations.var_path -repos_path = locations.repos_path test_repos_path = locations.test_repos_path mock_packages_path = locations.mock_packages_path mock_gpg_data_path = locations.mock_gpg_data_path From 0197b0d7f7723cc71b2e8c09fe71bcee724d9bf0 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Jan 2026 12:03:43 -0800 Subject: [PATCH 235/506] refactor session fixture according to review --- lib/spack/spack/paths.py | 5 +---- lib/spack/spack/test/conftest.py | 3 ++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index d7f11b945d011a..2c0d1de2163657 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -37,11 +37,8 @@ class XDG_overrides(Enum): # these vars will affect .default_test_path for the running instance, but # the unit tests will not see the env vars def _unset_xdg_vars(env): - saved = {} for xdg_var in itertools.chain(XDG_vars, XDG_overrides): - if xdg_var.value in env: - saved[xdg_var.value] = env.pop(xdg_var.value) - return saved + env.pop(xdg_var.value, None) def dir_is_occupied(x, except_for=None): diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 40b4e5e12dd207..03646cc2951b42 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -410,7 +410,8 @@ def clean_user_environment(): @pytest.fixture(scope="session", autouse=True) def clear_xdg_vars(): - saved = spack.paths._unset_xdg_vars(os.environ) + saved = os.environ.copy() + spack.paths._unset_xdg_vars(os.environ) yield os.environ.update(saved) From dd61aeb8e7738e689f6b217f3ebb69fbe1f8c8f5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Jan 2026 14:44:16 -0800 Subject: [PATCH 236/506] rewrite config comment --- etc/spack/defaults/base/concretizer.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/etc/spack/defaults/base/concretizer.yaml b/etc/spack/defaults/base/concretizer.yaml index 6b89f435f18fd8..afb70e615c4ec4 100644 --- a/etc/spack/defaults/base/concretizer.yaml +++ b/etc/spack/defaults/base/concretizer.yaml @@ -5,10 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by editing -# `$SPACK_ROOT/etc/spack/concretizer.yaml`, -# `~/.config/spack/concretizer.yaml`, or by adding a `concretizer:` -# section to an environment. +# Users can override part or all of these settings by defining their +# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). # ------------------------------------------------------------------------- concretizer: # Whether to consider installed packages or packages from buildcaches when From a060690fa964da7cf53f59ac7a81b182dc1a26ae Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Jan 2026 14:45:32 -0800 Subject: [PATCH 237/506] typo in docs --- lib/spack/docs/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index d524db3cf64ff9..33083ae9956504 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -269,7 +269,7 @@ Platform-specific Configuration There is often a need for platform-specific configuration settings. For example, on most platforms, GCC is the preferred compiler. However, on macOS (darwin), Clang often works for more packages, and is set as the default compiler. -This configuration is set in ``$(prefix)/etc/spack/defaults/darwin/packages.yaml``, which is included as by ``$(prefix)/etc/spack/defaults/include.yaml``. +This configuration is set in ``$(prefix)/etc/spack/defaults/darwin/packages.yaml``, which is included by ``$(prefix)/etc/spack/defaults/include.yaml``. Since it is an included configuration of the ``defaults`` scope, settings in the ``defaults`` scope will take precedence. For example, if ``$(prefix)/etc/spack/defaults/include.yaml`` contains: From abb128422c7dd4e0a59760d5c25a8136c2ccdccb Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Jan 2026 14:57:53 -0800 Subject: [PATCH 238/506] rm some out-of-date docs; update out-of-date docstr --- lib/spack/docs/configuration.rst | 46 -------------------------------- lib/spack/spack/util/path.py | 7 +++-- 2 files changed, 3 insertions(+), 50 deletions(-) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 33083ae9956504..a4ff3f4db8f943 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -679,7 +679,6 @@ Spack also, by default, keeps various caches and user data in ``~/.config/spack` Spack provides three environment variables that allow you to override or opt out of configuration locations: * ``SPACK_USER_CONFIG_PATH``: Override the path to use for the ``user`` scope (``~/.config/spack`` by default). - Note that ``XDG_CONFIG_HOME`` has the same effect, but with lower precedence. * ``SPACK_SYSTEM_CONFIG_PATH``: Override the path to use for the ``system`` scope (``/etc/spack`` by default). * ``SPACK_DISABLE_LOCAL_CONFIG``: Set this environment variable to completely disable **all** configurations from the system and user directories. Spack will then only consider its own defaults and ``site`` configuration locations. @@ -694,48 +693,3 @@ With these settings, if you want to isolate Spack in a CI environment, you can d $ export SPACK_DISABLE_LOCAL_CONFIG=true $ export SPACK_USER_CACHE_PATH=/tmp/spack - -.. _xdg_overrides: - -Overriding default paths with XDG variables -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -While Spack will by default use locations within the ``$spack`` and the user's home directory, it can now store all state in locations goverened by XDG variables. -Many default storage locations have changed in 1.0, so this section will summarize the changes, and how XDG variables can help users organize their Spack artifacts. - -``XDG_DATA_HOME`` is used to store long-lived data. -Its default value is ``$HOME/.local/share``. -The following items are always stored by default using ``$XDG_DATA_HOME``: - -* Modules: ``$XDG_DATA_HOME/spack/`` -* Package indices: ``$XDG_DATA_HOME/$spack_instance_id/spack`` - -The data listed below will be placed in ``$spack`` by default. -However, if ``XDG_DATA_HOME`` is set, they will be stored under that path. -The default values for these are as follows: - -* Source caches: ``$spack/var/spack/cache`` or ``$XDG_DATA_HOME/spack/downloads``. -* The install tree: ``$spack/opt/spack`` or ``$XDG_DATA_HOME/spack/installs``. -* Environment management: ``$spack/var/spack/environtments`` or ``$XDG_DATA_HOME/spack/environments`` - -``XDG_STATE_HOME`` is used to store data that is useful if persistent, but not integral to Spack's functionality. -If not defined, its default value is ``~/.local/state``. -``misc_cache`` is placed by default using this variable. - -* ``misc_cache``: ``$XDG_STATE_HOME/$spack_instance_id/spack`` - -``XDG_CACHE_HOME`` is used to store temporary data. -If not defined, its default value is ``~/.cache``. -Build stages are placed by default using this variable. - -* Build stages: ``$XDG_CACHE_HOME/spack`` - -The user configuration scope's files will be located with ``XDG_CONFIG_HOME``. -Its default value is ``~/.config``. - -* User configuration scope: ``$XDG_CONFIG_HOME/spack`` - -Spack also includes the variables ``SPACK_DATA_HOME``, ``SPACK_CONFIG_HOME``, and ``SPACK_STATE_HOME`` that map directly to the XDG variables described above. -They work the same way, but have higher precedence than the XDG variables. -If the Spack-specific environment variables *are not* defined, Spack will uses the XDG variables with a suffix of "/spack" to define ``$spack_state_home``, ``$spack_data_home``, and ``spack_cache_home``. -If they are defined, they are used directly without any additional suffix. diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index dc62325479493e..bb909bb692344a 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -175,10 +175,9 @@ def substitute_config_variables(path): - $default_install_root Where installs go by default - $default_envs_root Where environments are managed by default - $default_download_root Where downloads go by default - - $spack_data_home SPACK_DATA_HOME, XDG_DATA_HOME, or its default - - $spack_cache_home SPACK_CACHE_HOME, XDG_CACHE_HOME, or its default - - $spack_state_home SPACK_STATE_HOME, XDG_STATE_HOME, or its default - - $spack_config_home SPACK_CONFIG_HOME, XDG_CONFIG_HOME, or its default + - $data_home SPACK_DATA_HOME, XDG_DATA_HOME, or its default + - $cache_home SPACK_CACHE_HOME, XDG_CACHE_HOME, or its default + - $state_home SPACK_STATE_HOME, XDG_STATE_HOME, or its default - $spack_instance_id Hash that distinguishes Spack instances on the filesystem - $architecture The spack architecture triple for the current system - $arch The spack architecture triple for the current system From 1cf6a6c18ef9523b86a713988423476ad46a8e32 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Jan 2026 15:13:28 -0800 Subject: [PATCH 239/506] fix references in mirrors docs; rm overly-specific reference to scope in repositories docs --- lib/spack/docs/environments.rst | 2 +- lib/spack/docs/mirrors.rst | 2 +- lib/spack/docs/repositories.rst | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index c05a783170aad6..4a4c7af87caba2 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -63,7 +63,7 @@ An environment is created by: $ spack env create myenv -The directory ``$SPACK_ROOT/opt/data/environments/myenv`` is created to manage the environment. +The directory ``$data_home/envs/myenv`` is created to manage the environment. .. note:: diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index 1fe42916452aa5..308184ce3bc8cd 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -240,7 +240,7 @@ Local Default Cache Spack caches resources that are downloaded as part of installations. The cache is a valid Spack mirror: it uses the same directory structure and naming scheme as other Spack mirrors (so it can be copied anywhere and referenced with a URL like other mirrors). -The mirror is maintained locally (within the Spack installation directory) at :file:`$spack/opt/data`. +The mirror is maintained locally (within the Spack installation directory) at :file:`$data_home/downloads`. It is always enabled (and is always searched first when attempting to retrieve files for an installation) but can be cleared with ``spack clean --misc-cache``; the cache directory can also be deleted manually without issue. Caching includes retrieved tarball archives and source control repositories, but only resources with an associated digest or commit ID (e.g. a revision number for SVN) will be cached. diff --git a/lib/spack/docs/repositories.rst b/lib/spack/docs/repositories.rst index 0bb09c4d2239f7..f8dcc2f4b10a30 100644 --- a/lib/spack/docs/repositories.rst +++ b/lib/spack/docs/repositories.rst @@ -148,9 +148,9 @@ For example, to use ``~/custom_packages_clone`` for ``my_remote_repo``: destination: ~/custom_packages_clone -Spack uses the ``repos.yaml`` file in ``~/.config/spack`` (and :ref:`elsewhere `) to find repositories. +Spack uses the ``repos`` config section to find repositories. Note that the ``repos.yaml`` configuration file is distinct from the ``repo.yaml`` file in each repository. -For more on the YAML format, and on how configuration file precedence works in Spack, see :ref:`configuration `. +For more on the YAML format, where configuration files live, and on how configuration file precedence works in Spack, see :ref:`configuration `. If the ``git`` URL is defined in a lower-precedence configuration (like Spack's defaults for ``builtin``), you only need to specify the ``destination`` in your user-level ``repos.yaml``. Spack can make the configuration changes for you using ``spack repo set --destination ~/spack-packages builtin``, or you can directly edit your ``repos.yaml`` file: From 774f5d426672c29dc0420dafdaa2bd49b19080e8 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Jan 2026 16:34:20 -0800 Subject: [PATCH 240/506] updating docs --- etc/spack/defaults/base/config.yaml | 32 ++++++++++++----------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index 8d34768c654519..90b58ede91659d 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -5,12 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by editing files in: -# -# * $SPACK_ROOT/etc/spack/ - Spack instance settings -# * ~/.config/spack/ - user settings -# * $SPACK_ROOT/etc/spack/site - Spack "site" settings. Like instance settings -# but lower priority then user settings. +# Users can override part or all of these settings by defining their +# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). # # ------------------------------------------------------------------------- config: @@ -18,10 +14,10 @@ config: # You can use $spack here to refer to the root of the spack instance. install_tree: # If set to a non-default value, this setting has precedence. - # This default value tells spack to check (in order of precedence): - # $SPACK_DATA_HOME/installs - # $XDG_DATA_HOME/spack/installs - # $spack/opt/spack + # This default value tells spack to + # * Prefer $data_home/installs + # * The old install location is $spack/opt/spack: Spack will use this old + # location if the new location is empty and the old one isn't root: $default_install_root projections: all: "{architecture.platform}-{architecture.target}/{name}-{version}-{hash}" @@ -53,11 +49,9 @@ config: # to specify `$user_cache_path/stage`, which ensures each user builds in their # home directory. # - # A more traditional path uses the value of `$spack/var/spack/stage`, which - # builds directly inside Spack's instance without staging them in a - # temporary space. Problems with specifying a path inside a Spack instance - # are that it precludes its use as a system package and its ability to be - # pip installable. + # If you do Spack builds inside of allocations and want stages to persist + # after those allocations end, you will want to override this and avoid using + # a temporary directory. # # In Spack environment files, chaining onto existing system Spack # installations, the $env variable can be used to download, cache and build @@ -88,10 +82,10 @@ config: ## Directory where spack managed environments are created and stored # If set to a non-default value, this setting has precedence. - # This default value tells spack to check (in order of precedence): - # $SPACK_DATA_HOME/environments - # $XDG_DATA_HOME/spack/environments - # $spack/var/spack/environments + # This default value tells spack to: + # * Prefer $data_home/envs + # * The old location for envs is $spack/var/spack/environments: Spack will + # use this old location if the new location is empty and the old one isn't # environments_root: $default_envs_root From 85a96d7c8d28b3279139cb21f4188680a32577f6 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Jan 2026 16:49:01 -0800 Subject: [PATCH 241/506] modules docs; default_download_root unnecessary as a variable (it is ok to relocate it automatically and there is no need to use the old one if it exists) --- etc/spack/defaults/base/modules.yaml | 16 ++++------- lib/spack/spack/paths.py | 13 +++++---- lib/spack/spack/util/path.py | 43 ++++++++++++++-------------- 3 files changed, 35 insertions(+), 37 deletions(-) diff --git a/etc/spack/defaults/base/modules.yaml b/etc/spack/defaults/base/modules.yaml index 26a085b4a5c52b..72e949dc5db08c 100644 --- a/etc/spack/defaults/base/modules.yaml +++ b/etc/spack/defaults/base/modules.yaml @@ -5,13 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/modules.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.config/spack/modules.yaml +# Users can override these settings by defining their +# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). # ------------------------------------------------------------------------- modules: # This maps paths in the package install prefix to environment variables @@ -38,9 +33,10 @@ modules: default: # Where to install modules # If set to a non-default value, this setting has precedence. - # This default value tells spack to check (in order of precedence): - # $spack/share/spack/{modules/lmod} if previously used by an older version of spack - # $user_cache_path/modules/{modules/lmod} otherwise + # This default value tells spack to: + # * Prefer $data_home/ (e.g. $data_home/lmod) + # * The old module locations are $share/spack/: Spack will use + # this old location if the new location is empty and the old one isn't roots: tcl: $modules_base/modules lmod: $modules_base/lmod diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 2c0d1de2163657..e4e1dcb0b875c1 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -137,13 +137,16 @@ def gpg_keys_path(self): @property def modules_base(self): - modules_base = None + # This is similar to logic _fallback_old_location_if_used, but this + # moves the modules base if any component (typically one of lmod or + # tcl) has been relocated, so is examining one-layer deeper + for module_dir in ["lmod", "modules"]: + if dir_is_occupied(os.path.join(self.data_home, module_dir)): + return self.data_home for module_dir in ["lmod", "modules"]: if dir_is_occupied(os.path.join(self.base.share_path, module_dir)): - modules_base = self.base.share_path - if not modules_base: - modules_base = self.data_home - return modules_base + return self.base.share_path + return self.data_home @property def default_misc_cache_path(self): diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index bb909bb692344a..5c7782f803e507 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -71,7 +71,7 @@ def replacements(): "user_cache_path": lambda: paths.user_cache_path, "default_install_root": lambda: paths.default_install_location, "default_envs_root": lambda: paths.default_envs_path, - "default_download_root": lambda: paths.default_fetch_cache_path, + "modules_base": lambda: paths.modules_base, "data_home": lambda: paths.data_home, "cache_home": lambda: paths.cache_home, "state_home": lambda: paths.state_home, @@ -167,27 +167,26 @@ def substitute_config_variables(path): Spack allows paths in configs to have some placeholders, as follows: - - $env The active Spack environment. - - $spack The Spack instance's prefix - - $tempdir Default temporary directory returned by tempfile.gettempdir() - - $user The current user's username - - $user_cache_path The user cache directory ($XDG_DATA_HOME, unless overridden) - - $default_install_root Where installs go by default - - $default_envs_root Where environments are managed by default - - $default_download_root Where downloads go by default - - $data_home SPACK_DATA_HOME, XDG_DATA_HOME, or its default - - $cache_home SPACK_CACHE_HOME, XDG_CACHE_HOME, or its default - - $state_home SPACK_STATE_HOME, XDG_STATE_HOME, or its default - - $spack_instance_id Hash that distinguishes Spack instances on the filesystem - - $architecture The spack architecture triple for the current system - - $arch The spack architecture triple for the current system - - $platform The spack platform for the current system - - $os The OS of the current system - - $operating_system The OS of the current system - - $target The ISA target detected for the system - - $target_family The family of the target detected for the system - - $date The current date (YYYY-MM-DD) - - $spack_short_version The spack short version + - $env The active Spack environment. + - $spack The Spack instance's prefix + - $tempdir Default temporary directory returned by tempfile.gettempdir() + - $user The current user's username + - $user_cache_path The user cache directory ($XDG_DATA_HOME, unless overridden) + - $default_install_root Where installs go by default + - $default_envs_root Where environments are managed by default + - $data_home SPACK_DATA_HOME, XDG_DATA_HOME, or its default + - $cache_home SPACK_CACHE_HOME, XDG_CACHE_HOME, or its default + - $state_home SPACK_STATE_HOME, XDG_STATE_HOME, or its default + - $spack_instance_id Hash that distinguishes Spack instances on the filesystem + - $architecture The spack architecture triple for the current system + - $arch The spack architecture triple for the current system + - $platform The spack platform for the current system + - $os The OS of the current system + - $operating_system The OS of the current system + - $target The ISA target detected for the system + - $target_family The family of the target detected for the system + - $date The current date (YYYY-MM-DD) + - $spack_short_version The spack short version These are substituted case-insensitively into the path, and users can use either ``$var`` or ``${var}`` syntax for the variables. $env is only From 405e1d74bad809004eac8afc37412ffbabf5ece0 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Jan 2026 17:06:45 -0800 Subject: [PATCH 242/506] Avoid mentionig specific scopes in default configs Also: upstreams docs update --- etc/spack/defaults/base/packages.yaml | 9 ++------- etc/spack/defaults/base/repos.yaml | 9 ++------- etc/spack/defaults/darwin/modules.yaml | 9 ++------- etc/spack/defaults/darwin/packages.yaml | 9 ++------- etc/spack/defaults/linux/modules.yaml | 9 ++------- etc/spack/defaults/windows/packages.yaml | 9 ++------- lib/spack/docs/chain.rst | 10 +++++----- 7 files changed, 17 insertions(+), 47 deletions(-) diff --git a/etc/spack/defaults/base/packages.yaml b/etc/spack/defaults/base/packages.yaml index f347e6258de189..7162f2c3e6b0f3 100644 --- a/etc/spack/defaults/base/packages.yaml +++ b/etc/spack/defaults/base/packages.yaml @@ -5,13 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/packages.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.config/spack/packages.yaml +# Users can override these settings by defining their +# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). # ------------------------------------------------------------------------- packages: all: diff --git a/etc/spack/defaults/base/repos.yaml b/etc/spack/defaults/base/repos.yaml index 57c9c9321a1c79..2ac9c78c4c5d51 100644 --- a/etc/spack/defaults/base/repos.yaml +++ b/etc/spack/defaults/base/repos.yaml @@ -2,13 +2,8 @@ # This is the default spack repository configuration. It includes the # builtin spack package repository. # -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/repos.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.config/spack/repos.yaml +# Users can override these settings by defining their +# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). # ------------------------------------------------------------------------- repos: builtin: diff --git a/etc/spack/defaults/darwin/modules.yaml b/etc/spack/defaults/darwin/modules.yaml index 9192f8cc76f5d1..876feed8998191 100644 --- a/etc/spack/defaults/darwin/modules.yaml +++ b/etc/spack/defaults/darwin/modules.yaml @@ -5,13 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/modules.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.config/spack/modules.yaml +# Users can override these settings by defining their +# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). # ------------------------------------------------------------------------- modules: prefix_inspections: diff --git a/etc/spack/defaults/darwin/packages.yaml b/etc/spack/defaults/darwin/packages.yaml index 60dfaf8279af16..5c829934590e1b 100644 --- a/etc/spack/defaults/darwin/packages.yaml +++ b/etc/spack/defaults/darwin/packages.yaml @@ -5,13 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/packages.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.config/spack/packages.yaml +# Users can override these settings by defining their +# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). # ------------------------------------------------------------------------- packages: all: diff --git a/etc/spack/defaults/linux/modules.yaml b/etc/spack/defaults/linux/modules.yaml index 947ea7dc360049..cb5511f76940ba 100644 --- a/etc/spack/defaults/linux/modules.yaml +++ b/etc/spack/defaults/linux/modules.yaml @@ -5,12 +5,7 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/modules.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.config/spack/modules.yaml +# Users can override these settings by defining their +# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). # ------------------------------------------------------------------------- modules: {} diff --git a/etc/spack/defaults/windows/packages.yaml b/etc/spack/defaults/windows/packages.yaml index 71e1bf9b278fcd..67d38e01480a25 100644 --- a/etc/spack/defaults/windows/packages.yaml +++ b/etc/spack/defaults/windows/packages.yaml @@ -5,13 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/packages.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.config/spack/packages.yaml +# Users can override these settings by defining their +# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). # ------------------------------------------------------------------------- packages: all: diff --git a/lib/spack/docs/chain.rst b/lib/spack/docs/chain.rst index e7d92289b35697..b1e1c7f698b64a 100644 --- a/lib/spack/docs/chain.rst +++ b/lib/spack/docs/chain.rst @@ -16,12 +16,12 @@ To register the other Spack instance, you can add it as an entry to ``upstreams. .. code-block:: yaml upstreams: - spack-instance-1: - install_tree: /path/to/other/spack/opt/spack - spack-instance-2: - install_tree: /path/to/another/spack/opt/spack + install-tree-1: + install_tree: /path/to/other/dir/containing/.spack-db + install-tree-2: + install_tree: /path/to/another/dir/containing/.spack-db -The ``install_tree`` must point to the ``opt/data/installs`` directory inside of the Spack base directory, or the location of the ``install_tree`` defined in :ref:`config.yaml `. +The ``install_tree`` must point to a directory containing ``.spack-db``, for example ``install_tree`` defined in :ref:`config.yaml `. Once the upstream Spack instance has been added, ``spack find`` will automatically check the upstream instance when querying installed packages, and new package installations for the local Spack installation will use any dependencies that are installed in the upstream instance. From 130ba9be6d8dce19e1b6e8a69e5decf10cae46de Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Jan 2026 17:30:02 -0800 Subject: [PATCH 243/506] expanduser on Windoes doesnt appear to fall back on HOME but rather HOMEPATH --- lib/spack/spack/test/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 2445f9928f4270..2f00f06f427016 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -32,8 +32,8 @@ def paths_base_empty_old_install(): os.environ.pop("USERPROFILE", None) os.environ.pop("HOMEDRIVE", None) os.environ.pop("HOMEPATH", None) + os.environ["HOMEPATH"] = home_prefix - os.environ["HOME"] = home_prefix p1 = SpackPaths(paths_base_empty_old_install()) assert p1.default_install_location == str( pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs" From 93dd905d9810f851f3deb0ed3fdcf94d7b83c146 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Jan 2026 17:48:30 -0800 Subject: [PATCH 244/506] handle expanduser on linux as well in unit test --- lib/spack/spack/test/paths.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 2f00f06f427016..4e35383346aacb 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -34,6 +34,9 @@ def paths_base_empty_old_install(): os.environ.pop("HOMEPATH", None) os.environ["HOMEPATH"] = home_prefix + # For expanduser on Linux + os.environ["HOME"] = home_prefix + p1 = SpackPaths(paths_base_empty_old_install()) assert p1.default_install_location == str( pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs" From 2f7097e02580fee579e9e2be5d7e133a621e0a56 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Jan 2026 17:51:37 -0800 Subject: [PATCH 245/506] repos_dir is not a paths var anymore (on that note, how are we cleaning .pyo files from package repos?) --- lib/spack/spack/test/cmd/clean.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/test/cmd/clean.py b/lib/spack/spack/test/cmd/clean.py index 7462507dcdb249..883890d52a9575 100644 --- a/lib/spack/spack/test/cmd/clean.py +++ b/lib/spack/spack/test/cmd/clean.py @@ -93,17 +93,13 @@ def _check_files(directory): assert not os.path.exists(fs.join_path(directory, "__pycache__")) source_dir = fs.join_path(tmp_path, "lib", "spack", "spack") - repos_dir = fs.join_path(tmp_path, "var", "spack", "repos") - for d in [source_dir, repos_dir]: - _setup_files(d) + _setup_files(source_dir) # spack.cmd.clean references paths from spack.paths: we want to # update them for the duration of this test. override_path("lib_path", source_dir) - override_path("repos_path", repos_dir) spack.cmd.clean.remove_python_cache() - for d in [source_dir, repos_dir]: - _check_files(d) + _check_files(source_dir) From 8472e60a39fb6ae8fcd443a78cd9225242dc5283 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Jan 2026 17:59:36 -0800 Subject: [PATCH 246/506] no longer using XDG_CONFIG_HOME - remove that doc --- lib/spack/docs/config_yaml.rst | 2 +- lib/spack/docs/configuration.rst | 12 ------------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index 9fb11515d24c47..798e686c987cdf 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -91,7 +91,7 @@ By default, Spack's ``build_stage`` is configured like this: This can be an ordered list of paths that Spack should search when trying to find a temporary directory for the build stage. The list is searched in order, and Spack will use the first directory to which it has write access. -Specifying `$spack_cache_home` first will ensure each user builds in their home directory, or wherever the user overrides ``XDG_CACHE_HOME`` to be - see :ref:`xdg_overrides` and :ref:`config-file-variables` for more on ``$tempdir``, XDG variables, and ``$spack``. +Specifying `$cache_home` first will ensure each user builds in their home directory, or wherever the user overrides ``XDG_CACHE_HOME`` to be - see :ref:`config-file-variables` for more on ``$tempdir``, XDG variables, and ``$spack``. When Spack builds a package, it creates a temporary directory within the ``build_stage``. After the package is successfully installed, Spack deletes the temporary directory it used to build. diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index a4ff3f4db8f943..c55ccbc1858d46 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -140,18 +140,6 @@ If multiple scopes are provided: #. Each must be preceded with the ``--config-scope`` or ``-C`` flag. #. They must be ordered from lowest to highest precedence. -.. _scopes-and-xdg-compliance: - -^^^^^^^^^^^^^^^^^^^^^^^^^ -Scopes and XDG Compliance -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Spack respects XDG variables, and the search location for user configuration files can be affected by them. -When ``XDG_CONFIG_HOME`` is not defined, Spack assumes the XDG default value of ``~/.config``, and will apply user and this-spack configuration files located in ``~/.config/spack``. -Defining ``XDG_CONFIG_HOME`` will change where Spack searches for configuration files. -To override this behavior, define ``SPACK_USER_CONFIG_PATH`` to be the desired path. -For more information, see the :ref:`xdg_overrides` for more details. - """"""""""""""""""""""""""""""""""""""""""" Example: scopes for release and development From a7fd5decdc10e1bac66a32ce382e18fecf48d355 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Jan 2026 22:58:19 -0800 Subject: [PATCH 247/506] relocate xdg logic into paths_base; make config include cache configurable with SPACK_ and XDG_ state home env vars (but not with config vars) --- lib/spack/spack/config.py | 4 +++- lib/spack/spack/paths.py | 26 -------------------- lib/spack/spack/paths_base.py | 41 ++++++++++++++++++++++++++++++-- lib/spack/spack/test/conftest.py | 2 +- 4 files changed, 43 insertions(+), 30 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index a089bdee21a27d..16bb48bae9a53b 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -138,7 +138,9 @@ def _include_cache_location(): """Location to cache included configuration files.""" - return os.path.join(spack.paths_base.default_xdg_cache_home, "includes") + # This is always the default value of XDG_STATE_HOME: it is not + # influenced by XDG_STATE_HOME or any configuration values in Spack + return os.path.join(spack.paths_base.locations.env_based_state_home, "includes") class ConfigScope: diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index e4e1dcb0b875c1..84a28bedee12d0 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -8,39 +8,14 @@ throughout Spack and should bring in a minimal number of external dependencies. """ -import itertools import os import pathlib -from enum import Enum import spack.config as config import spack.paths_base as paths_base import spack.util.hash as hash -class XDG_vars(Enum): - config_home = "XDG_CONFIG_HOME" - state_home = "XDG_STATE_HOME" - data_home = "XDG_DATA_HOME" - cache_home = "XDG_CACHE_HOME" - - -class XDG_overrides(Enum): - config_home = "SPACK_CONFIG_HOME" - state_home = "SPACK_STATE_HOME" - data_home = "SPACK_DATA_HOME" - cache_home = "SPACK_CACHE_HOME" - - -# This is for tests that want to clean the environment of XDG_ variables that -# affect spack behavior (and the corresponding SPACK_ overrides). Note that -# these vars will affect .default_test_path for the running instance, but -# the unit tests will not see the env vars -def _unset_xdg_vars(env): - for xdg_var in itertools.chain(XDG_vars, XDG_overrides): - env.pop(xdg_var.value, None) - - def dir_is_occupied(x, except_for=None): x = pathlib.Path(x) except_for = except_for or set() @@ -265,6 +240,5 @@ def _fallback_old_location_if_used(self, old_location, new_location): mock_packages_path = locations.mock_packages_path mock_gpg_data_path = locations.mock_gpg_data_path mock_gpg_keys_path = locations.mock_gpg_keys_path -default_xdg_cache_home = locations.default_xdg_cache_home system_config_path = locations.system_config_path user_config_path = locations.user_config_path diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index 561dac93f7194a..284bfc4a6482eb 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -2,12 +2,37 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import itertools import os +from enum import Enum from pathlib import PurePath import spack.llnl.util.filesystem +class XDG_vars(Enum): + config_home = "XDG_CONFIG_HOME" + state_home = "XDG_STATE_HOME" + data_home = "XDG_DATA_HOME" + cache_home = "XDG_CACHE_HOME" + + +class XDG_overrides(Enum): + config_home = "SPACK_CONFIG_HOME" + state_home = "SPACK_STATE_HOME" + data_home = "SPACK_DATA_HOME" + cache_home = "SPACK_CACHE_HOME" + + +# This is for tests that want to clean the environment of XDG_ variables that +# affect spack behavior (and the corresponding SPACK_ overrides). Note that +# these vars will affect .default_test_path for the running instance, but +# the unit tests will not see the env vars +def _unset_xdg_vars(env): + for xdg_var in itertools.chain(XDG_vars, XDG_overrides): + env.pop(xdg_var.value, None) + + class SpackPathsBase: def __init__(self, _prefix=None): #: This file lives in $prefix/lib/spack/spack/__file__ @@ -56,7 +81,7 @@ def __init__(self, _prefix=None): self.old_gpg_path = os.path.join(self.prefix, "opt", "spack", "gpg") self.old_gpg_keys_path = os.path.join(self.var_path, "gpg") - self.default_xdg_cache_home = os.path.join("~", ".cache", "spack") + self.default_xdg_state_home = os.path.join("~", ".state", "spack") #: User configuration location self.user_config_path = os.path.expanduser( @@ -68,6 +93,18 @@ def __init__(self, _prefix=None): os.getenv("SPACK_SYSTEM_CONFIG_PATH") or os.sep + os.path.join("etc", "spack") ) + @property + def env_based_state_home(self): + """Spack has config-based logic for choosing a home for most state, but + this is specifically for caching state related to the config system + itself: it is based entirely on env vars and not on configuration + variables. + """ + override = lambda: os.environ.get(XDG_overrides.state_home.value) + xdg = lambda: os.environ.get(XDG_vars.state_home.value) + default = lambda: self.default_xdg_state_home + return override() or xdg() or default() + locations = SpackPathsBase() prefix = locations.prefix @@ -94,7 +131,7 @@ def __init__(self, _prefix=None): mock_packages_path = locations.mock_packages_path mock_gpg_data_path = locations.mock_gpg_data_path mock_gpg_keys_path = locations.mock_gpg_keys_path -default_xdg_cache_home = locations.default_xdg_cache_home +default_xdg_state_home = locations.default_xdg_state_home system_config_path = locations.system_config_path user_config_path = locations.user_config_path diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 03646cc2951b42..0b6b283296380f 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -411,7 +411,7 @@ def clean_user_environment(): @pytest.fixture(scope="session", autouse=True) def clear_xdg_vars(): saved = os.environ.copy() - spack.paths._unset_xdg_vars(os.environ) + spack.paths_base._unset_xdg_vars(os.environ) yield os.environ.update(saved) From a9be31035431396d732870ec90d9cb45991b2089 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Jan 2026 23:05:45 -0800 Subject: [PATCH 248/506] collect windows/linux home env logic into fixture --- lib/spack/spack/paths_base.py | 2 +- lib/spack/spack/test/paths.py | 32 +++++++++++++++++++++----------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index 284bfc4a6482eb..0357da5b1e0e90 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -98,7 +98,7 @@ def env_based_state_home(self): """Spack has config-based logic for choosing a home for most state, but this is specifically for caching state related to the config system itself: it is based entirely on env vars and not on configuration - variables. + variables. It is not affected by `config:locations:disable_env`. """ override = lambda: os.environ.get(XDG_overrides.state_home.value) xdg = lambda: os.environ.get(XDG_vars.state_home.value) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 4e35383346aacb..56cbff8c687f24 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -5,6 +5,8 @@ import os import pathlib +import pytest + import spack.config from spack.paths import SpackPaths from spack.paths_base import SpackPathsBase @@ -15,7 +17,22 @@ def _ensure_dir(pathlike): return str(pathlike) -def test_install_location(working_env, tmp_path, mutable_config): +@pytest.fixture +def set_home(): + def _set_home(val): + # Clear some env vars that can interfere w/ expanduser(~) on Windows + os.environ.pop("USERPROFILE", None) + os.environ.pop("HOMEDRIVE", None) + os.environ.pop("HOMEPATH", None) + os.environ["HOMEPATH"] = val + + # For expanduser on Linux + os.environ["HOME"] = val + + yield _set_home + + +def test_install_location(working_env, tmp_path, mutable_config, set_home): # If prior default install dir inside spack prefix does not # exist, place installs in $HOME base_prefix = _ensure_dir(tmp_path / "spack-root") @@ -28,14 +45,7 @@ def paths_base_empty_old_install(): pb.old_install_path = empty_dir return pb - # Clear some env vars that can interfere w/ expanduser(~) on Windows - os.environ.pop("USERPROFILE", None) - os.environ.pop("HOMEDRIVE", None) - os.environ.pop("HOMEPATH", None) - os.environ["HOMEPATH"] = home_prefix - - # For expanduser on Linux - os.environ["HOME"] = home_prefix + set_home(home_prefix) p1 = SpackPaths(paths_base_empty_old_install()) assert p1.default_install_location == str( @@ -114,9 +124,9 @@ def test_user_cache_path_is_overridable(working_env, tmp_path): assert p1.package_repos_path == str(pathlib.Path(redirect_usr_cache) / "package_repos") -def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmp_path): +def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmp_path, set_home): base_prefix = _ensure_dir(tmp_path / "base-prefix") - os.environ["HOME"] = base_prefix + set_home(base_prefix) new_default_gpg_base = pathlib.Path(base_prefix) / ".local" / "share" / "spack" From 8e2b3468318d973831798b12ca1b78ca467557b1 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Jan 2026 23:21:31 -0800 Subject: [PATCH 249/506] rm reference to default xdg value --- lib/spack/spack/paths_base.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index 0357da5b1e0e90..3fcb3b73e2a121 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -81,8 +81,6 @@ def __init__(self, _prefix=None): self.old_gpg_path = os.path.join(self.prefix, "opt", "spack", "gpg") self.old_gpg_keys_path = os.path.join(self.var_path, "gpg") - self.default_xdg_state_home = os.path.join("~", ".state", "spack") - #: User configuration location self.user_config_path = os.path.expanduser( os.getenv("SPACK_USER_CONFIG_PATH") or os.path.join("~", ".config", "spack") @@ -102,7 +100,7 @@ def env_based_state_home(self): """ override = lambda: os.environ.get(XDG_overrides.state_home.value) xdg = lambda: os.environ.get(XDG_vars.state_home.value) - default = lambda: self.default_xdg_state_home + default = lambda: os.path.expanduser(os.path.join("~", ".state", "spack")) return override() or xdg() or default() @@ -131,7 +129,6 @@ def env_based_state_home(self): mock_packages_path = locations.mock_packages_path mock_gpg_data_path = locations.mock_gpg_data_path mock_gpg_keys_path = locations.mock_gpg_keys_path -default_xdg_state_home = locations.default_xdg_state_home system_config_path = locations.system_config_path user_config_path = locations.user_config_path From 8bf16b1efc01b5090a9b5dd75ebec26997e78b0c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 13 Jan 2026 10:02:03 -0800 Subject: [PATCH 250/506] xdg unset actions should actually just be in paths tests (in case you want to change test stage root) --- lib/spack/spack/test/conftest.py | 8 -------- lib/spack/spack/test/paths.py | 9 +++++++++ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 0b6b283296380f..29d707169b900e 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -408,14 +408,6 @@ def clean_user_environment(): os.environ[ev.spack_env_var] = spack_env_value -@pytest.fixture(scope="session", autouse=True) -def clear_xdg_vars(): - saved = os.environ.copy() - spack.paths_base._unset_xdg_vars(os.environ) - yield - os.environ.update(saved) - - # # Make sure global state of active env does not leak between tests. # diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 56cbff8c687f24..9d356c31fa227a 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -8,6 +8,7 @@ import pytest import spack.config +import spack.paths_base from spack.paths import SpackPaths from spack.paths_base import SpackPathsBase @@ -17,6 +18,14 @@ def _ensure_dir(pathlike): return str(pathlike) +@pytest.fixture(scope="module", autouse=True) +def clear_xdg_vars(): + saved = os.environ.copy() + spack.paths_base._unset_xdg_vars(os.environ) + yield + os.environ.update(saved) + + @pytest.fixture def set_home(): def _set_home(val): From 91019ebdb05b82d0225cf5be599f40e6f9722f28 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 13 Jan 2026 10:13:42 -0800 Subject: [PATCH 251/506] typos in base/default config yamls --- etc/spack/defaults/base/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index 90b58ede91659d..938c5a789fc6ac 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -67,7 +67,7 @@ config: # identifies Spack staging to avoid accidentally wiping out non-Spack work. build_stage: - $tempdir/$user/spack-stage - - $spack_cache_home/stage + - $cache_home/stage # Directory in which to run tests and store test results. # Tests will be stored in directories named by date/time and package @@ -91,7 +91,7 @@ config: # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` - misc_cache: $spack_state_home/$spack_instance_id/cache + misc_cache: $state_home/$spack_instance_id/cache # Abort downloads after this many seconds if not data is received. From be5e081271f89b34e692ecb144697b006568dcbe Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 13 Jan 2026 10:59:39 -0800 Subject: [PATCH 252/506] rm unused import --- lib/spack/spack/caches.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/spack/spack/caches.py b/lib/spack/spack/caches.py index cc0cc86b44a8d1..889fe17ff6d870 100644 --- a/lib/spack/spack/caches.py +++ b/lib/spack/spack/caches.py @@ -20,8 +20,6 @@ def misc_cache_location(): Currently the ``MISC_CACHE`` stores indexes for virtual dependency providers and for which packages provide which tags. """ - import spack.paths - path = spack.config.get("config:misc_cache", paths.default_misc_cache_path) return spack.util.path.canonicalize_path(path) From 21394cdb1707236b9eb00875f802dcec4f4a7c82 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 13 Jan 2026 11:02:10 -0800 Subject: [PATCH 253/506] update doc - removed scope --- lib/spack/docs/config_yaml.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index 798e686c987cdf..88cf4af72abea2 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -18,7 +18,7 @@ You can see the default settings by looking at ``etc/spack/defaults/config.yaml` .. literalinclude:: _spack_root/etc/spack/defaults/base/config.yaml :language: yaml -These settings can be overridden in ``etc/spack/config.yaml``, or ``~/.config/spack/config.yaml``, or ``~/.config/spack/$spack_instance_id/config.yaml``. +These settings can be overridden in ``etc/spack/config.yaml``, or ``~/.config/spack/config.yaml``, or in an active environment. See :ref:`configuration-scopes` for details. ``install_tree:root`` From dc67197053f762b6cfd26ea09ed3a7f1850f2419 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 13 Jan 2026 11:51:12 -0800 Subject: [PATCH 254/506] added docs for data_home and similar config vars --- lib/spack/docs/config_yaml.rst | 2 +- lib/spack/docs/configuration.rst | 45 +++++++++++++++++++++++++++----- lib/spack/spack/util/path.py | 2 +- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index 88cf4af72abea2..7b2d24729af57c 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -86,7 +86,7 @@ By default, Spack's ``build_stage`` is configured like this: build_stage: - $tempdir/$user/spack-stage - - $spack_cache_home/stage + - $cache_home/stage This can be an ordered list of paths that Spack should search when trying to find a temporary directory for the build stage. The list is searched in order, and Spack will use the first directory to which it has write access. diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index c55ccbc1858d46..c0c629956d01aa 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -509,7 +509,6 @@ These are: * ``$spack``: path to the prefix of this Spack installation * ``$tempdir``: default system temporary directory (as specified in Python's `tempfile.tempdir `_ variable. * ``$user``: name of the current user -* ``$user_cache_path``: user cache directory (``~/.local/share/spack`` unless :ref:`overridden `) * ``$architecture``: the architecture triple of the current host, as detected by Spack. * ``$arch``: alias for ``$architecture``. * ``$platform``: the platform of the current host, as detected by Spack. @@ -524,17 +523,49 @@ These are: ``x86_64`` or ``aarch64``. * ``$date``: the current date in the format YYYY-MM-DD * ``$spack_short_version``: the Spack version truncated to the first components. -* ``$spack_instance_id``: a hash that distinguishes Spack instances on the filesystem. -* ``$spack_state_home``: the XDG-derived location for long-lived but not-essential cache. -* ``$spack_cache_home``: the XDG-derived location for temporary data. -* ``$default_download_root``: the location where downloads go by default. -* ``$default_envs_root``: the location where environments are managed by default. -* ``$default_install_root``: the location where installs go by default. Note that, as with shell variables, you can write these as ``$varname`` or with braces to distinguish the variable from surrounding characters: ``${varname}``. Their names are also case insensitive, meaning that ``$SPACK`` works just as well as ``$spack``. These special variables are substituted first, so any environment variables with the same name will not be used. +.. _configuration_data_variables: + +Spack-specific variables controlling data location +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Files generated and stored by spack are organized roughly into three categories: + +* Persistent, large quantities of data (e.g. installs and environments) +* Temporary (or assumed temporary) large quantities of data (e.g. stages for installs) +* Persistent caches/indices used by spack to speed up its commands (small quantities of data) + +The corresponding variables that describe where this data is placed are: + +* ``$data_home`` +* ``$cache_home`` +* ``$state_home`` +* ``$user_cache_path``: legacy variable, equivalent to ``$state_home`` + +You can refer to these variables when configuring locations for stages, misc cache, etc. Furthermore each is controlled by a fall-through scheme. For example ``$data_home`` evaluates to one of the following (highest-priority first): + +#. ``SPACK_DATA_HOME`` env var if that is set +#. ``XDG_DATA_HOME`` env var if that is set +#. ``SPACK_HOME`` env var +#. ``config:locations:data`` +#. ``config:locations:home`` +#. Under the default for ``XDG_DATA_HOME``: ``~/.local/share/spack`` + +Of particular interest is where the environments and installs are placed by Spack, because these can take up a lot of space. +Generally speaking these are controlled by ``$data_home``. +Older installs of spack placed these within ``$spack``, and fallback scheme in these cases is augmented to prefer these old locations if no data is detected in the corresponding new locations: + +* ``$default_install_root``: the location where installs go by default. + Overridden by ``config:install_tree:root``. + Prefers ``$data_home/installs``, but if there are no installs there but there are installs in the old location within ``$spack``, then the old location will be used. +* ``$default_envs_root``: the location where environments are managed by default. + Overridden by ``config:environments_root``. + Prefers ``$data_home/envs`` but if there are no envs there and there are envs in the old location in ``$spack``, then the old location will be used. + Environment variables ^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 5c7782f803e507..ef3faebe212c79 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -171,12 +171,12 @@ def substitute_config_variables(path): - $spack The Spack instance's prefix - $tempdir Default temporary directory returned by tempfile.gettempdir() - $user The current user's username - - $user_cache_path The user cache directory ($XDG_DATA_HOME, unless overridden) - $default_install_root Where installs go by default - $default_envs_root Where environments are managed by default - $data_home SPACK_DATA_HOME, XDG_DATA_HOME, or its default - $cache_home SPACK_CACHE_HOME, XDG_CACHE_HOME, or its default - $state_home SPACK_STATE_HOME, XDG_STATE_HOME, or its default + - $user_cache_path The user cache directory (same as state_home) - $spack_instance_id Hash that distinguishes Spack instances on the filesystem - $architecture The spack architecture triple for the current system - $arch The spack architecture triple for the current system From e9c1309f6fe293c4d955c09e7b65071149196365 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 13 Jan 2026 11:57:06 -0800 Subject: [PATCH 255/506] update other references in configuration doc --- lib/spack/docs/configuration.rst | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index c0c629956d01aa..84db6d8670c401 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -38,10 +38,10 @@ Here is an example ``config.yaml`` file: config: install_tree: - root: $spack/opt/spack + root: $data_home/installs build_stage: - $tempdir/$user/spack-stage - - $spack_cache_home/stage + - $cache_home/stage Each Spack configuration file is nested under a top-level section corresponding to its name. So, ``config.yaml`` starts with ``config:``, ``mirrors.yaml`` starts with ``mirrors:``, etc. @@ -312,10 +312,10 @@ If your configurations look like this: config: install_tree: - root: $spack/opt/spack + root: $data_home/installs build_stage: - $tempdir/$user/spack-stage - - $spack_cache_home/stage + - $cache_home/stage .. code-block:: yaml @@ -339,7 +339,7 @@ You can see the final, combined configuration with the ``spack config get Date: Tue, 13 Jan 2026 12:09:08 -0800 Subject: [PATCH 256/506] update other config file var references and add pointer to new doc --- lib/spack/docs/config_yaml.rst | 14 +++++--------- lib/spack/docs/configuration.rst | 2 +- lib/spack/docs/environments.rst | 6 +++--- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index 7b2d24729af57c..b8257053913543 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -25,7 +25,7 @@ See :ref:`configuration-scopes` for details. --------------------- The location where Spack will install packages and their dependencies. -The default is ``$spack/opt/data/installs``. +The default is ``$data_home/installs``. ``projections`` --------------- @@ -91,7 +91,7 @@ By default, Spack's ``build_stage`` is configured like this: This can be an ordered list of paths that Spack should search when trying to find a temporary directory for the build stage. The list is searched in order, and Spack will use the first directory to which it has write access. -Specifying `$cache_home` first will ensure each user builds in their home directory, or wherever the user overrides ``XDG_CACHE_HOME`` to be - see :ref:`config-file-variables` for more on ``$tempdir``, XDG variables, and ``$spack``. +Specifying `$cache_home` first will ensure each user builds in their home directory, or whatever the user overrides ``XDG_CACHE_HOME`` to be - see :ref:`config-file-variables` and :ref:`config-file-data-variables` for more on ``$tempdir``, XDG variables, and ``$spack``. When Spack builds a package, it creates a temporary directory within the ``build_stage``. After the package is successfully installed, Spack deletes the temporary directory it used to build. @@ -105,7 +105,7 @@ Unsuccessful builds are not deleted, but you can manually purge them with ``spac -------------------- Location to cache downloaded tarballs and repositories. -By default, these are stored in ``$spack/opt/data/downloads``. +By default, these are stored in ``$data_home/downloads``. These are stored indefinitely by default and can be purged with ``spack clean --downloads``. .. _Misc Cache: @@ -114,14 +114,10 @@ These are stored indefinitely by default and can be purged with ``spack clean -- -------------------- Temporary directory to store long-lived cache files, such as indices of packages available in repositories. -Defaults to ``~/.local/state/spack/$spack_instance_id/spack``. +Defaults to ``$state_home/$spack_instance_id/spack``. Can be purged with ``spack clean --misc-cache``. -In some cases, e.g., if you work with many Spack instances or many different versions of Spack, it makes sense to have a cache per instance or per version. -You can do that by changing the value to either: - -* ``~/.spack/$spack_instance_id/cache`` for per-instance caches, or -* ``~/.spack/$spack_short_version/cache`` for per-spack-version caches. +If you have several Spack instances with the same version and want them to share this cache, you can use ``~/.spack/$spack_short_version/cache``. ``verify_ssl`` -------------------- diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 84db6d8670c401..7599b4ea8f578a 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -528,7 +528,7 @@ Note that, as with shell variables, you can write these as ``$varname`` or with Their names are also case insensitive, meaning that ``$SPACK`` works just as well as ``$spack``. These special variables are substituted first, so any environment variables with the same name will not be used. -.. _configuration_data_variables: +.. _config_file_data_variables: Spack-specific variables controlling data location ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index 4a4c7af87caba2..421227765d4c87 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -67,10 +67,10 @@ The directory ``$data_home/envs/myenv`` is created to manage the environment. .. note:: - By default, all managed environments are stored in the ``$SPACK_ROOT/opt/data/environments`` folder. + By default, all managed environments are stored in the ``$data_home/envs`` folder. This location can be changed by setting the ``environments_root`` variable in ``config.yaml``. -Spack creates the file ``spack.yaml``, hidden directory ``.spack-env``, and ``spack.lock`` file under ``$SPACK_ROOT/opt/data/environments/environments/myenv``. +Spack creates the file ``spack.yaml``, hidden directory ``.spack-env``, and ``spack.lock`` file under ``$data_home/envs/myenv``. User interaction occurs through the ``spack.yaml`` file and the Spack commands that affect it. Metadata and, by default, the view are stored in the ``.spack-env`` directory. When the environment is concretized, Spack creates the ``spack.lock`` file with the fully configured specs and dependencies for the environment. @@ -124,7 +124,7 @@ It will also include any other files included in the environment directory, such Environment creation also accepts a full path to the file. - If the path is not under the ``$SPACK_ROOT/opt/data/environments`` directory then the source is referred to as an :ref:`independent environment `. + If the path is not under the ``config:environments_root`` directory then the source is referred to as an :ref:`independent environment `. The name of an environment can be a nested path to help organize environments via subdirectories. From f8b0226e4d0f06335855ad3d9b82e9b1b820d6aa Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 13 Jan 2026 12:19:17 -0800 Subject: [PATCH 257/506] docs formatting --- lib/spack/docs/chain.rst | 4 ++-- lib/spack/docs/configuration.rst | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/spack/docs/chain.rst b/lib/spack/docs/chain.rst index b1e1c7f698b64a..487e45b4a9510c 100644 --- a/lib/spack/docs/chain.rst +++ b/lib/spack/docs/chain.rst @@ -16,9 +16,9 @@ To register the other Spack instance, you can add it as an entry to ``upstreams. .. code-block:: yaml upstreams: - install-tree-1: + install-tree-1: install_tree: /path/to/other/dir/containing/.spack-db - install-tree-2: + install-tree-2: install_tree: /path/to/another/dir/containing/.spack-db The ``install_tree`` must point to a directory containing ``.spack-db``, for example ``install_tree`` defined in :ref:`config.yaml `. diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 7599b4ea8f578a..0bed9179e15aab 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -546,7 +546,9 @@ The corresponding variables that describe where this data is placed are: * ``$state_home`` * ``$user_cache_path``: legacy variable, equivalent to ``$state_home`` -You can refer to these variables when configuring locations for stages, misc cache, etc. Furthermore each is controlled by a fall-through scheme. For example ``$data_home`` evaluates to one of the following (highest-priority first): +You can refer to these variables when configuring locations for stages, misc cache, etc. +Furthermore each is controlled by a fall-through scheme. +For example ``$data_home`` evaluates to one of the following (highest-priority first): #. ``SPACK_DATA_HOME`` env var if that is set #. ``XDG_DATA_HOME`` env var if that is set From bd26ed1546d3fa9df2a67996ab20b71141c02fbb Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 13 Jan 2026 13:04:10 -0800 Subject: [PATCH 258/506] bad docs ref --- lib/spack/docs/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 0bed9179e15aab..ccf006b01eb4ed 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -528,7 +528,7 @@ Note that, as with shell variables, you can write these as ``$varname`` or with Their names are also case insensitive, meaning that ``$SPACK`` works just as well as ``$spack``. These special variables are substituted first, so any environment variables with the same name will not be used. -.. _config_file_data_variables: +.. _config-file-data-variables: Spack-specific variables controlling data location ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From d615ea99f4dad20c0dda6692af472e032529d1fc Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 13 Jan 2026 13:30:44 -0800 Subject: [PATCH 259/506] update tests based on review --- lib/spack/spack/test/paths.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 9d356c31fa227a..b50639bf62867c 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -123,14 +123,19 @@ def test_user_config_path_is_default_when_env_var_is_empty(working_env, tmp_path def test_user_cache_path_is_overridable(working_env, tmp_path): - redirect_usr_cache = str(pathlib.Path(tmp_path) / "redirected_usr_cache") - os.environ["SPACK_USER_CACHE_PATH"] = redirect_usr_cache + redirect1 = str(pathlib.Path(tmp_path) / "redirected_usr_cache") + os.environ["SPACK_USER_CACHE_PATH"] = redirect1 p1 = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "base-prefix"))) - assert p1.user_cache_path == redirect_usr_cache - + assert p1.user_cache_path == redirect1 # Check that things that are supposed to be bundled inside of # $user_cache_path are also relocated - assert p1.package_repos_path == str(pathlib.Path(redirect_usr_cache) / "package_repos") + assert p1.package_repos_path == str(pathlib.Path(redirect1) / "package_repos") + + # Now check that $SPACK_STATE_HOME takes precedence when both are set + redirect2 = str(pathlib.Path(tmp_path) / "redirected_usr_cache2") + os.environ["SPACK_STATE_HOME"] = redirect2 + assert p1.user_cache_path == redirect2 + assert p1.package_repos_path == str(pathlib.Path(redirect2) / "package_repos") def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmp_path, set_home): @@ -148,19 +153,31 @@ def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmp_path, set_home): p1 = SpackPaths(SpackPathsBase(base_prefix)) # Old dir exists, but is empty, so it should not be used assert p1.gpg_path == str(new_default_gpg_base / "gpg") + + # Put something in the old dir: it should now redirect (old_gpg_dir / "something").touch() p1 = SpackPaths(SpackPathsBase(base_prefix)) - # Now it should redirect assert p1.gpg_path == str(old_gpg_dir) + # But the keys are handled separately and should use the new path - assert p1.gpg_keys_path == str(new_default_gpg_base / "gpg-keys") + new_gpg_keys_dir = pathlib.Path(new_default_gpg_base / "gpg-keys") + assert p1.gpg_keys_path == str(new_gpg_keys_dir) + # Check that the keys will also redirect old_gpg_keys_dir = pathlib.Path(base_prefix) / "var" / "spack" / "gpg" old_gpg_keys_dir.mkdir(parents=True) (old_gpg_keys_dir / "something").touch() p1 = SpackPaths(SpackPathsBase(base_prefix)) assert p1.gpg_keys_path == str(old_gpg_keys_dir) + # When something is in both the new and old locations, prefer the new + new_gpg_keys_dir.mkdir(parents=True) + (new_gpg_keys_dir / "something").touch() + assert p1.gpg_keys_path == str(new_gpg_keys_dir) + + # And the gpg dir itself remains the old dir: reaffirm that + assert p1.gpg_path == str(old_gpg_dir) + def test_user_cache_path_is_default_when_env_var_is_empty(working_env, tmp_path): os.environ["SPACK_USER_CACHE_PATH"] = "" From 00bf7b28d146d3fefacce6a9afd33d8bf19cf77c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 13 Jan 2026 13:50:55 -0800 Subject: [PATCH 260/506] document disable_env --- lib/spack/docs/config_yaml.rst | 12 ++++++++++++ lib/spack/spack/schema/config.py | 2 ++ 2 files changed, 14 insertions(+) diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index b8257053913543..faf77bba25ac96 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -119,6 +119,18 @@ Can be purged with ``spack clean --misc-cache``. If you have several Spack instances with the same version and want them to share this cache, you can use ``~/.spack/$spack_short_version/cache``. +``locations`` +-------------------- + +The ``locations`` section includes variables that control where spack stores data it generates, as described in :ref:`config-file-data-variables`: + +* ``locations:data`` +* ``locations:state`` +* ``locations:cache`` +* ``locations:home`` + +The additional flag setting ``locations:disable_env`` will prevent environment variables from influencing these locations. + ``verify_ssl`` -------------------- diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py index 4e041b6fc74238..a5e242f56c75c7 100644 --- a/lib/spack/spack/schema/config.py +++ b/lib/spack/spack/schema/config.py @@ -78,10 +78,12 @@ "type": "object", "properties": { "home": {"type": "string"}, + "data": {"type": "string"}, "cache": {"type": "string"}, "state": {"type": "string"}, "disable_env": {"type": "boolean"}, }, + "additionalProperties": False, }, "install_hash_length": { "type": "integer", From 4c6717c2562ca3619c0828b6fb79862f1d819e91 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 13 Jan 2026 13:54:07 -0800 Subject: [PATCH 261/506] rm analyzers_path as globally understood path --- lib/spack/spack/paths.py | 1 - lib/spack/spack/paths_base.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 84a28bedee12d0..fa8432ead29497 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -226,7 +226,6 @@ def _fallback_old_location_if_used(self, old_location, new_location): module_path = locations.module_path vendor_path = locations.vendor_path command_path = locations.command_path -analyzers_path = locations.analyzers_path platform_path = locations.platform_path compilers_path = locations.compilers_path operating_system_path = locations.operating_system_path diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index 3fcb3b73e2a121..92378842c4ea2d 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -56,7 +56,6 @@ def __init__(self, _prefix=None): self.module_path = os.path.join(self.lib_path, "spack") self.vendor_path = os.path.join(self.module_path, "vendor") self.command_path = os.path.join(self.module_path, "cmd") - self.analyzers_path = os.path.join(self.module_path, "analyzers") self.platform_path = os.path.join(self.module_path, "platforms") self.compilers_path = os.path.join(self.module_path, "compilers") self.operating_system_path = os.path.join(self.module_path, "operating_systems") @@ -115,7 +114,6 @@ def env_based_state_home(self): module_path = locations.module_path vendor_path = locations.vendor_path command_path = locations.command_path -analyzers_path = locations.analyzers_path platform_path = locations.platform_path compilers_path = locations.compilers_path operating_system_path = locations.operating_system_path From 7d0c8d948946c1ad42ec57b521a3d4e6ac8c8d3f Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 13 Jan 2026 17:02:16 -0800 Subject: [PATCH 262/506] move xdg env vars from high priority to lowest priority (used to override SPACK_HOME and config settings) --- lib/spack/docs/configuration.rst | 2 +- lib/spack/spack/paths.py | 2 +- lib/spack/spack/test/paths.py | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index ccf006b01eb4ed..e73f678a7a5305 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -551,10 +551,10 @@ Furthermore each is controlled by a fall-through scheme. For example ``$data_home`` evaluates to one of the following (highest-priority first): #. ``SPACK_DATA_HOME`` env var if that is set -#. ``XDG_DATA_HOME`` env var if that is set #. ``SPACK_HOME`` env var #. ``config:locations:data`` #. ``config:locations:home`` +#. ``XDG_DATA_HOME`` env var if that is set #. Under the default for ``XDG_DATA_HOME``: ``~/.local/share/spack`` Of particular interest is where the environments and installs are placed by Spack, because these can take up a lot of space. diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index fa8432ead29497..56dc02a2e4f10b 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -186,10 +186,10 @@ def spack_home_cfg_check(): for check in [ spack_env_check, - xdg_env_check, spack_home_env_check, cfg_check, spack_home_cfg_check, + xdg_env_check, ]: possible_resolution = check() if possible_resolution: diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index b50639bf62867c..3dbd9c2015d231 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -63,7 +63,13 @@ def paths_base_empty_old_install(): spack.config.set("config:locations", {}) - # "config:locations:home" variable overrides default + # $XDG_DATA_HOME overrides the default + xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") + os.environ["XDG_DATA_HOME"] = xdg_data_home + p4 = SpackPaths(paths_base_empty_old_install()) + assert p4.default_install_location == str(pathlib.Path(xdg_data_home) / "spack" / "installs") + + # "config:locations:home" variable overrides the above spack_home_prefix = _ensure_dir(tmp_path / "spack-home") spack.config.set("config:locations:home", spack_home_prefix) p2 = SpackPaths(paths_base_empty_old_install()) @@ -77,12 +83,6 @@ def paths_base_empty_old_install(): p3 = SpackPaths(paths_base_empty_old_install()) assert p3.default_install_location == str(pathlib.Path(spack_data_prefix) / "installs") - # $XDG_DATA_HOME overrides all the above - xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") - os.environ["XDG_DATA_HOME"] = xdg_data_home - p4 = SpackPaths(paths_base_empty_old_install()) - assert p4.default_install_location == str(pathlib.Path(xdg_data_home) / "spack" / "installs") - # Check that $SPACK_DATA_HOME overrides all the above spack_data_home = _ensure_dir(tmp_path / "spack_data_home") os.environ["SPACK_DATA_HOME"] = spack_data_home From 40085dce09c1ab8e24a143ed8c39076be4b9eae4 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 13 Jan 2026 17:45:50 -0800 Subject: [PATCH 263/506] resolve symlinks as part of detecting writes into spack via trace --- lib/spack/spack/trace.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/trace.py b/lib/spack/spack/trace.py index 3c68dc2b208ea5..f40a4d8747886e 100644 --- a/lib/spack/spack/trace.py +++ b/lib/spack/spack/trace.py @@ -41,6 +41,17 @@ def _attempted_modify_internal(msg): warnings.warn(msg) +def _real(path): + return pathlib.Path(path).absolute().resolve() + + +_real_spack_prefix = _real(paths_base.prefix) + + +def _is_in_spack_prefix(path): + return _real_spack_prefix in _real(path).parents + + def _guard_writes(event, args): # Note: this doesn't catch files opened in "r" mode and then # later upgraded to "w" mode (e.g. our locks). I think to track @@ -58,18 +69,16 @@ def _guard_writes(event, args): return abs_path = os.path.abspath(path) intent_to_modify = bool((set(mode) & set("wax")) or "r+" in mode) - if abs_path.startswith(paths_base.prefix) and intent_to_modify: + if _is_in_spack_prefix(path) and intent_to_modify: _attempted_modify_internal(f"Open {path} in mode [{mode}]") elif event in ["shutil.copyfile", "os.rename", "shutil.move"]: _, dst = args[:2] - abs_dst = os.path.abspath(dst) - if abs_dst.startswith(paths_base.prefix): - _attempted_modify_internal(f"copy dst {abs_dst}") + if _is_in_spack_prefix(dst): + _attempted_modify_internal(f"copy dst {str(_real(dst))}") elif event == "os.mkdir": path = args[0] - abs_path = os.path.abspath(path) - if abs_path.startswith(paths_base.prefix): - _attempted_modify_internal(f"mkdir {abs_path}") + if _is_in_spack_prefix(path): + _attempted_modify_internal(f"mkdir {str(_real(path))}") def warn_writes_into_spack(): From 274207b36689ba6fc9e67ab154dbad4fddef3880 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 13 Jan 2026 17:55:22 -0800 Subject: [PATCH 264/506] style edit --- lib/spack/spack/trace.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/spack/spack/trace.py b/lib/spack/spack/trace.py index f40a4d8747886e..d932c5247b9918 100644 --- a/lib/spack/spack/trace.py +++ b/lib/spack/spack/trace.py @@ -2,7 +2,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import inspect -import os import pathlib import warnings @@ -67,7 +66,6 @@ def _guard_writes(event, args): if not isinstance(path, str): # Skip instances of open() that function like fdopen return - abs_path = os.path.abspath(path) intent_to_modify = bool((set(mode) & set("wax")) or "r+" in mode) if _is_in_spack_prefix(path) and intent_to_modify: _attempted_modify_internal(f"Open {path} in mode [{mode}]") From a4a1dd8cce54d03ce4e35e829aab3c406026d199 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Feb 2026 15:45:41 -0800 Subject: [PATCH 265/506] minor review edits --- lib/spack/docs/mirrors.rst | 4 ++-- lib/spack/spack/bootstrap/config.py | 2 +- lib/spack/spack/util/path.py | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index 308184ce3bc8cd..ee4f7e9e699733 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -222,7 +222,7 @@ To remove a mirror by name, run: Mirror precedence ----------------- -Adding a mirror really adds a line in ``~/.config/spack/mirrors.yaml``: +Adding a mirror really adds an entry in the mirrors config, by default stored in the user environment scope at ``~/.config/spack/mirrors.yaml``: .. code-block:: yaml @@ -240,7 +240,7 @@ Local Default Cache Spack caches resources that are downloaded as part of installations. The cache is a valid Spack mirror: it uses the same directory structure and naming scheme as other Spack mirrors (so it can be copied anywhere and referenced with a URL like other mirrors). -The mirror is maintained locally (within the Spack installation directory) at :file:`$data_home/downloads`. +The mirror is maintained locally at :file:`$data_home/downloads`. It is always enabled (and is always searched first when attempting to retrieve files for an installation) but can be cleared with ``spack clean --misc-cache``; the cache directory can also be deleted manually without issue. Caching includes retrieved tarball archives and source control repositories, but only resources with an associated digest or commit ID (e.g. a revision number for SVN) will be cached. diff --git a/lib/spack/spack/bootstrap/config.py b/lib/spack/spack/bootstrap/config.py index b27d382ea293e6..e11e69108a1c23 100644 --- a/lib/spack/spack/bootstrap/config.py +++ b/lib/spack/spack/bootstrap/config.py @@ -42,7 +42,7 @@ def spec_for_current_python() -> str: def root_path() -> str: """Root of all the bootstrap related folders""" return spack.util.path.canonicalize_path( - spack.config.get("bootstrap:root", "$user_cache_path/bootstrap") + spack.config.get("bootstrap:root", "$state_home/bootstrap") ) diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index ef3faebe212c79..954036a7146050 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -75,7 +75,6 @@ def replacements(): "data_home": lambda: paths.data_home, "cache_home": lambda: paths.cache_home, "state_home": lambda: paths.state_home, - # "spack_config_home": lambda: paths.spack_config_home, "spack_instance_id": lambda: paths.spack_instance_id, "architecture": lambda: arch, "arch": lambda: arch, From 58444f1caf1a8e6082df655e3e509c24c1698550 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Feb 2026 16:29:20 -0800 Subject: [PATCH 266/506] wasn't properly expanding homedir when extracting paths vars from config vars (using canonicalize_path would be cyclic dependency) --- lib/spack/spack/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 56dc02a2e4f10b..c4269c1fdd7d0f 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -193,7 +193,7 @@ def spack_home_cfg_check(): ]: possible_resolution = check() if possible_resolution: - return possible_resolution + return os.path.expanduser(possible_resolution) return os.path.join(os.path.expanduser("~"), home_rel, "spack") From 9c3cafa6c39a047f3ec11f1e6458ebe398cdec5b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Feb 2026 17:09:32 -0800 Subject: [PATCH 267/506] path origins with a provenance other than not-set or xdg-based should override the presence of installs in the original spack path (xdg-based vars still override old install location if installs are found there) --- lib/spack/spack/paths.py | 71 +++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 23 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index c4269c1fdd7d0f..6abcfe84ee03ca 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -8,6 +8,7 @@ throughout Spack and should bring in a minimal number of external dependencies. """ +from enum import Enum import os import pathlib @@ -22,6 +23,19 @@ def dir_is_occupied(x, except_for=None): return x.is_dir() and bool(set(x.iterdir()) - except_for) +class Provenance(Enum): + SPACK_ENV = 1 # SPACK_x_HOME + SPACK_HOME_ENV = 2 # SPACK_HOME + CONFIG_VAR = 3 # config:locations:x + CONFIG_HOME_VAR = 4 # config:locations:home + XDG_VAR = 5 # XDG_x_HOME + NOTHING_SET = 6 # None of the above are set + + def respectable(self): + return self in {Provenance.SPACK_ENV, Provenance.SPACK_HOME_ENV, Provenance.CONFIG_VAR, + Provenance.CONFIG_HOME_VAR} + + class SpackPaths: def __init__(self, base): self.base = base @@ -33,22 +47,28 @@ def __init__(self, base): @property def state_home(self): - return self.resolve_a_home( - ["SPACK_STATE_HOME", "SPACK_USER_CACHE_PATH"], - "XDG_STATE_HOME", - "state", - os.path.join(".local", "state"), - ) + if not self._state_home: + self._state_home, self._state_home_provenance = self.resolve_a_home( + ["SPACK_STATE_HOME", "SPACK_USER_CACHE_PATH"], + "XDG_STATE_HOME", + "state", + os.path.join(".local", "state"), + ) + return self._state_home @property def cache_home(self): - return self.resolve_a_home("SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", ".cache") + if not self._cache_home: + self._cache_home, self._cache_home_provenance = self.resolve_a_home("SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", ".cache") + return self._cache_home @property def data_home(self): - return self.resolve_a_home( - "SPACK_DATA_HOME", "XDG_DATA_HOME", "data", os.path.join(".local", "share") - ) + if not self._data_home: + self._data_home, self._data_home_provenance = self.resolve_a_home( + "SPACK_DATA_HOME", "XDG_DATA_HOME", "data", os.path.join(".local", "share") + ) + return self._data_home @property def user_cache_path(self): @@ -57,13 +77,15 @@ def user_cache_path(self): @property def default_install_location(self): return self._fallback_old_location_if_used( - self.base.old_install_path, os.path.join(self.data_home, "installs") + self.base.old_install_path, os.path.join(self.data_home, "installs"), + self._data_home_provenance ) @property def default_envs_path(self): return self._fallback_old_location_if_used( - self.base.old_envs_path, os.path.join(self.data_home, "envs") + self.base.old_envs_path, os.path.join(self.data_home, "envs"), + self._data_home_provenance ) @property @@ -101,13 +123,15 @@ def package_repos_path(self): @property def gpg_path(self): return self._fallback_old_location_if_used( - self.base.old_gpg_path, os.path.join(self.data_home, "gpg") + self.base.old_gpg_path, os.path.join(self.data_home, "gpg"), + self._data_home_provenance ) @property def gpg_keys_path(self): return self._fallback_old_location_if_used( - self.base.old_gpg_keys_path, os.path.join(self.data_home, "gpg-keys") + self.base.old_gpg_keys_path, os.path.join(self.data_home, "gpg-keys"), + self._data_home_provenance ) @property @@ -162,27 +186,27 @@ def spack_env_check(): x = env_vars for n in x: if n in os.environ: - return os.environ[n] + return os.environ[n], Provenance.SPACK_ENV def xdg_env_check(): if disable_env: return if xdg_var in os.environ: - return os.path.join(os.environ[xdg_var], "spack") + return os.path.join(os.environ[xdg_var], "spack"), Provenance.XDG_VAR def spack_home_env_check(): if disable_env: return if "SPACK_HOME" in os.environ: - return os.path.join(os.environ["SPACK_HOME"], home_rel, "spack") + return os.path.join(os.environ["SPACK_HOME"], home_rel, "spack"), Provenance.SPACK_HOME_ENV def cfg_check(): - return config.get(f"config:locations:{config_var}", None) + return config.get(f"config:locations:{config_var}", None), Provenance.CONFIG_VAR def spack_home_cfg_check(): h = config.get("config:locations:home", None) if h: - return os.path.join(h, home_rel, "spack") + return os.path.join(h, home_rel, "spack"), Provenance.CONFIG_HOME_VAR for check in [ spack_env_check, @@ -193,14 +217,15 @@ def spack_home_cfg_check(): ]: possible_resolution = check() if possible_resolution: - return os.path.expanduser(possible_resolution) + path, provenance = possible_resolution + return os.path.expanduser(path), provenance - return os.path.join(os.path.expanduser("~"), home_rel, "spack") + return os.path.join(os.path.expanduser("~"), home_rel, "spack"), Provenance.NOTHING_SET - def _fallback_old_location_if_used(self, old_location, new_location): + def _fallback_old_location_if_used(self, old_location, new_location, provenance): # TODO: perhaps it should be configurable whether old locations # are used. Other option is to relocate downloads & gpg keys. - if dir_is_occupied(new_location): + if dir_is_occupied(new_location) or provenance.respectable(): return new_location elif dir_is_occupied(old_location): # TODO: should probably raise a deprecation warning here encouraging From c6d753279549880a62a4eb167c9ed884bd80a445 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Feb 2026 17:29:40 -0800 Subject: [PATCH 268/506] fix a couple bugs with prior commit --- lib/spack/spack/paths.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 6abcfe84ee03ca..19b5f798c7b576 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -40,6 +40,10 @@ class SpackPaths: def __init__(self, base): self.base = base + self._state_home = None + self._data_home = None + self._cache_home = None + #: Not a location itself, but used for when Spack instances #: share the same cache base directory for caches that should #: not be shared between those instances. @@ -201,7 +205,9 @@ def spack_home_env_check(): return os.path.join(os.environ["SPACK_HOME"], home_rel, "spack"), Provenance.SPACK_HOME_ENV def cfg_check(): - return config.get(f"config:locations:{config_var}", None), Provenance.CONFIG_VAR + val = config.get(f"config:locations:{config_var}", None) + if val: + return val, Provenance.CONFIG_VAR def spack_home_cfg_check(): h = config.get("config:locations:home", None) From 8ecfc683f78574bc964b0c0b5d61f78cd2bc1c12 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Feb 2026 17:43:46 -0800 Subject: [PATCH 269/506] new test for when old install dir is occupied --- lib/spack/spack/test/paths.py | 63 +++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 3dbd9c2015d231..d01169807f0b39 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -96,6 +96,69 @@ def paths_base_empty_old_install(): assert p6.default_install_location == str(pathlib.Path(spack_data_prefix) / "installs") +def test_install_location_old_installs_exist(working_env, tmp_path, mutable_config, set_home): + # If prior default install dir inside spack prefix does not + # exist, place installs in $HOME + base_prefix = _ensure_dir(tmp_path / "spack-root") + home_prefix = _ensure_dir(tmp_path / "home-prefix") + + not_empty_dir = _ensure_dir(tmp_path / "not-empty") + (pathlib.Path(not_empty_dir) / "afile").touch() + + def paths_base_nonempty_old_install(): + pb = SpackPathsBase(base_prefix) + pb.old_install_path = not_empty_dir + return pb + + set_home(home_prefix) + + p1 = SpackPaths(paths_base_nonempty_old_install()) + assert p1.default_install_location == not_empty_dir + + spack.config.set("config:locations", {}) + + # $XDG_DATA_HOME does *not* override if it is empty + xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") + os.environ["XDG_DATA_HOME"] = xdg_data_home + p4 = SpackPaths(paths_base_nonempty_old_install()) + assert p4.default_install_location == not_empty_dir + + # If there are installs XDG_DATA_HOME, prefer that, even if + # there are also installs in old location + xdg_installs_location = _ensure_dir(pathlib.Path(xdg_data_home) / "spack" / "installs") + (pathlib.Path(xdg_installs_location) / "afile").touch() + assert p4.default_install_location == str(pathlib.Path(xdg_data_home) / "spack" / "installs") + + # NOTE: the rest of this test is the same as the prior test + + # "config:locations:home" variable overrides the above (even if there + # are no installs there and there are installs in the old location) + spack_home_prefix = _ensure_dir(tmp_path / "spack-home") + spack.config.set("config:locations:home", spack_home_prefix) + p2 = SpackPaths(paths_base_nonempty_old_install()) + assert p2.default_install_location == str( + pathlib.Path(spack_home_prefix) / ".local" / "share" / "spack" / "installs" + ) + + # "config:locations:data" overrides the above + spack_data_prefix = _ensure_dir(tmp_path / "spack-data") + spack.config.set("config:locations:data", spack_data_prefix) + p3 = SpackPaths(paths_base_nonempty_old_install()) + assert p3.default_install_location == str(pathlib.Path(spack_data_prefix) / "installs") + + # Check that $SPACK_DATA_HOME overrides all the above + spack_data_home = _ensure_dir(tmp_path / "spack_data_home") + os.environ["SPACK_DATA_HOME"] = spack_data_home + p5 = SpackPaths(paths_base_nonempty_old_install()) + assert p5.default_install_location == str(pathlib.Path(spack_data_home) / "installs") + + # Disable all location-based env vars: this will then defer + # to using "config:locations:data" + spack.config.set("config:locations:disable_env", True) + p6 = SpackPaths(paths_base_nonempty_old_install()) + assert p6.default_install_location == str(pathlib.Path(spack_data_prefix) / "installs") + + def test_system_config_path_is_overridable(working_env, tmp_path): redirect_syscfg_path = str(pathlib.Path(tmp_path) / "redirected_syscfg") os.environ["SPACK_SYSTEM_CONFIG_PATH"] = redirect_syscfg_path From ca7490453e02421fd19da12c6f97b84ef6ea136d Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Feb 2026 17:44:52 -0800 Subject: [PATCH 270/506] style edit --- lib/spack/spack/paths.py | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 19b5f798c7b576..076f13d7ed7846 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -8,9 +8,9 @@ throughout Spack and should bring in a minimal number of external dependencies. """ -from enum import Enum import os import pathlib +from enum import Enum import spack.config as config import spack.paths_base as paths_base @@ -32,8 +32,12 @@ class Provenance(Enum): NOTHING_SET = 6 # None of the above are set def respectable(self): - return self in {Provenance.SPACK_ENV, Provenance.SPACK_HOME_ENV, Provenance.CONFIG_VAR, - Provenance.CONFIG_HOME_VAR} + return self in { + Provenance.SPACK_ENV, + Provenance.SPACK_HOME_ENV, + Provenance.CONFIG_VAR, + Provenance.CONFIG_HOME_VAR, + } class SpackPaths: @@ -63,7 +67,9 @@ def state_home(self): @property def cache_home(self): if not self._cache_home: - self._cache_home, self._cache_home_provenance = self.resolve_a_home("SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", ".cache") + self._cache_home, self._cache_home_provenance = self.resolve_a_home( + "SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", ".cache" + ) return self._cache_home @property @@ -81,15 +87,17 @@ def user_cache_path(self): @property def default_install_location(self): return self._fallback_old_location_if_used( - self.base.old_install_path, os.path.join(self.data_home, "installs"), - self._data_home_provenance + self.base.old_install_path, + os.path.join(self.data_home, "installs"), + self._data_home_provenance, ) @property def default_envs_path(self): return self._fallback_old_location_if_used( - self.base.old_envs_path, os.path.join(self.data_home, "envs"), - self._data_home_provenance + self.base.old_envs_path, + os.path.join(self.data_home, "envs"), + self._data_home_provenance, ) @property @@ -127,15 +135,15 @@ def package_repos_path(self): @property def gpg_path(self): return self._fallback_old_location_if_used( - self.base.old_gpg_path, os.path.join(self.data_home, "gpg"), - self._data_home_provenance + self.base.old_gpg_path, os.path.join(self.data_home, "gpg"), self._data_home_provenance ) @property def gpg_keys_path(self): return self._fallback_old_location_if_used( - self.base.old_gpg_keys_path, os.path.join(self.data_home, "gpg-keys"), - self._data_home_provenance + self.base.old_gpg_keys_path, + os.path.join(self.data_home, "gpg-keys"), + self._data_home_provenance, ) @property @@ -202,7 +210,10 @@ def spack_home_env_check(): if disable_env: return if "SPACK_HOME" in os.environ: - return os.path.join(os.environ["SPACK_HOME"], home_rel, "spack"), Provenance.SPACK_HOME_ENV + return ( + os.path.join(os.environ["SPACK_HOME"], home_rel, "spack"), + Provenance.SPACK_HOME_ENV, + ) def cfg_check(): val = config.get(f"config:locations:{config_var}", None) From 19af450921ce0a3fd5bae8aeef7d5933d1d6b89b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Feb 2026 18:23:10 -0800 Subject: [PATCH 271/506] edit var name --- lib/spack/spack/test/paths.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index d01169807f0b39..15c0fde4090ed7 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -102,18 +102,18 @@ def test_install_location_old_installs_exist(working_env, tmp_path, mutable_conf base_prefix = _ensure_dir(tmp_path / "spack-root") home_prefix = _ensure_dir(tmp_path / "home-prefix") - not_empty_dir = _ensure_dir(tmp_path / "not-empty") - (pathlib.Path(not_empty_dir) / "afile").touch() + nonempty_dir = _ensure_dir(tmp_path / "not-empty") + (pathlib.Path(nonempty_dir) / "afile").touch() def paths_base_nonempty_old_install(): pb = SpackPathsBase(base_prefix) - pb.old_install_path = not_empty_dir + pb.old_install_path = nonempty_dir return pb set_home(home_prefix) p1 = SpackPaths(paths_base_nonempty_old_install()) - assert p1.default_install_location == not_empty_dir + assert p1.default_install_location == nonempty_dir spack.config.set("config:locations", {}) @@ -121,7 +121,7 @@ def paths_base_nonempty_old_install(): xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") os.environ["XDG_DATA_HOME"] = xdg_data_home p4 = SpackPaths(paths_base_nonempty_old_install()) - assert p4.default_install_location == not_empty_dir + assert p4.default_install_location == nonempty_dir # If there are installs XDG_DATA_HOME, prefer that, even if # there are also installs in old location From 63aaf3bdc2081dc56d3068d441e6fbaf86d764a2 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Feb 2026 18:30:36 -0800 Subject: [PATCH 272/506] refine test to highlight problem (not yet fixed): XDG_DATA_HOME is set but has no spack installs, there are installs in the old dir, and there are installs in the default new installs dir: in this case the default new installs dir should be chosen --- lib/spack/spack/test/paths.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 15c0fde4090ed7..75a03e302475f9 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -112,16 +112,24 @@ def paths_base_nonempty_old_install(): set_home(home_prefix) + # The new default installs dir is ignored if it is empty and + # the old install location has anything in it p1 = SpackPaths(paths_base_nonempty_old_install()) assert p1.default_install_location == nonempty_dir + # If there are spack installs in the new installs dir, it is + # preferred over the old dir + new_default_installs_dir = _ensure_dir(pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs") + (pathlib.Path(new_default_installs_dir) / "afile").touch() + assert p1.default_install_location == new_default_installs_dir + spack.config.set("config:locations", {}) # $XDG_DATA_HOME does *not* override if it is empty xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") os.environ["XDG_DATA_HOME"] = xdg_data_home p4 = SpackPaths(paths_base_nonempty_old_install()) - assert p4.default_install_location == nonempty_dir + assert p4.default_install_location == new_default_installs_dir # If there are installs XDG_DATA_HOME, prefer that, even if # there are also installs in old location From 24dc1ef5c446b3aceba59ae6e22ec8f43f454cd7 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Feb 2026 23:46:34 -0800 Subject: [PATCH 273/506] partial reorg for updated test --- lib/spack/spack/paths.py | 102 ++++++++++++++++++++------------------- 1 file changed, 52 insertions(+), 50 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 076f13d7ed7846..c7ca87063d2400 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -40,6 +40,32 @@ def respectable(self): } +class HomeResolution: + def __init__(self, basedir): + self.basedir = pathlib.Path(basedir) + + def resolve(self, subdir, old_location=None): + return self.basedir / subdir + + +class XdgOrDefaultResolution: + def __init__(self, defaultdir, xdgdir=None): + self.defaultdir = pathlib.Path(defaultdir) + self.xdgdir = pathlib.Path(xdgdir) if xdgdir else None + + def resolve(self, subdir, old_location=None): + if old_location and dir_is_occupied(old_location): + return old_location + elif self.xdgdir and dir_is_occupied(self.xdgdir / subdir): + return self.xdgdir + elif dir_is_occupied(self.defaultdir / subdir): + return self.defaultdir + elif self.xdgdir: + return self.xdgdir / subdir + else: + return self.defaultdir / subdir + + class SpackPaths: def __init__(self, base): self.base = base @@ -56,7 +82,7 @@ def __init__(self, base): @property def state_home(self): if not self._state_home: - self._state_home, self._state_home_provenance = self.resolve_a_home( + self._state_home = self.resolve_a_home( ["SPACK_STATE_HOME", "SPACK_USER_CACHE_PATH"], "XDG_STATE_HOME", "state", @@ -67,7 +93,7 @@ def state_home(self): @property def cache_home(self): if not self._cache_home: - self._cache_home, self._cache_home_provenance = self.resolve_a_home( + self._cache_home = self.resolve_a_home( "SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", ".cache" ) return self._cache_home @@ -75,7 +101,7 @@ def cache_home(self): @property def data_home(self): if not self._data_home: - self._data_home, self._data_home_provenance = self.resolve_a_home( + self._data_home = self.resolve_a_home( "SPACK_DATA_HOME", "XDG_DATA_HOME", "data", os.path.join(".local", "share") ) return self._data_home @@ -86,29 +112,21 @@ def user_cache_path(self): @property def default_install_location(self): - return self._fallback_old_location_if_used( - self.base.old_install_path, - os.path.join(self.data_home, "installs"), - self._data_home_provenance, - ) + return self.data_home.resolve("installs", self.base.old_install_path) @property def default_envs_path(self): - return self._fallback_old_location_if_used( - self.base.old_envs_path, - os.path.join(self.data_home, "envs"), - self._data_home_provenance, - ) + return self.data_home.resolve("envs", self.base.old_envs_path) @property def reports_path(self): #: junit, cdash, etc. reports about builds - return os.path.join(self.state_home, "reports") + return self.state_home.resolve("reports") @property def default_test_path(self): #: installation test (spack test) output - return os.path.join(self.state_home, "test") + return self.state_home.resolve("test") @property def default_monitor_path(self): @@ -120,7 +138,7 @@ def user_repos_cache_path(self): #: git repositories fetched to compare commits to versions if hasattr(self, "_user_repos_cache_path"): return self._user_repos_cache_path - return os.path.join(self.state_home, "git_repos") + return self.state_home.resolve("git_repos") @user_repos_cache_path.setter def user_repos_cache_path(self, val): @@ -130,21 +148,15 @@ def user_repos_cache_path(self, val): @property def package_repos_path(self): #: default location where remote package repositories are cloned - return os.path.join(self.state_home, "package_repos") + return self.state_home.resolve("package_repos") @property def gpg_path(self): - return self._fallback_old_location_if_used( - self.base.old_gpg_path, os.path.join(self.data_home, "gpg"), self._data_home_provenance - ) + return self.data_home.resolve("gpg", self.base.old_gpg_path) @property def gpg_keys_path(self): - return self._fallback_old_location_if_used( - self.base.old_gpg_keys_path, - os.path.join(self.data_home, "gpg-keys"), - self._data_home_provenance, - ) + return self.data_home.resolve("gpg-keys", self.base.old_gpg_keys_path) @property def modules_base(self): @@ -163,7 +175,7 @@ def modules_base(self): def default_misc_cache_path(self): #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) #: overridden by `config:misc_cache` - return os.path.join(self.state_home, self.spack_instance_id, "cache") + return self.state_home.resolve(pathlib.Path(self.spack_instance_id) / "cache") def __getattr__(self, name): # Things that aren't sensitive to import cycles can import the @@ -198,59 +210,49 @@ def spack_env_check(): x = env_vars for n in x: if n in os.environ: - return os.environ[n], Provenance.SPACK_ENV + return os.environ[n] def xdg_env_check(): if disable_env: return if xdg_var in os.environ: - return os.path.join(os.environ[xdg_var], "spack"), Provenance.XDG_VAR + return os.path.join(os.environ[xdg_var], "spack") def spack_home_env_check(): if disable_env: return if "SPACK_HOME" in os.environ: return ( - os.path.join(os.environ["SPACK_HOME"], home_rel, "spack"), - Provenance.SPACK_HOME_ENV, + os.path.join(os.environ["SPACK_HOME"], home_rel, "spack") ) def cfg_check(): val = config.get(f"config:locations:{config_var}", None) if val: - return val, Provenance.CONFIG_VAR + return val def spack_home_cfg_check(): h = config.get("config:locations:home", None) if h: - return os.path.join(h, home_rel, "spack"), Provenance.CONFIG_HOME_VAR + return os.path.join(h, home_rel, "spack") for check in [ spack_env_check, spack_home_env_check, cfg_check, spack_home_cfg_check, - xdg_env_check, ]: possible_resolution = check() if possible_resolution: - path, provenance = possible_resolution - return os.path.expanduser(path), provenance - - return os.path.join(os.path.expanduser("~"), home_rel, "spack"), Provenance.NOTHING_SET - - def _fallback_old_location_if_used(self, old_location, new_location, provenance): - # TODO: perhaps it should be configurable whether old locations - # are used. Other option is to relocate downloads & gpg keys. - if dir_is_occupied(new_location) or provenance.respectable(): - return new_location - elif dir_is_occupied(old_location): - # TODO: should probably raise a deprecation warning here encouraging - # them to set their config explicitly back to the old value that - # will allow us to eventually remove these fallbacks - return old_location - else: - return new_location + return HomeResolution(os.path.expanduser(possible_resolution)) + + # TODO: need to update this + # encapsulate the final fallback and the XDG_x var in one value + # this object needs a resolve(subdir) method + # if XDG_x is set but subdir of it is empty, return the default dir if it is nonempty + + return XdgOrDefaultResolution(os.path.join(os.path.expanduser("~"), home_rel, "spack"), + xdg_env_check()) locations = SpackPaths(paths_base.locations) From 0a8bbbb2fd472ab58d698171767be70f0202c805 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Feb 2026 10:19:27 -0800 Subject: [PATCH 274/506] Revert "partial reorg for updated test" This reverts commit 24dc1ef5c446b3aceba59ae6e22ec8f43f454cd7. --- lib/spack/spack/paths.py | 102 +++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 52 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index c7ca87063d2400..076f13d7ed7846 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -40,32 +40,6 @@ def respectable(self): } -class HomeResolution: - def __init__(self, basedir): - self.basedir = pathlib.Path(basedir) - - def resolve(self, subdir, old_location=None): - return self.basedir / subdir - - -class XdgOrDefaultResolution: - def __init__(self, defaultdir, xdgdir=None): - self.defaultdir = pathlib.Path(defaultdir) - self.xdgdir = pathlib.Path(xdgdir) if xdgdir else None - - def resolve(self, subdir, old_location=None): - if old_location and dir_is_occupied(old_location): - return old_location - elif self.xdgdir and dir_is_occupied(self.xdgdir / subdir): - return self.xdgdir - elif dir_is_occupied(self.defaultdir / subdir): - return self.defaultdir - elif self.xdgdir: - return self.xdgdir / subdir - else: - return self.defaultdir / subdir - - class SpackPaths: def __init__(self, base): self.base = base @@ -82,7 +56,7 @@ def __init__(self, base): @property def state_home(self): if not self._state_home: - self._state_home = self.resolve_a_home( + self._state_home, self._state_home_provenance = self.resolve_a_home( ["SPACK_STATE_HOME", "SPACK_USER_CACHE_PATH"], "XDG_STATE_HOME", "state", @@ -93,7 +67,7 @@ def state_home(self): @property def cache_home(self): if not self._cache_home: - self._cache_home = self.resolve_a_home( + self._cache_home, self._cache_home_provenance = self.resolve_a_home( "SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", ".cache" ) return self._cache_home @@ -101,7 +75,7 @@ def cache_home(self): @property def data_home(self): if not self._data_home: - self._data_home = self.resolve_a_home( + self._data_home, self._data_home_provenance = self.resolve_a_home( "SPACK_DATA_HOME", "XDG_DATA_HOME", "data", os.path.join(".local", "share") ) return self._data_home @@ -112,21 +86,29 @@ def user_cache_path(self): @property def default_install_location(self): - return self.data_home.resolve("installs", self.base.old_install_path) + return self._fallback_old_location_if_used( + self.base.old_install_path, + os.path.join(self.data_home, "installs"), + self._data_home_provenance, + ) @property def default_envs_path(self): - return self.data_home.resolve("envs", self.base.old_envs_path) + return self._fallback_old_location_if_used( + self.base.old_envs_path, + os.path.join(self.data_home, "envs"), + self._data_home_provenance, + ) @property def reports_path(self): #: junit, cdash, etc. reports about builds - return self.state_home.resolve("reports") + return os.path.join(self.state_home, "reports") @property def default_test_path(self): #: installation test (spack test) output - return self.state_home.resolve("test") + return os.path.join(self.state_home, "test") @property def default_monitor_path(self): @@ -138,7 +120,7 @@ def user_repos_cache_path(self): #: git repositories fetched to compare commits to versions if hasattr(self, "_user_repos_cache_path"): return self._user_repos_cache_path - return self.state_home.resolve("git_repos") + return os.path.join(self.state_home, "git_repos") @user_repos_cache_path.setter def user_repos_cache_path(self, val): @@ -148,15 +130,21 @@ def user_repos_cache_path(self, val): @property def package_repos_path(self): #: default location where remote package repositories are cloned - return self.state_home.resolve("package_repos") + return os.path.join(self.state_home, "package_repos") @property def gpg_path(self): - return self.data_home.resolve("gpg", self.base.old_gpg_path) + return self._fallback_old_location_if_used( + self.base.old_gpg_path, os.path.join(self.data_home, "gpg"), self._data_home_provenance + ) @property def gpg_keys_path(self): - return self.data_home.resolve("gpg-keys", self.base.old_gpg_keys_path) + return self._fallback_old_location_if_used( + self.base.old_gpg_keys_path, + os.path.join(self.data_home, "gpg-keys"), + self._data_home_provenance, + ) @property def modules_base(self): @@ -175,7 +163,7 @@ def modules_base(self): def default_misc_cache_path(self): #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) #: overridden by `config:misc_cache` - return self.state_home.resolve(pathlib.Path(self.spack_instance_id) / "cache") + return os.path.join(self.state_home, self.spack_instance_id, "cache") def __getattr__(self, name): # Things that aren't sensitive to import cycles can import the @@ -210,49 +198,59 @@ def spack_env_check(): x = env_vars for n in x: if n in os.environ: - return os.environ[n] + return os.environ[n], Provenance.SPACK_ENV def xdg_env_check(): if disable_env: return if xdg_var in os.environ: - return os.path.join(os.environ[xdg_var], "spack") + return os.path.join(os.environ[xdg_var], "spack"), Provenance.XDG_VAR def spack_home_env_check(): if disable_env: return if "SPACK_HOME" in os.environ: return ( - os.path.join(os.environ["SPACK_HOME"], home_rel, "spack") + os.path.join(os.environ["SPACK_HOME"], home_rel, "spack"), + Provenance.SPACK_HOME_ENV, ) def cfg_check(): val = config.get(f"config:locations:{config_var}", None) if val: - return val + return val, Provenance.CONFIG_VAR def spack_home_cfg_check(): h = config.get("config:locations:home", None) if h: - return os.path.join(h, home_rel, "spack") + return os.path.join(h, home_rel, "spack"), Provenance.CONFIG_HOME_VAR for check in [ spack_env_check, spack_home_env_check, cfg_check, spack_home_cfg_check, + xdg_env_check, ]: possible_resolution = check() if possible_resolution: - return HomeResolution(os.path.expanduser(possible_resolution)) - - # TODO: need to update this - # encapsulate the final fallback and the XDG_x var in one value - # this object needs a resolve(subdir) method - # if XDG_x is set but subdir of it is empty, return the default dir if it is nonempty - - return XdgOrDefaultResolution(os.path.join(os.path.expanduser("~"), home_rel, "spack"), - xdg_env_check()) + path, provenance = possible_resolution + return os.path.expanduser(path), provenance + + return os.path.join(os.path.expanduser("~"), home_rel, "spack"), Provenance.NOTHING_SET + + def _fallback_old_location_if_used(self, old_location, new_location, provenance): + # TODO: perhaps it should be configurable whether old locations + # are used. Other option is to relocate downloads & gpg keys. + if dir_is_occupied(new_location) or provenance.respectable(): + return new_location + elif dir_is_occupied(old_location): + # TODO: should probably raise a deprecation warning here encouraging + # them to set their config explicitly back to the old value that + # will allow us to eventually remove these fallbacks + return old_location + else: + return new_location locations = SpackPaths(paths_base.locations) From 518017ccbaa31c77c005efbb97f679e80ab5be48 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Feb 2026 10:31:12 -0800 Subject: [PATCH 275/506] update test to reflect new desired behavior (still fails) --- lib/spack/spack/test/paths.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 75a03e302475f9..ff8b86ffe52b41 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -125,17 +125,18 @@ def paths_base_nonempty_old_install(): spack.config.set("config:locations", {}) - # $XDG_DATA_HOME does *not* override if it is empty + # XDG_DATA_HOME overrides the old install location *even if it is empty + # and the old install location is not*, if there are installs in the new + # default location xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") os.environ["XDG_DATA_HOME"] = xdg_data_home p4 = SpackPaths(paths_base_nonempty_old_install()) - assert p4.default_install_location == new_default_installs_dir - - # If there are installs XDG_DATA_HOME, prefer that, even if - # there are also installs in old location xdg_installs_location = _ensure_dir(pathlib.Path(xdg_data_home) / "spack" / "installs") + assert p4.default_install_location == str(xdg_installs_location) + + # (sanity) XDG_DATA_HOME still overrides when there is something in it (pathlib.Path(xdg_installs_location) / "afile").touch() - assert p4.default_install_location == str(pathlib.Path(xdg_data_home) / "spack" / "installs") + assert p4.default_install_location == str(xdg_installs_location) # NOTE: the rest of this test is the same as the prior test From a80fe558a7aa01d0f381a694aeb05f2264726a3a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Feb 2026 10:32:15 -0800 Subject: [PATCH 276/506] remove stale comment in modules.yaml default --- etc/spack/defaults/base/modules.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/etc/spack/defaults/base/modules.yaml b/etc/spack/defaults/base/modules.yaml index 72e949dc5db08c..5834f31c98841f 100644 --- a/etc/spack/defaults/base/modules.yaml +++ b/etc/spack/defaults/base/modules.yaml @@ -33,10 +33,6 @@ modules: default: # Where to install modules # If set to a non-default value, this setting has precedence. - # This default value tells spack to: - # * Prefer $data_home/ (e.g. $data_home/lmod) - # * The old module locations are $share/spack/: Spack will use - # this old location if the new location is empty and the old one isn't roots: tcl: $modules_base/modules lmod: $modules_base/lmod From f5c9d487676b5b9f9120e77186da229cfb5c3c9e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Feb 2026 15:29:53 -0800 Subject: [PATCH 277/506] updated logic passes updated test --- lib/spack/spack/paths.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 076f13d7ed7846..f4f60d985852cd 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -41,6 +41,11 @@ def respectable(self): class SpackPaths: + + relative_state_home = os.path.join(".local", "state") + relative_data_home = os.path.join(".local", "share") + relative_cache_home = ".cache" + def __init__(self, base): self.base = base @@ -48,6 +53,12 @@ def __init__(self, base): self._data_home = None self._cache_home = None + self.default_state_home, self.default_data_home, self.default_cache_home = ( + os.path.join(os.path.expanduser("~"), x, "spack") for x in + [SpackPaths.relative_state_home, SpackPaths.relative_data_home, + SpackPaths.relative_cache_home] + ) + #: Not a location itself, but used for when Spack instances #: share the same cache base directory for caches that should #: not be shared between those instances. @@ -89,6 +100,7 @@ def default_install_location(self): return self._fallback_old_location_if_used( self.base.old_install_path, os.path.join(self.data_home, "installs"), + os.path.join(self.default_data_home, "installs"), self._data_home_provenance, ) @@ -97,6 +109,7 @@ def default_envs_path(self): return self._fallback_old_location_if_used( self.base.old_envs_path, os.path.join(self.data_home, "envs"), + os.path.join(self.default_data_home, "envs"), self._data_home_provenance, ) @@ -135,7 +148,10 @@ def package_repos_path(self): @property def gpg_path(self): return self._fallback_old_location_if_used( - self.base.old_gpg_path, os.path.join(self.data_home, "gpg"), self._data_home_provenance + self.base.old_gpg_path, + os.path.join(self.data_home, "gpg"), + os.path.join(self.default_data_home, "gpg"), + self._data_home_provenance ) @property @@ -143,6 +159,7 @@ def gpg_keys_path(self): return self._fallback_old_location_if_used( self.base.old_gpg_keys_path, os.path.join(self.data_home, "gpg-keys"), + os.path.join(self.default_data_home, "gpg-keys"), self._data_home_provenance, ) @@ -239,15 +256,15 @@ def spack_home_cfg_check(): return os.path.join(os.path.expanduser("~"), home_rel, "spack"), Provenance.NOTHING_SET - def _fallback_old_location_if_used(self, old_location, new_location, provenance): - # TODO: perhaps it should be configurable whether old locations - # are used. Other option is to relocate downloads & gpg keys. + def _fallback_old_location_if_used(self, old_location, new_location, default_new_location, provenance): if dir_is_occupied(new_location) or provenance.respectable(): return new_location + elif dir_is_occupied(default_new_location): + # This can occur e.g. if someone clones a new instance of spack, + # which would write into the default new location, and then later + # they set XDG_DATA_HOME + return new_location elif dir_is_occupied(old_location): - # TODO: should probably raise a deprecation warning here encouraging - # them to set their config explicitly back to the old value that - # will allow us to eventually remove these fallbacks return old_location else: return new_location From e7c7fc4beb9443396a5f67f00a873c575a7a54e0 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Feb 2026 16:40:49 -0800 Subject: [PATCH 278/506] consistent use of reference var --- lib/spack/spack/paths.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index f4f60d985852cd..1772384f484d87 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -71,7 +71,7 @@ def state_home(self): ["SPACK_STATE_HOME", "SPACK_USER_CACHE_PATH"], "XDG_STATE_HOME", "state", - os.path.join(".local", "state"), + SpackPaths.relative_state_home, ) return self._state_home @@ -79,7 +79,7 @@ def state_home(self): def cache_home(self): if not self._cache_home: self._cache_home, self._cache_home_provenance = self.resolve_a_home( - "SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", ".cache" + "SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", SpackPaths.relative_cache_home ) return self._cache_home @@ -87,7 +87,7 @@ def cache_home(self): def data_home(self): if not self._data_home: self._data_home, self._data_home_provenance = self.resolve_a_home( - "SPACK_DATA_HOME", "XDG_DATA_HOME", "data", os.path.join(".local", "share") + "SPACK_DATA_HOME", "XDG_DATA_HOME", "data", SpackPaths.relative_data_home ) return self._data_home From 36acab1567150d5d84d0b2725b5a185dd8d025d2 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Feb 2026 16:41:39 -0800 Subject: [PATCH 279/506] auto style fix --- lib/spack/spack/paths.py | 15 ++++++++++----- lib/spack/spack/test/paths.py | 4 +++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 1772384f484d87..1172daa272d5fc 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -54,9 +54,12 @@ def __init__(self, base): self._cache_home = None self.default_state_home, self.default_data_home, self.default_cache_home = ( - os.path.join(os.path.expanduser("~"), x, "spack") for x in - [SpackPaths.relative_state_home, SpackPaths.relative_data_home, - SpackPaths.relative_cache_home] + os.path.join(os.path.expanduser("~"), x, "spack") + for x in [ + SpackPaths.relative_state_home, + SpackPaths.relative_data_home, + SpackPaths.relative_cache_home, + ] ) #: Not a location itself, but used for when Spack instances @@ -151,7 +154,7 @@ def gpg_path(self): self.base.old_gpg_path, os.path.join(self.data_home, "gpg"), os.path.join(self.default_data_home, "gpg"), - self._data_home_provenance + self._data_home_provenance, ) @property @@ -256,7 +259,9 @@ def spack_home_cfg_check(): return os.path.join(os.path.expanduser("~"), home_rel, "spack"), Provenance.NOTHING_SET - def _fallback_old_location_if_used(self, old_location, new_location, default_new_location, provenance): + def _fallback_old_location_if_used( + self, old_location, new_location, default_new_location, provenance + ): if dir_is_occupied(new_location) or provenance.respectable(): return new_location elif dir_is_occupied(default_new_location): diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index ff8b86ffe52b41..15057a95cd606f 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -119,7 +119,9 @@ def paths_base_nonempty_old_install(): # If there are spack installs in the new installs dir, it is # preferred over the old dir - new_default_installs_dir = _ensure_dir(pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs") + new_default_installs_dir = _ensure_dir( + pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs" + ) (pathlib.Path(new_default_installs_dir) / "afile").touch() assert p1.default_install_location == new_default_installs_dir From e5a081738af2bd7f79318b806a8f8b8ae9d9f99a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Feb 2026 16:52:04 -0800 Subject: [PATCH 280/506] considate shared test checks --- lib/spack/spack/test/paths.py | 38 ++++++++--------------------------- 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 15057a95cd606f..d9c466a00a7bde 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -69,31 +69,7 @@ def paths_base_empty_old_install(): p4 = SpackPaths(paths_base_empty_old_install()) assert p4.default_install_location == str(pathlib.Path(xdg_data_home) / "spack" / "installs") - # "config:locations:home" variable overrides the above - spack_home_prefix = _ensure_dir(tmp_path / "spack-home") - spack.config.set("config:locations:home", spack_home_prefix) - p2 = SpackPaths(paths_base_empty_old_install()) - assert p2.default_install_location == str( - pathlib.Path(spack_home_prefix) / ".local" / "share" / "spack" / "installs" - ) - - # "config:locations:data" overrides the above - spack_data_prefix = _ensure_dir(tmp_path / "spack-data") - spack.config.set("config:locations:data", spack_data_prefix) - p3 = SpackPaths(paths_base_empty_old_install()) - assert p3.default_install_location == str(pathlib.Path(spack_data_prefix) / "installs") - - # Check that $SPACK_DATA_HOME overrides all the above - spack_data_home = _ensure_dir(tmp_path / "spack_data_home") - os.environ["SPACK_DATA_HOME"] = spack_data_home - p5 = SpackPaths(paths_base_empty_old_install()) - assert p5.default_install_location == str(pathlib.Path(spack_data_home) / "installs") - - # Disable all location-based env vars: this will then defer - # to using "config:locations:data" - spack.config.set("config:locations:disable_env", True) - p6 = SpackPaths(paths_base_empty_old_install()) - assert p6.default_install_location == str(pathlib.Path(spack_data_prefix) / "installs") + _unconditional_path_override_checks(tmp_path, paths_base_empty_old_install) def test_install_location_old_installs_exist(working_env, tmp_path, mutable_config, set_home): @@ -140,13 +116,15 @@ def paths_base_nonempty_old_install(): (pathlib.Path(xdg_installs_location) / "afile").touch() assert p4.default_install_location == str(xdg_installs_location) - # NOTE: the rest of this test is the same as the prior test + _unconditional_path_override_checks(tmp_path, paths_base_nonempty_old_install) + +def _unconditional_path_override_checks(tmp_path, base_paths_generator): # "config:locations:home" variable overrides the above (even if there # are no installs there and there are installs in the old location) spack_home_prefix = _ensure_dir(tmp_path / "spack-home") spack.config.set("config:locations:home", spack_home_prefix) - p2 = SpackPaths(paths_base_nonempty_old_install()) + p2 = SpackPaths(base_paths_generator()) assert p2.default_install_location == str( pathlib.Path(spack_home_prefix) / ".local" / "share" / "spack" / "installs" ) @@ -154,19 +132,19 @@ def paths_base_nonempty_old_install(): # "config:locations:data" overrides the above spack_data_prefix = _ensure_dir(tmp_path / "spack-data") spack.config.set("config:locations:data", spack_data_prefix) - p3 = SpackPaths(paths_base_nonempty_old_install()) + p3 = SpackPaths(base_paths_generator()) assert p3.default_install_location == str(pathlib.Path(spack_data_prefix) / "installs") # Check that $SPACK_DATA_HOME overrides all the above spack_data_home = _ensure_dir(tmp_path / "spack_data_home") os.environ["SPACK_DATA_HOME"] = spack_data_home - p5 = SpackPaths(paths_base_nonempty_old_install()) + p5 = SpackPaths(base_paths_generator()) assert p5.default_install_location == str(pathlib.Path(spack_data_home) / "installs") # Disable all location-based env vars: this will then defer # to using "config:locations:data" spack.config.set("config:locations:disable_env", True) - p6 = SpackPaths(paths_base_nonempty_old_install()) + p6 = SpackPaths(base_paths_generator()) assert p6.default_install_location == str(pathlib.Path(spack_data_prefix) / "installs") From e3fe0b6d810e66a7cee3ae264fcf01438c462408 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Feb 2026 17:37:03 -0800 Subject: [PATCH 281/506] augment modules_base calculation with check for whether new default is occupied --- lib/spack/spack/paths.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 1172daa272d5fc..53688e0f33e6b1 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -166,19 +166,6 @@ def gpg_keys_path(self): self._data_home_provenance, ) - @property - def modules_base(self): - # This is similar to logic _fallback_old_location_if_used, but this - # moves the modules base if any component (typically one of lmod or - # tcl) has been relocated, so is examining one-layer deeper - for module_dir in ["lmod", "modules"]: - if dir_is_occupied(os.path.join(self.data_home, module_dir)): - return self.data_home - for module_dir in ["lmod", "modules"]: - if dir_is_occupied(os.path.join(self.base.share_path, module_dir)): - return self.base.share_path - return self.data_home - @property def default_misc_cache_path(self): #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) @@ -274,6 +261,28 @@ def _fallback_old_location_if_used( else: return new_location + @property + def modules_base(self): + # This is similar to logic _fallback_old_location_if_used, but this + # moves the modules base if any component (typically one of lmod or + # tcl) has been relocated, so is examining one-layer deeper + for module_dir in ["lmod", "modules"]: + if dir_is_occupied(os.path.join(self.data_home, module_dir)): + return self.data_home + + new_default_is_occupied = False + for module_dir in ["lmod", "modules"]: + if dir_is_occupied(self.default_data_home, module_dir): + new_default_is_occupied = True + break + if new_default_is_occupied: + return self.data_home + + for module_dir in ["lmod", "modules"]: + if dir_is_occupied(os.path.join(self.base.share_path, module_dir)): + return self.base.share_path + return self.data_home + locations = SpackPaths(paths_base.locations) From 29f80a82efe7f356e41679d4b5b81f0029840103 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Feb 2026 17:40:19 -0800 Subject: [PATCH 282/506] fix comment --- lib/spack/spack/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 53688e0f33e6b1..59fddea8610296 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -174,7 +174,7 @@ def default_misc_cache_path(self): def __getattr__(self, name): # Things that aren't sensitive to import cycles can import the - # paths module and access all items from it + # paths module and access all items from paths_base return getattr(self.base, name) def resolve_a_home(self, env_vars, xdg_var, config_var, home_rel): From f7b2c5027f7d6f0377c571ad34ec798eb828aae2 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Feb 2026 17:42:51 -0800 Subject: [PATCH 283/506] rename for clarity --- lib/spack/spack/paths.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 59fddea8610296..2c97fec74a9b2d 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -100,7 +100,7 @@ def user_cache_path(self): @property def default_install_location(self): - return self._fallback_old_location_if_used( + return self._decide_old_or_new_location( self.base.old_install_path, os.path.join(self.data_home, "installs"), os.path.join(self.default_data_home, "installs"), @@ -109,7 +109,7 @@ def default_install_location(self): @property def default_envs_path(self): - return self._fallback_old_location_if_used( + return self._decide_old_or_new_location( self.base.old_envs_path, os.path.join(self.data_home, "envs"), os.path.join(self.default_data_home, "envs"), @@ -150,7 +150,7 @@ def package_repos_path(self): @property def gpg_path(self): - return self._fallback_old_location_if_used( + return self._decide_old_or_new_location( self.base.old_gpg_path, os.path.join(self.data_home, "gpg"), os.path.join(self.default_data_home, "gpg"), @@ -159,7 +159,7 @@ def gpg_path(self): @property def gpg_keys_path(self): - return self._fallback_old_location_if_used( + return self._decide_old_or_new_location( self.base.old_gpg_keys_path, os.path.join(self.data_home, "gpg-keys"), os.path.join(self.default_data_home, "gpg-keys"), @@ -246,7 +246,7 @@ def spack_home_cfg_check(): return os.path.join(os.path.expanduser("~"), home_rel, "spack"), Provenance.NOTHING_SET - def _fallback_old_location_if_used( + def _decide_old_or_new_location( self, old_location, new_location, default_new_location, provenance ): if dir_is_occupied(new_location) or provenance.respectable(): @@ -263,7 +263,7 @@ def _fallback_old_location_if_used( @property def modules_base(self): - # This is similar to logic _fallback_old_location_if_used, but this + # This is similar to logic _decide_old_or_new_location, but this # moves the modules base if any component (typically one of lmod or # tcl) has been relocated, so is examining one-layer deeper for module_dir in ["lmod", "modules"]: From 09c7f3020a8098c1d3e73e52391a7cdb3544bf03 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Feb 2026 17:44:22 -0800 Subject: [PATCH 284/506] another function rename --- lib/spack/spack/paths.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 2c97fec74a9b2d..08e31e5cc0c3d4 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -31,7 +31,7 @@ class Provenance(Enum): XDG_VAR = 5 # XDG_x_HOME NOTHING_SET = 6 # None of the above are set - def respectable(self): + def unilateral_override(self): return self in { Provenance.SPACK_ENV, Provenance.SPACK_HOME_ENV, @@ -249,7 +249,7 @@ def spack_home_cfg_check(): def _decide_old_or_new_location( self, old_location, new_location, default_new_location, provenance ): - if dir_is_occupied(new_location) or provenance.respectable(): + if dir_is_occupied(new_location) or provenance.unilateral_override(): return new_location elif dir_is_occupied(default_new_location): # This can occur e.g. if someone clones a new instance of spack, From 319b843d94eb552c1db3d6a8d8cda2a9628c6fd4 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Feb 2026 18:01:27 -0800 Subject: [PATCH 285/506] add comments to explain purpose of class --- lib/spack/spack/paths.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 08e31e5cc0c3d4..1432606402119c 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -24,6 +24,11 @@ def dir_is_occupied(x, except_for=None): class Provenance(Enum): + # Used entirely inside this module, for recording configuration + # or environment options that the user set in order to influence + # the location of data that used to live in $spack and following + # #47615 now lives outside of it + SPACK_ENV = 1 # SPACK_x_HOME SPACK_HOME_ENV = 2 # SPACK_HOME CONFIG_VAR = 3 # config:locations:x @@ -32,6 +37,9 @@ class Provenance(Enum): NOTHING_SET = 6 # None of the above are set def unilateral_override(self): + # The following mechanisms for indicating user preference + # override the existence of data stored in its old location + # in $spack prior to #47615 return self in { Provenance.SPACK_ENV, Provenance.SPACK_HOME_ENV, From 5b00eef80cbe8b6948dce6d349e2a44753239819 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Feb 2026 18:25:25 -0800 Subject: [PATCH 286/506] update docs --- lib/spack/docs/environments.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index b29c3807b1d2bb..6b0fbd6101c961 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -246,7 +246,7 @@ The same rule applies to the ``install`` and ``uninstall`` commands. ==> No binary for zlib-1.2.8-yfc7epf57nsfn2gn4notccaiyxha6z7x found: installing from source ==> zlib: Executing phase: 'install' [+] ~/spack/opt/spack/linux-rhel7-broadwell/gcc-8.1.0/zlib-1.2.8-yfc7epf57nsfn2gn4notccaiyxha6z7x - ==> Updating view at ~/spack/opt/data/environments/myenv/.spack-env/view + ==> Updating view at ~/.local/share/spack/envs/myenv/.spack-env/view $ spack find ==> In environment myenv From f020fe9706f66a30218cdc0394b74b45b228588a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Feb 2026 18:29:57 -0800 Subject: [PATCH 287/506] update doc based on review --- lib/spack/docs/configuration.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index a90ea6aed66d1f..ecc8eda2067bb5 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -563,10 +563,10 @@ Older installs of spack placed these within ``$spack``, and fallback scheme in t * ``$default_install_root``: the location where installs go by default. Overridden by ``config:install_tree:root``. - Prefers ``$data_home/installs``, but if there are no installs there but there are installs in the old location within ``$spack``, then the old location will be used. + Prefers ``$data_home/installs``, but if there are no installs there but there are installs in the old location ``$spack/opt/spack``, then the old location will be used. * ``$default_envs_root``: the location where environments are managed by default. Overridden by ``config:environments_root``. - Prefers ``$data_home/envs`` but if there are no envs there and there are envs in the old location in ``$spack``, then the old location will be used. + Prefers ``$data_home/envs`` but if there are no envs there and there are envs in the old location ``$spack/var/spack/environments``, then the old location will be used. Environment variables ^^^^^^^^^^^^^^^^^^^^^ From 41229cbf9b6ccffa274a6dfa66088692afa8a22e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Feb 2026 12:40:23 -0800 Subject: [PATCH 288/506] accidentally passed 2 args into function, vs. joining them into 1 --- lib/spack/spack/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 1432606402119c..773a50a8931904 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -280,7 +280,7 @@ def modules_base(self): new_default_is_occupied = False for module_dir in ["lmod", "modules"]: - if dir_is_occupied(self.default_data_home, module_dir): + if dir_is_occupied(os.path.join(self.data_home, module_dir)): new_default_is_occupied = True break if new_default_is_occupied: From 94ee916e24336cc4e8cf4cd521559853e97c8da3 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Feb 2026 13:51:31 -0800 Subject: [PATCH 289/506] add 'spack location --install-root' and update docs for upstreams based on review --- lib/spack/docs/chain.rst | 12 +++++++++--- lib/spack/spack/cmd/location.py | 10 ++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/spack/docs/chain.rst b/lib/spack/docs/chain.rst index 487e45b4a9510c..0bab7ab8d371d4 100644 --- a/lib/spack/docs/chain.rst +++ b/lib/spack/docs/chain.rst @@ -17,11 +17,17 @@ To register the other Spack instance, you can add it as an entry to ``upstreams. upstreams: install-tree-1: - install_tree: /path/to/other/dir/containing/.spack-db + install_tree: /path/to/another/spack/install/tree install-tree-2: - install_tree: /path/to/another/dir/containing/.spack-db + install_tree: /path/to/old/spack/opt/spack -The ``install_tree`` must point to a directory containing ``.spack-db``, for example ``install_tree`` defined in :ref:`config.yaml `. +The ``install_tree`` must point to a directory containing a Spack install tree, as defined in :ref:`config.yaml `. +For a given instance of Spack, you can determine the directory to add here by running ``spack location --install-root`` in that instance, or with ``spack config get config``. + +.. note: + + The ``install_tree`` directory is the directory containing the ``.spack-db`` hidden directory. + This can be used to confirm you have the correct directory level listed for your upstream. Once the upstream Spack instance has been added, ``spack find`` will automatically check the upstream instance when querying installed packages, and new package installations for the local Spack installation will use any dependencies that are installed in the upstream instance. diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 7b495160397bf2..09d67a947e4822 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -11,6 +11,7 @@ import spack.llnl.util.tty as tty import spack.repo import spack.stage +import spack.store from spack.cmd.common import arguments from spack.paths import locations as paths @@ -35,6 +36,11 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: action="store_true", help="install prefix for spec (spec need not be installed)", ) + directories.add_argument( + "--install-root", + action="store_true", + help="where spack installs specs", + ) directories.add_argument( "-p", "--package-dir", @@ -100,6 +106,10 @@ def location(parser, args): print(paths.prefix) return + if args.install_root: + print(spack.store.STORE.root) + return + # no -e corresponds to False, -e without arg to None, -e name to the string name. if args.location_env is not False: if args.location_env is None: From ccda02da30e7b24f5ddefcd1172eb4546cdfacd3 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Feb 2026 13:57:20 -0800 Subject: [PATCH 290/506] auto style --- lib/spack/spack/cmd/location.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 09d67a947e4822..9c6c201568d095 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -37,9 +37,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: help="install prefix for spec (spec need not be installed)", ) directories.add_argument( - "--install-root", - action="store_true", - help="where spack installs specs", + "--install-root", action="store_true", help="where spack installs specs" ) directories.add_argument( "-p", From b13e509388e829718d088328005df51d2bd0c058 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Feb 2026 15:06:25 -0800 Subject: [PATCH 291/506] warnings when an old installs are ignored --- lib/spack/spack/environment/environment.py | 15 ++++++++--- lib/spack/spack/paths.py | 31 ++++++++++++++++++++++ lib/spack/spack/store.py | 6 ++++- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index 105b7f501fdfc7..53cc169574d3a8 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -65,9 +65,6 @@ #: Validation error for a currently activate environment that failed to parse _active_environment_error: Optional[spack.config.ConfigFormatError] = None -#: default path where environments are stored in the spack tree -default_env_path = lambda: paths.default_envs_path - #: Name of the input yaml file for an environment manifest_name = "spack.yaml" @@ -81,6 +78,18 @@ env_subdir_name = ".spack-env" +#: default path where environments are stored in the spack tree +_default_env_path = None + + +def default_env_path(): + global _default_env_path + if not _default_env_path: + _default_env_path = paths.default_envs_path + paths.bypassed_old_envs_warning() + return _default_env_path + + def env_root_path() -> str: """Override default root path if the user specified it""" return spack.util.path.canonicalize_path( diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 773a50a8931904..974d083327f371 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -13,6 +13,7 @@ from enum import Enum import spack.config as config +import spack.llnl.util.tty as tty import spack.paths_base as paths_base import spack.util.hash as hash @@ -115,6 +116,36 @@ def default_install_location(self): self._data_home_provenance, ) + def bypassed_old_installs_warning(self): + if ( + self.default_install_location != self.base.old_install_path + and dir_is_occupied(self.base.old_install_path) + and not self._data_home_provenance.unilateral_override() + ): + tty.warn( + f"Detected installs in {self.base.old_install_path}; Spack's default" + " install path resolution mechanism is active and determined that" + f" {self.default_install_location} is where it should look for and" + " place new installs. You can suppress this warning by setting" + " config:install_tree:root, config:locations:home, config:locations:data," + " SPACK_DATA_HOME, or SPACK_HOME" + ) + + def bypassed_old_envs_warning(self): + if ( + self.default_envs_path != self.base.old_envs_path + and dir_is_occupied(self.base.old_envs_path) + and not self._data_home_provenance.unilateral_override() + ): + tty.warn( + f"Detected environments in {self.base.old_envs_path}; Spack's default" + " environment path resolution mechanism is active and determined that" + f" {self.default_envs_path} is where it should look for and" + " place new environments. You can suppress this warning by setting" + " config:install_tree:root, config:locations:home, config:locations:data," + " SPACK_DATA_HOME, or SPACK_HOME" + ) + @property def default_envs_path(self): return self._decide_old_or_new_location( diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index c1baa5a377772a..b0521f842958f2 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -75,7 +75,11 @@ def parse_install_tree(config_dict: dict) -> Tuple[str, str, Dict[str, str]]: projections = {"all": all_projection} else: - unpadded_root = install_tree.get("root", paths.default_install_location) + unpadded_root = install_tree.get("root", None) + if not unpadded_root: + unpadded_root = paths.default_install_location + paths.bypassed_old_installs_warning() + unpadded_root = spack.util.path.canonicalize_path(unpadded_root) padded_length = install_tree.get("padded_length", False) From 46d374a1726b72a222bd60db656d28a367b43aa2 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Feb 2026 15:18:39 -0800 Subject: [PATCH 292/506] add tests for warnings --- lib/spack/spack/paths.py | 16 ++++++++++++---- lib/spack/spack/test/paths.py | 3 +++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 974d083327f371..de1807a18855ba 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -116,13 +116,13 @@ def default_install_location(self): self._data_home_provenance, ) - def bypassed_old_installs_warning(self): + def bypassed_old_installs_warning(self, _show=True): if ( self.default_install_location != self.base.old_install_path and dir_is_occupied(self.base.old_install_path) and not self._data_home_provenance.unilateral_override() ): - tty.warn( + msg = ( f"Detected installs in {self.base.old_install_path}; Spack's default" " install path resolution mechanism is active and determined that" f" {self.default_install_location} is where it should look for and" @@ -130,14 +130,18 @@ def bypassed_old_installs_warning(self): " config:install_tree:root, config:locations:home, config:locations:data," " SPACK_DATA_HOME, or SPACK_HOME" ) + if _show: + tty.warn(msg) + return msg + return "" - def bypassed_old_envs_warning(self): + def bypassed_old_envs_warning(self, _show=True): if ( self.default_envs_path != self.base.old_envs_path and dir_is_occupied(self.base.old_envs_path) and not self._data_home_provenance.unilateral_override() ): - tty.warn( + msg = ( f"Detected environments in {self.base.old_envs_path}; Spack's default" " environment path resolution mechanism is active and determined that" f" {self.default_envs_path} is where it should look for and" @@ -145,6 +149,10 @@ def bypassed_old_envs_warning(self): " config:install_tree:root, config:locations:home, config:locations:data," " SPACK_DATA_HOME, or SPACK_HOME" ) + if _show: + tty.warn(msg) + return msg + return "" @property def default_envs_path(self): diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index d9c466a00a7bde..b6858e5957f221 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -100,6 +100,7 @@ def paths_base_nonempty_old_install(): ) (pathlib.Path(new_default_installs_dir) / "afile").touch() assert p1.default_install_location == new_default_installs_dir + assert f"{new_default_installs_dir} is where it should look" in p1.bypassed_old_installs_warning(_show=False) spack.config.set("config:locations", {}) @@ -111,6 +112,7 @@ def paths_base_nonempty_old_install(): p4 = SpackPaths(paths_base_nonempty_old_install()) xdg_installs_location = _ensure_dir(pathlib.Path(xdg_data_home) / "spack" / "installs") assert p4.default_install_location == str(xdg_installs_location) + assert f"{xdg_installs_location} is where it should look" in p4.bypassed_old_installs_warning(_show=False) # (sanity) XDG_DATA_HOME still overrides when there is something in it (pathlib.Path(xdg_installs_location) / "afile").touch() @@ -128,6 +130,7 @@ def _unconditional_path_override_checks(tmp_path, base_paths_generator): assert p2.default_install_location == str( pathlib.Path(spack_home_prefix) / ".local" / "share" / "spack" / "installs" ) + assert not p2.bypassed_old_installs_warning(_show=False) # "config:locations:data" overrides the above spack_data_prefix = _ensure_dir(tmp_path / "spack-data") From 27b82ba45c5cc785456933ea6ca951e7aca28925 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Feb 2026 15:31:13 -0800 Subject: [PATCH 293/506] better management of env for tests --- lib/spack/spack/test/paths.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index b6858e5957f221..260e247cacafad 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -19,9 +19,11 @@ def _ensure_dir(pathlike): @pytest.fixture(scope="module", autouse=True) -def clear_xdg_vars(): +def clear_env_vars(): saved = os.environ.copy() spack.paths_base._unset_xdg_vars(os.environ) + for x in ["SPACK_USER_CACHE_PATH", "SPACK_HOME", "SPACK_DATA_HOME", "SPACK_STATE_HOME", "SPACK_CACHE_HOME"]: + os.environ.pop(x, None) yield os.environ.update(saved) @@ -189,8 +191,9 @@ def test_user_cache_path_is_overridable(working_env, tmp_path): # Now check that $SPACK_STATE_HOME takes precedence when both are set redirect2 = str(pathlib.Path(tmp_path) / "redirected_usr_cache2") os.environ["SPACK_STATE_HOME"] = redirect2 - assert p1.user_cache_path == redirect2 - assert p1.package_repos_path == str(pathlib.Path(redirect2) / "package_repos") + p2 = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "base-prefix"))) + assert p2.user_cache_path == redirect2 + assert p2.package_repos_path == str(pathlib.Path(redirect2) / "package_repos") def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmp_path, set_home): @@ -234,7 +237,8 @@ def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmp_path, set_home): assert p1.gpg_path == str(old_gpg_dir) -def test_user_cache_path_is_default_when_env_var_is_empty(working_env, tmp_path): - os.environ["SPACK_USER_CACHE_PATH"] = "" +def test_user_cache_path_is_default_when_env_var_is_empty(tmp_path, set_home): + homedir = _ensure_dir(tmp_path / "base-prefix") + set_home(homedir) p1 = SpackPaths(SpackPathsBase(str(tmp_path))) - assert os.path.expanduser(os.path.join("~", ".local", "state", "spack")) == p1.user_cache_path + assert str(pathlib.Path(homedir) / os.path.join(".local", "state", "spack")) == p1.user_cache_path From 69769e0930aa9ce1b6bee0293231d2443281120a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Feb 2026 15:32:01 -0800 Subject: [PATCH 294/506] auto style --- lib/spack/spack/test/paths.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 260e247cacafad..81146a090ea8d3 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -22,7 +22,13 @@ def _ensure_dir(pathlike): def clear_env_vars(): saved = os.environ.copy() spack.paths_base._unset_xdg_vars(os.environ) - for x in ["SPACK_USER_CACHE_PATH", "SPACK_HOME", "SPACK_DATA_HOME", "SPACK_STATE_HOME", "SPACK_CACHE_HOME"]: + for x in [ + "SPACK_USER_CACHE_PATH", + "SPACK_HOME", + "SPACK_DATA_HOME", + "SPACK_STATE_HOME", + "SPACK_CACHE_HOME", + ]: os.environ.pop(x, None) yield os.environ.update(saved) @@ -102,7 +108,10 @@ def paths_base_nonempty_old_install(): ) (pathlib.Path(new_default_installs_dir) / "afile").touch() assert p1.default_install_location == new_default_installs_dir - assert f"{new_default_installs_dir} is where it should look" in p1.bypassed_old_installs_warning(_show=False) + assert ( + f"{new_default_installs_dir} is where it should look" + in p1.bypassed_old_installs_warning(_show=False) + ) spack.config.set("config:locations", {}) @@ -114,7 +123,9 @@ def paths_base_nonempty_old_install(): p4 = SpackPaths(paths_base_nonempty_old_install()) xdg_installs_location = _ensure_dir(pathlib.Path(xdg_data_home) / "spack" / "installs") assert p4.default_install_location == str(xdg_installs_location) - assert f"{xdg_installs_location} is where it should look" in p4.bypassed_old_installs_warning(_show=False) + assert f"{xdg_installs_location} is where it should look" in p4.bypassed_old_installs_warning( + _show=False + ) # (sanity) XDG_DATA_HOME still overrides when there is something in it (pathlib.Path(xdg_installs_location) / "afile").touch() @@ -241,4 +252,6 @@ def test_user_cache_path_is_default_when_env_var_is_empty(tmp_path, set_home): homedir = _ensure_dir(tmp_path / "base-prefix") set_home(homedir) p1 = SpackPaths(SpackPathsBase(str(tmp_path))) - assert str(pathlib.Path(homedir) / os.path.join(".local", "state", "spack")) == p1.user_cache_path + assert ( + str(pathlib.Path(homedir) / os.path.join(".local", "state", "spack")) == p1.user_cache_path + ) From 834361bab26f40e8552328785e6924a9192d59e1 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Feb 2026 19:34:30 -0800 Subject: [PATCH 295/506] proper reinitialization of paths in subprocs --- lib/spack/spack/build_environment.py | 4 ++- lib/spack/spack/paths.py | 42 +++++++++++++++++++++------ lib/spack/spack/subprocess_context.py | 8 +++-- lib/spack/spack/test/database.py | 3 +- 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index c7fbb6346bb901..b55db01d1fc7c9 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -71,6 +71,7 @@ import spack.llnl.util.tty as tty import spack.multimethod import spack.package_base +import spack.paths import spack.paths_base import spack.platforms import spack.schema.environment @@ -1403,7 +1404,8 @@ def child_fun(): pkg=pkg, ) - p.start() + with spack.paths.locations.subprocess_override(): + p.start() # We close the writable end of the pipe now to be sure that p is the # only process which owns a handle for it. This ensures that when p diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index de1807a18855ba..1bcc81043cb1da 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -8,6 +8,7 @@ throughout Spack and should bring in a minimal number of external dependencies. """ +import contextlib import os import pathlib from enum import Enum @@ -24,24 +25,31 @@ def dir_is_occupied(x, except_for=None): return x.is_dir() and bool(set(x.iterdir()) - except_for) +_data_home_parent = "SPACK_PARENT_DATA_HOME" +_state_home_parent = "SPACK_PARENT_STATE_HOME" +_cache_home_parent = "SPACK_PARENT_CACHE_HOME" + + class Provenance(Enum): # Used entirely inside this module, for recording configuration # or environment options that the user set in order to influence # the location of data that used to live in $spack and following # #47615 now lives outside of it - SPACK_ENV = 1 # SPACK_x_HOME - SPACK_HOME_ENV = 2 # SPACK_HOME - CONFIG_VAR = 3 # config:locations:x - CONFIG_HOME_VAR = 4 # config:locations:home - XDG_VAR = 5 # XDG_x_HOME - NOTHING_SET = 6 # None of the above are set + PARENT_SETTING = 1 + SPACK_ENV = 2 # SPACK_x_HOME + SPACK_HOME_ENV = 3 # SPACK_HOME + CONFIG_VAR = 4 # config:locations:x + CONFIG_HOME_VAR = 5 # config:locations:home + XDG_VAR = 6 # XDG_x_HOME + NOTHING_SET = 7 # None of the above are set def unilateral_override(self): # The following mechanisms for indicating user preference # override the existence of data stored in its old location # in $spack prior to #47615 return self in { + Provenance.PARENT_SETTING, Provenance.SPACK_ENV, Provenance.SPACK_HOME_ENV, Provenance.CONFIG_VAR, @@ -84,6 +92,7 @@ def state_home(self): "XDG_STATE_HOME", "state", SpackPaths.relative_state_home, + _state_home_parent, ) return self._state_home @@ -91,7 +100,7 @@ def state_home(self): def cache_home(self): if not self._cache_home: self._cache_home, self._cache_home_provenance = self.resolve_a_home( - "SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", SpackPaths.relative_cache_home + "SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", SpackPaths.relative_cache_home, _cache_home_parent ) return self._cache_home @@ -99,7 +108,7 @@ def cache_home(self): def data_home(self): if not self._data_home: self._data_home, self._data_home_provenance = self.resolve_a_home( - "SPACK_DATA_HOME", "XDG_DATA_HOME", "data", SpackPaths.relative_data_home + "SPACK_DATA_HOME", "XDG_DATA_HOME", "data", SpackPaths.relative_data_home, _data_home_parent ) return self._data_home @@ -224,7 +233,7 @@ def __getattr__(self, name): # paths module and access all items from paths_base return getattr(self.base, name) - def resolve_a_home(self, env_vars, xdg_var, config_var, home_rel): + def resolve_a_home(self, env_vars, xdg_var, config_var, home_rel, parent_env_var): """ Data stored by spack is split into state, data, and cache components. This function can resolve where each of these components should be @@ -279,6 +288,11 @@ def spack_home_cfg_check(): if h: return os.path.join(h, home_rel, "spack"), Provenance.CONFIG_HOME_VAR + def parent_env_check(): + # This does not follow disable_env + if parent_env_var in os.environ: + return os.environ[parent_env_var], Provenance.PARENT_SETTING + for check in [ spack_env_check, spack_home_env_check, @@ -330,6 +344,16 @@ def modules_base(self): return self.base.share_path return self.data_home + @contextlib.contextmanager + def subprocess_override(self): + # This is not thread-safe + os.environ[_data_home_parent] = self.default_data_home + os.environ[_state_home_parent] = self.default_state_home + os.environ[_cache_home_parent] = self.default_cache_home + yield + for x in [_data_home_parent, _state_home_parent, _cache_home_parent]: + del os.environ[x] + locations = SpackPaths(paths_base.locations) diff --git a/lib/spack/spack/subprocess_context.py b/lib/spack/spack/subprocess_context.py index 52feb2e51782b2..0aa011019d004b 100644 --- a/lib/spack/spack/subprocess_context.py +++ b/lib/spack/spack/subprocess_context.py @@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Optional import spack.config +import spack.paths import spack.paths_base import spack.platforms import spack.repo @@ -55,9 +56,12 @@ def _restore_and_run(self, fn, test_state): test_state.restore() fn() - def create(self): + def create_and_start(self): test_state = GlobalStateMarshaler() - return multiprocessing.Process(target=self._restore_and_run, args=(self.fn, test_state)) + p = multiprocessing.Process(target=self._restore_and_run, args=(self.fn, test_state)) + with spack.paths.locations.subprocess_override(): + p.start() + return p class PackageInstallContext: diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 9dd08f2eaa7504..a356e196ffb47d 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -642,8 +642,7 @@ def __call__(self): def test_030_db_sanity_from_another_process(mutable_database): spack_process = spack.subprocess_context.SpackTestProcess(ReadModify()) - p = spack_process.create() - p.start() + p = spack_process.create_and_start() p.join() # ensure child process change is visible in parent process From 14a33c6c1e7fc737d9a97aec017436d52a8d1f63 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Feb 2026 19:35:16 -0800 Subject: [PATCH 296/506] auto style --- lib/spack/spack/paths.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 1bcc81043cb1da..40eee891eecfd3 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -100,7 +100,11 @@ def state_home(self): def cache_home(self): if not self._cache_home: self._cache_home, self._cache_home_provenance = self.resolve_a_home( - "SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", SpackPaths.relative_cache_home, _cache_home_parent + "SPACK_CACHE_HOME", + "XDG_CACHE_HOME", + "cache", + SpackPaths.relative_cache_home, + _cache_home_parent, ) return self._cache_home @@ -108,7 +112,11 @@ def cache_home(self): def data_home(self): if not self._data_home: self._data_home, self._data_home_provenance = self.resolve_a_home( - "SPACK_DATA_HOME", "XDG_DATA_HOME", "data", SpackPaths.relative_data_home, _data_home_parent + "SPACK_DATA_HOME", + "XDG_DATA_HOME", + "data", + SpackPaths.relative_data_home, + _data_home_parent, ) return self._data_home From b29526218badbad2759bce752533edf9cf9e586c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Feb 2026 22:13:01 -0800 Subject: [PATCH 297/506] uncomment default val in config for env root --- etc/spack/defaults/base/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index 938c5a789fc6ac..a34abdce0dca6d 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -86,7 +86,7 @@ config: # * Prefer $data_home/envs # * The old location for envs is $spack/var/spack/environments: Spack will # use this old location if the new location is empty and the old one isn't - # environments_root: $default_envs_root + environments_root: $default_envs_root # Cache directory for miscellaneous files, like the package index. From cf85fdc51127d8a027c7210c8ee259dea6852a6e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Feb 2026 22:34:32 -0800 Subject: [PATCH 298/506] update command completion --- share/spack/spack-completion.bash | 4 ++-- share/spack/spack-completion.fish | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index d0853b3e7a30fd..d772ec5c44c068 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -681,7 +681,7 @@ _spack_buildcache_migrate() { _spack_cd() { if $list_options then - SPACK_COMPREPLY="-h --help -m --module-dir -r --spack-root -i --install-dir -p --package-dir --repo --packages -P -s --stage-dir -S --stages -c --source-dir -b --build-dir -e --env --first" + SPACK_COMPREPLY="-h --help -m --module-dir -r --spack-root -i --install-dir --install-root -p --package-dir --repo --packages -P -s --stage-dir -S --stages -c --source-dir -b --build-dir -e --env --first" else _all_packages fi @@ -1410,7 +1410,7 @@ _spack_load() { _spack_location() { if $list_options then - SPACK_COMPREPLY="-h --help -m --module-dir -r --spack-root -i --install-dir -p --package-dir --repo --packages -P -s --stage-dir -S --stages -c --source-dir -b --build-dir -e --env --first" + SPACK_COMPREPLY="-h --help -m --module-dir -r --spack-root -i --install-dir --install-root -p --package-dir --repo --packages -P -s --stage-dir -S --stages -c --source-dir -b --build-dir -e --env --first" else _all_packages fi diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index c61a2409b36fd1..a196160b4bcc8a 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -920,7 +920,7 @@ complete -c spack -n '__fish_spack_using_command buildcache migrate' -s y -l yes complete -c spack -n '__fish_spack_using_command buildcache migrate' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request' # spack cd -set -g __fish_spack_optspecs_spack_cd h/help m/module-dir r/spack-root i/install-dir p/package-dir repo= s/stage-dir S/stages c/source-dir b/build-dir e/env= first +set -g __fish_spack_optspecs_spack_cd h/help m/module-dir r/spack-root i/install-dir install-root p/package-dir repo= s/stage-dir S/stages c/source-dir b/build-dir e/env= first complete -c spack -n '__fish_spack_using_command_pos_remainder 0 cd' -f -k -a '(__fish_spack_specs)' complete -c spack -n '__fish_spack_using_command cd' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command cd' -s h -l help -d 'show this help message and exit' @@ -930,6 +930,8 @@ complete -c spack -n '__fish_spack_using_command cd' -s r -l spack-root -f -a sp complete -c spack -n '__fish_spack_using_command cd' -s r -l spack-root -d 'spack installation root' complete -c spack -n '__fish_spack_using_command cd' -s i -l install-dir -f -a install_dir complete -c spack -n '__fish_spack_using_command cd' -s i -l install-dir -d 'install prefix for spec (spec need not be installed)' +complete -c spack -n '__fish_spack_using_command cd' -l install-root -f -a install_root +complete -c spack -n '__fish_spack_using_command cd' -l install-root -d 'where spack installs specs' complete -c spack -n '__fish_spack_using_command cd' -s p -l package-dir -f -a package_dir complete -c spack -n '__fish_spack_using_command cd' -s p -l package-dir -d 'directory enclosing a spec'"'"'s package.py file' complete -c spack -n '__fish_spack_using_command cd' -l repo -l packages -s P -r -f -a repo @@ -2255,7 +2257,7 @@ complete -c spack -n '__fish_spack_using_command load' -l list -f -a list complete -c spack -n '__fish_spack_using_command load' -l list -d 'show loaded packages: same as ``spack find --loaded``' # spack location -set -g __fish_spack_optspecs_spack_location h/help m/module-dir r/spack-root i/install-dir p/package-dir repo= s/stage-dir S/stages c/source-dir b/build-dir e/env= first +set -g __fish_spack_optspecs_spack_location h/help m/module-dir r/spack-root i/install-dir install-root p/package-dir repo= s/stage-dir S/stages c/source-dir b/build-dir e/env= first complete -c spack -n '__fish_spack_using_command_pos_remainder 0 location' -f -k -a '(__fish_spack_specs)' complete -c spack -n '__fish_spack_using_command location' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command location' -s h -l help -d 'show this help message and exit' @@ -2265,6 +2267,8 @@ complete -c spack -n '__fish_spack_using_command location' -s r -l spack-root -f complete -c spack -n '__fish_spack_using_command location' -s r -l spack-root -d 'spack installation root' complete -c spack -n '__fish_spack_using_command location' -s i -l install-dir -f -a install_dir complete -c spack -n '__fish_spack_using_command location' -s i -l install-dir -d 'install prefix for spec (spec need not be installed)' +complete -c spack -n '__fish_spack_using_command location' -l install-root -f -a install_root +complete -c spack -n '__fish_spack_using_command location' -l install-root -d 'where spack installs specs' complete -c spack -n '__fish_spack_using_command location' -s p -l package-dir -f -a package_dir complete -c spack -n '__fish_spack_using_command location' -s p -l package-dir -d 'directory enclosing a spec'"'"'s package.py file' complete -c spack -n '__fish_spack_using_command location' -l repo -l packages -s P -r -f -a repo From a5470e39f955af73c683fc44f7e6c9a0e9cecb77 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 10 Feb 2026 10:32:36 -0800 Subject: [PATCH 299/506] no SPACK_CONFIG_HOME --- lib/spack/spack/paths_base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index 92378842c4ea2d..0030b77a31e95c 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -11,14 +11,12 @@ class XDG_vars(Enum): - config_home = "XDG_CONFIG_HOME" state_home = "XDG_STATE_HOME" data_home = "XDG_DATA_HOME" cache_home = "XDG_CACHE_HOME" class XDG_overrides(Enum): - config_home = "SPACK_CONFIG_HOME" state_home = "SPACK_STATE_HOME" data_home = "SPACK_DATA_HOME" cache_home = "SPACK_CACHE_HOME" From 5c3968e7cc2812fb13d9548ad98bb01c9823752a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 10 Feb 2026 10:55:21 -0800 Subject: [PATCH 300/506] forgot to actually register the parent env var check --- lib/spack/spack/paths.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 40eee891eecfd3..9310658988cb90 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -302,6 +302,7 @@ def parent_env_check(): return os.environ[parent_env_var], Provenance.PARENT_SETTING for check in [ + parent_env_check, spack_env_check, spack_home_env_check, cfg_check, From 898e9321c04474c509ddbe2ef0b6ad7cfb90c2b5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 10 Feb 2026 11:34:55 -0800 Subject: [PATCH 301/506] temporarily move misc cache to non-spack-id-customized location (otherwise ci script in spack-packages fails) --- lib/spack/spack/paths.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 9310658988cb90..f569bbf97f0138 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -234,7 +234,10 @@ def gpg_keys_path(self): def default_misc_cache_path(self): #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) #: overridden by `config:misc_cache` - return os.path.join(self.state_home, self.spack_instance_id, "cache") + # TODO: restore when .ci/gitlab/configs/ci.yaml in spack-packages is + # updated to tolerate this relocation of misc caches + # return os.path.join(self.state_home, self.spack_instance_id, "cache") + return os.path.join(self.state_home, "cache") def __getattr__(self, name): # Things that aren't sensitive to import cycles can import the From b98dbf630d709567abd76a35a1100fa423a89307 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 10 Feb 2026 13:45:40 -0800 Subject: [PATCH 302/506] fix config default setting --- etc/spack/defaults/base/config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index a34abdce0dca6d..2191c9b4c1707f 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -91,7 +91,8 @@ config: # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` - misc_cache: $state_home/$spack_instance_id/cache + # TODO: see note in paths.py - default_misc_cache_path + misc_cache: $state_home/cache # Abort downloads after this many seconds if not data is received. From 0073000e501ac79a1d1a150965232edb5e6960e1 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 11 Feb 2026 15:44:40 -0800 Subject: [PATCH 303/506] add spack_home config var; update spack scope to use it to resolve user scope; you can set locations:home in the defaults scope to change where the user config scope is retrieved from --- etc/spack/include.yaml | 2 +- lib/spack/spack/paths.py | 13 +++++++++++++ lib/spack/spack/util/path.py | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/etc/spack/include.yaml b/etc/spack/include.yaml index b8ab4ebfeb3b8b..a4c04b94616405 100644 --- a/etc/spack/include.yaml +++ b/etc/spack/include.yaml @@ -2,7 +2,7 @@ include: # user configuration scope - name: "user" path_override_env_var: SPACK_USER_CONFIG_PATH - path: "~/.config/spack/" + path: "$spack_home/.config/spack/" optional: true prefer_modify: true when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index f569bbf97f0138..d1513b27d7d66b 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -120,6 +120,19 @@ def data_home(self): ) return self._data_home + @property + def spack_home(self): + disable_env = config.get("config:locations:disable_env", False) + spack_home_env = os.environ.get("SPACK_HOME", None) + if not disable_env and spack_home_env: + return spack_home_env + + spack_home_cfg = config.get("config:locations:home", None) + if spack_home_cfg: + return spack_home_cfg + + return os.path.expanduser("~") + @property def user_cache_path(self): return self.state_home diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 954036a7146050..6cdb80d21b6d02 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -75,6 +75,7 @@ def replacements(): "data_home": lambda: paths.data_home, "cache_home": lambda: paths.cache_home, "state_home": lambda: paths.state_home, + "spack_home": lambda: paths.spack_home, "spack_instance_id": lambda: paths.spack_instance_id, "architecture": lambda: arch, "arch": lambda: arch, From 76f0ee99a94ffc7859ceccb92456d7342bbcb733 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 23 Feb 2026 09:37:24 -0800 Subject: [PATCH 304/506] Revert "forgot to actually register the parent env var check" This reverts commit 5c3968e7cc2812fb13d9548ad98bb01c9823752a. --- lib/spack/spack/paths.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index d1513b27d7d66b..2046c926641a8d 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -318,7 +318,6 @@ def parent_env_check(): return os.environ[parent_env_var], Provenance.PARENT_SETTING for check in [ - parent_env_check, spack_env_check, spack_home_env_check, cfg_check, From 59776dce9459ef1c8b0b9b7dd7e082502ba62dd3 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 23 Feb 2026 09:58:50 -0800 Subject: [PATCH 305/506] semantic undo of 834361b --- lib/spack/spack/build_environment.py | 4 +-- lib/spack/spack/paths.py | 50 +++++---------------------- lib/spack/spack/subprocess_context.py | 8 ++--- lib/spack/spack/test/database.py | 3 +- 4 files changed, 14 insertions(+), 51 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index b55db01d1fc7c9..c7fbb6346bb901 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -71,7 +71,6 @@ import spack.llnl.util.tty as tty import spack.multimethod import spack.package_base -import spack.paths import spack.paths_base import spack.platforms import spack.schema.environment @@ -1404,8 +1403,7 @@ def child_fun(): pkg=pkg, ) - with spack.paths.locations.subprocess_override(): - p.start() + p.start() # We close the writable end of the pipe now to be sure that p is the # only process which owns a handle for it. This ensures that when p diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 2046c926641a8d..c72ec2c3058441 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -8,7 +8,6 @@ throughout Spack and should bring in a minimal number of external dependencies. """ -import contextlib import os import pathlib from enum import Enum @@ -25,31 +24,24 @@ def dir_is_occupied(x, except_for=None): return x.is_dir() and bool(set(x.iterdir()) - except_for) -_data_home_parent = "SPACK_PARENT_DATA_HOME" -_state_home_parent = "SPACK_PARENT_STATE_HOME" -_cache_home_parent = "SPACK_PARENT_CACHE_HOME" - - class Provenance(Enum): # Used entirely inside this module, for recording configuration # or environment options that the user set in order to influence # the location of data that used to live in $spack and following # #47615 now lives outside of it - PARENT_SETTING = 1 - SPACK_ENV = 2 # SPACK_x_HOME - SPACK_HOME_ENV = 3 # SPACK_HOME - CONFIG_VAR = 4 # config:locations:x - CONFIG_HOME_VAR = 5 # config:locations:home - XDG_VAR = 6 # XDG_x_HOME - NOTHING_SET = 7 # None of the above are set + SPACK_ENV = 1 # SPACK_x_HOME + SPACK_HOME_ENV = 2 # SPACK_HOME + CONFIG_VAR = 3 # config:locations:x + CONFIG_HOME_VAR = 4 # config:locations:home + XDG_VAR = 5 # XDG_x_HOME + NOTHING_SET = 6 # None of the above are set def unilateral_override(self): # The following mechanisms for indicating user preference # override the existence of data stored in its old location # in $spack prior to #47615 return self in { - Provenance.PARENT_SETTING, Provenance.SPACK_ENV, Provenance.SPACK_HOME_ENV, Provenance.CONFIG_VAR, @@ -92,7 +84,6 @@ def state_home(self): "XDG_STATE_HOME", "state", SpackPaths.relative_state_home, - _state_home_parent, ) return self._state_home @@ -100,11 +91,7 @@ def state_home(self): def cache_home(self): if not self._cache_home: self._cache_home, self._cache_home_provenance = self.resolve_a_home( - "SPACK_CACHE_HOME", - "XDG_CACHE_HOME", - "cache", - SpackPaths.relative_cache_home, - _cache_home_parent, + "SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", SpackPaths.relative_cache_home ) return self._cache_home @@ -112,11 +99,7 @@ def cache_home(self): def data_home(self): if not self._data_home: self._data_home, self._data_home_provenance = self.resolve_a_home( - "SPACK_DATA_HOME", - "XDG_DATA_HOME", - "data", - SpackPaths.relative_data_home, - _data_home_parent, + "SPACK_DATA_HOME", "XDG_DATA_HOME", "data", SpackPaths.relative_data_home ) return self._data_home @@ -257,7 +240,7 @@ def __getattr__(self, name): # paths module and access all items from paths_base return getattr(self.base, name) - def resolve_a_home(self, env_vars, xdg_var, config_var, home_rel, parent_env_var): + def resolve_a_home(self, env_vars, xdg_var, config_var, home_rel): """ Data stored by spack is split into state, data, and cache components. This function can resolve where each of these components should be @@ -312,11 +295,6 @@ def spack_home_cfg_check(): if h: return os.path.join(h, home_rel, "spack"), Provenance.CONFIG_HOME_VAR - def parent_env_check(): - # This does not follow disable_env - if parent_env_var in os.environ: - return os.environ[parent_env_var], Provenance.PARENT_SETTING - for check in [ spack_env_check, spack_home_env_check, @@ -368,16 +346,6 @@ def modules_base(self): return self.base.share_path return self.data_home - @contextlib.contextmanager - def subprocess_override(self): - # This is not thread-safe - os.environ[_data_home_parent] = self.default_data_home - os.environ[_state_home_parent] = self.default_state_home - os.environ[_cache_home_parent] = self.default_cache_home - yield - for x in [_data_home_parent, _state_home_parent, _cache_home_parent]: - del os.environ[x] - locations = SpackPaths(paths_base.locations) diff --git a/lib/spack/spack/subprocess_context.py b/lib/spack/spack/subprocess_context.py index 0aa011019d004b..52feb2e51782b2 100644 --- a/lib/spack/spack/subprocess_context.py +++ b/lib/spack/spack/subprocess_context.py @@ -20,7 +20,6 @@ from typing import TYPE_CHECKING, Optional import spack.config -import spack.paths import spack.paths_base import spack.platforms import spack.repo @@ -56,12 +55,9 @@ def _restore_and_run(self, fn, test_state): test_state.restore() fn() - def create_and_start(self): + def create(self): test_state = GlobalStateMarshaler() - p = multiprocessing.Process(target=self._restore_and_run, args=(self.fn, test_state)) - with spack.paths.locations.subprocess_override(): - p.start() - return p + return multiprocessing.Process(target=self._restore_and_run, args=(self.fn, test_state)) class PackageInstallContext: diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index a356e196ffb47d..9dd08f2eaa7504 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -642,7 +642,8 @@ def __call__(self): def test_030_db_sanity_from_another_process(mutable_database): spack_process = spack.subprocess_context.SpackTestProcess(ReadModify()) - p = spack_process.create_and_start() + p = spack_process.create() + p.start() p.join() # ensure child process change is visible in parent process From a3a836c026b0d3b0c738dfff19fb5dfa27fdbcce Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 23 Feb 2026 14:28:35 -0800 Subject: [PATCH 306/506] alternative to high-priority env vars: subprocess_context runs paths.freeze on child process initialization in order to avoid issues with a package setting XDG_ variables --- lib/spack/spack/paths.py | 15 +++++++++++++++ lib/spack/spack/subprocess_context.py | 2 ++ 2 files changed, 17 insertions(+) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index c72ec2c3058441..d9f713f96f974d 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -349,6 +349,21 @@ def modules_base(self): locations = SpackPaths(paths_base.locations) + +def freeze(): + """ + Run this first-thing when starting a spack child process that is + used for builds. Resolve all variables that are XDG-dependent, in + case the build sets those XDG variables. + + Note that for this reason, the private variables like _data_home + are important for more than just caching. + """ + locations.state_home + locations.data_home + locations.cache_home + + # At least one builtin spack package expects that spack.paths is # importable and that it has spack_script as a module-level attribute. # Some test packages expect other paths (like test_path) diff --git a/lib/spack/spack/subprocess_context.py b/lib/spack/spack/subprocess_context.py index 52feb2e51782b2..54e56c505fabfc 100644 --- a/lib/spack/spack/subprocess_context.py +++ b/lib/spack/spack/subprocess_context.py @@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Optional import spack.config +import spack.paths import spack.paths_base import spack.platforms import spack.repo @@ -108,6 +109,7 @@ def restore(self): if self.is_forked: return spack.config.CONFIG = self.config + spack.paths.freeze() spack.repo.enable_repo(spack.repo.RepoPath.from_config(self.config)) spack.platforms.host = self.platform spack.store.STORE = self.store From 808a2926b555f35b5a785d570a3dd1799820b244 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 23 Feb 2026 15:02:24 -0800 Subject: [PATCH 307/506] intermediate work on testing freeze logic --- lib/spack/spack/test/paths.py | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 81146a090ea8d3..fd3409c3eb8773 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -11,6 +11,7 @@ import spack.paths_base from spack.paths import SpackPaths from spack.paths_base import SpackPathsBase +import spack.subprocess_context def _ensure_dir(pathlike): @@ -255,3 +256,49 @@ def test_user_cache_path_is_default_when_env_var_is_empty(tmp_path, set_home): assert ( str(pathlib.Path(homedir) / os.path.join(".local", "state", "spack")) == p1.user_cache_path ) + + +class SetAnXdgVarAndReadDataHome: + """Provide a function which can execute in a separate process that removes + a spec from the database. + """ + def __init__(self, home_prefix, base_prefix, empty_dir): + self.home_prefix = home_prefix + self.base_prefix = base_prefix + self.empty_dir = empty_dir + + def __call__(self): + import spack.paths + import spack.paths_base + + os.environ["XDG_DATA_HOME"] = "/made-up-value-that-shouldnt-matter" + + + expected = str( + pathlib.Path(self.home_prefix) / ".local" / "share" / "spack" / "installs" + ) + assert spack.paths.locations.default_install_location == expected, \ + f"Expected {expected}\nGot {spack.paths.locations.default_install_location}" + + +def test_child_sanity(working_env, tmp_path, set_home, monkeypatch): + base_prefix = _ensure_dir(tmp_path / "spack-root") + home_prefix = _ensure_dir(tmp_path / "home-prefix") + + empty_dir = _ensure_dir(tmp_path / "empty") + + set_home(home_prefix) + + import spack.paths + + pb = SpackPathsBase(base_prefix) + pb.old_install_path = empty_dir + ptest = spack.paths.SpackPaths(pb) + + monkeypatch.setattr(spack.paths, "locations", ptest) + + spack_process = spack.subprocess_context.SpackTestProcess(SetAnXdgVarAndReadDataHome(home_prefix, base_prefix, empty_dir)) + p = spack_process.create() + p.start() + p.join() + assert p.exitcode == 0 From 13de5a9d4f609aec02b3a7a1b3b4cb38d615c1ae Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 23 Feb 2026 15:37:19 -0800 Subject: [PATCH 308/506] test actually works now, a bit clunky though --- lib/spack/spack/test/paths.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index fd3409c3eb8773..a75ea6a3a9131a 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -281,6 +281,10 @@ def __call__(self): f"Expected {expected}\nGot {spack.paths.locations.default_install_location}" +def all_dirs_empty(a_dir): + return False + + def test_child_sanity(working_env, tmp_path, set_home, monkeypatch): base_prefix = _ensure_dir(tmp_path / "spack-root") home_prefix = _ensure_dir(tmp_path / "home-prefix") @@ -291,11 +295,7 @@ def test_child_sanity(working_env, tmp_path, set_home, monkeypatch): import spack.paths - pb = SpackPathsBase(base_prefix) - pb.old_install_path = empty_dir - ptest = spack.paths.SpackPaths(pb) - - monkeypatch.setattr(spack.paths, "locations", ptest) + monkeypatch.setattr(spack.paths, "dir_is_occupied", all_dirs_empty) spack_process = spack.subprocess_context.SpackTestProcess(SetAnXdgVarAndReadDataHome(home_prefix, base_prefix, empty_dir)) p = spack_process.create() From c3f78cda84d918daeab0f9d2e3b3f32a5dadc59f Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 23 Feb 2026 16:01:21 -0800 Subject: [PATCH 309/506] auto style --- lib/spack/spack/test/paths.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index a75ea6a3a9131a..005675003bbd58 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -9,9 +9,9 @@ import spack.config import spack.paths_base +import spack.subprocess_context from spack.paths import SpackPaths from spack.paths_base import SpackPathsBase -import spack.subprocess_context def _ensure_dir(pathlike): @@ -262,6 +262,7 @@ class SetAnXdgVarAndReadDataHome: """Provide a function which can execute in a separate process that removes a spec from the database. """ + def __init__(self, home_prefix, base_prefix, empty_dir): self.home_prefix = home_prefix self.base_prefix = base_prefix @@ -273,12 +274,10 @@ def __call__(self): os.environ["XDG_DATA_HOME"] = "/made-up-value-that-shouldnt-matter" - - expected = str( - pathlib.Path(self.home_prefix) / ".local" / "share" / "spack" / "installs" - ) - assert spack.paths.locations.default_install_location == expected, \ - f"Expected {expected}\nGot {spack.paths.locations.default_install_location}" + expected = str(pathlib.Path(self.home_prefix) / ".local" / "share" / "spack" / "installs") + assert ( + spack.paths.locations.default_install_location == expected + ), f"Expected {expected}\nGot {spack.paths.locations.default_install_location}" def all_dirs_empty(a_dir): @@ -297,7 +296,9 @@ def test_child_sanity(working_env, tmp_path, set_home, monkeypatch): monkeypatch.setattr(spack.paths, "dir_is_occupied", all_dirs_empty) - spack_process = spack.subprocess_context.SpackTestProcess(SetAnXdgVarAndReadDataHome(home_prefix, base_prefix, empty_dir)) + spack_process = spack.subprocess_context.SpackTestProcess( + SetAnXdgVarAndReadDataHome(home_prefix, base_prefix, empty_dir) + ) p = spack_process.create() p.start() p.join() From 383cbfee837737d74af45c8308eed7963b7f66e9 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 23 Feb 2026 16:07:55 -0800 Subject: [PATCH 310/506] update docstring and test name --- lib/spack/spack/test/paths.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 005675003bbd58..957af88547cd17 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -259,8 +259,8 @@ def test_user_cache_path_is_default_when_env_var_is_empty(tmp_path, set_home): class SetAnXdgVarAndReadDataHome: - """Provide a function which can execute in a separate process that removes - a spec from the database. + """Access an XDG-dependent variable from spack.paths as quickly as + possible. """ def __init__(self, home_prefix, base_prefix, empty_dir): @@ -270,7 +270,6 @@ def __init__(self, home_prefix, base_prefix, empty_dir): def __call__(self): import spack.paths - import spack.paths_base os.environ["XDG_DATA_HOME"] = "/made-up-value-that-shouldnt-matter" @@ -284,7 +283,7 @@ def all_dirs_empty(a_dir): return False -def test_child_sanity(working_env, tmp_path, set_home, monkeypatch): +def test_child_proc_sanity_xdg_based_paths(tmp_path, set_home, monkeypatch): base_prefix = _ensure_dir(tmp_path / "spack-root") home_prefix = _ensure_dir(tmp_path / "home-prefix") From 847ed97ecc4f90f2842d002d2cb5137b1ce03429 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 23 Feb 2026 17:03:31 -0800 Subject: [PATCH 311/506] improvements in test --- lib/spack/spack/paths.py | 6 +++++- lib/spack/spack/subprocess_context.py | 2 +- lib/spack/spack/test/paths.py | 8 +++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index d9f713f96f974d..350dba4d6b8a47 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -238,7 +238,11 @@ def default_misc_cache_path(self): def __getattr__(self, name): # Things that aren't sensitive to import cycles can import the # paths module and access all items from paths_base - return getattr(self.base, name) + try: + base = object.__getattribute__(self, "base") + except AttributeError: + raise AttributeError(name) + return getattr(base, name) def resolve_a_home(self, env_vars, xdg_var, config_var, home_rel): """ diff --git a/lib/spack/spack/subprocess_context.py b/lib/spack/spack/subprocess_context.py index 54e56c505fabfc..6bad47c5e03b17 100644 --- a/lib/spack/spack/subprocess_context.py +++ b/lib/spack/spack/subprocess_context.py @@ -109,11 +109,11 @@ def restore(self): if self.is_forked: return spack.config.CONFIG = self.config - spack.paths.freeze() spack.repo.enable_repo(spack.repo.RepoPath.from_config(self.config)) spack.platforms.host = self.platform spack.store.STORE = self.store self.test_patches.restore() + spack.paths.freeze() if self.env: from spack.environment import activate diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 957af88547cd17..c7d4971e98d339 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -293,7 +293,13 @@ def test_child_proc_sanity_xdg_based_paths(tmp_path, set_home, monkeypatch): import spack.paths - monkeypatch.setattr(spack.paths, "dir_is_occupied", all_dirs_empty) + #monkeypatch.setattr(spack.paths, "dir_is_occupied", all_dirs_empty) + + pbtest = SpackPathsBase(base_prefix) + pbtest.old_install_path = empty_dir + + ptest = SpackPaths(pbtest) + monkeypatch.setattr(spack.paths, "locations", ptest) spack_process = spack.subprocess_context.SpackTestProcess( SetAnXdgVarAndReadDataHome(home_prefix, base_prefix, empty_dir) From b21090a26fe554b487dcca9a4b9c0fa9a8790810 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 23 Feb 2026 17:14:25 -0800 Subject: [PATCH 312/506] cleanup --- lib/spack/spack/test/paths.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index c7d4971e98d339..1bda7298b6c51a 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -263,10 +263,8 @@ class SetAnXdgVarAndReadDataHome: possible. """ - def __init__(self, home_prefix, base_prefix, empty_dir): + def __init__(self, home_prefix): self.home_prefix = home_prefix - self.base_prefix = base_prefix - self.empty_dir = empty_dir def __call__(self): import spack.paths @@ -284,6 +282,9 @@ def all_dirs_empty(a_dir): def test_child_proc_sanity_xdg_based_paths(tmp_path, set_home, monkeypatch): + # Unlike the other tests in this module, this is specifically testing + # the behavior of the spack.paths module vs. (the more targeted testing + # of) classes defined within it. base_prefix = _ensure_dir(tmp_path / "spack-root") home_prefix = _ensure_dir(tmp_path / "home-prefix") @@ -293,16 +294,13 @@ def test_child_proc_sanity_xdg_based_paths(tmp_path, set_home, monkeypatch): import spack.paths - #monkeypatch.setattr(spack.paths, "dir_is_occupied", all_dirs_empty) - pbtest = SpackPathsBase(base_prefix) pbtest.old_install_path = empty_dir - ptest = SpackPaths(pbtest) monkeypatch.setattr(spack.paths, "locations", ptest) spack_process = spack.subprocess_context.SpackTestProcess( - SetAnXdgVarAndReadDataHome(home_prefix, base_prefix, empty_dir) + SetAnXdgVarAndReadDataHome(home_prefix) ) p = spack_process.create() p.start() From f0e5660ee6ad661625766671bbb03e1d3ecade3c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 23 Feb 2026 18:02:50 -0800 Subject: [PATCH 313/506] address forking case --- lib/spack/spack/subprocess_context.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/spack/spack/subprocess_context.py b/lib/spack/spack/subprocess_context.py index 6bad47c5e03b17..f403d649c576a5 100644 --- a/lib/spack/spack/subprocess_context.py +++ b/lib/spack/spack/subprocess_context.py @@ -94,6 +94,9 @@ def __init__( ) -> None: ctx = ctx or multiprocessing.get_context() self.is_forked = ctx.get_start_method() == "fork" + # Note: you must call "freeze" before for fork; you must call freeze + # after for spawn. + spack.paths.freeze() if self.is_forked: return From 78c809e6782a9bf5a410abaa3e5b8f8c5df8c4ca Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 23 Feb 2026 18:06:54 -0800 Subject: [PATCH 314/506] update comment --- lib/spack/spack/subprocess_context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/subprocess_context.py b/lib/spack/spack/subprocess_context.py index f403d649c576a5..a79f41de5d4651 100644 --- a/lib/spack/spack/subprocess_context.py +++ b/lib/spack/spack/subprocess_context.py @@ -94,8 +94,8 @@ def __init__( ) -> None: ctx = ctx or multiprocessing.get_context() self.is_forked = ctx.get_start_method() == "fork" - # Note: you must call "freeze" before for fork; you must call freeze - # after for spawn. + # Note: freezing before is sufficient for fork. Spawn starts + # must also freeze after spack.paths.freeze() if self.is_forked: return From 563759dd62308384466845f06a8fc32f4febe5fa Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sat, 28 Feb 2026 00:03:02 -0800 Subject: [PATCH 315/506] redirect via config --- lib/spack/spack/test/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index ec9e3de016e37e..6b776638314a52 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -1424,7 +1424,7 @@ def module_configuration(monkeypatch, request, mutable_config): @pytest.fixture() -def mock_gnupghome(monkeypatch, tmp_path): +def mock_gnupghome(tmp_path, mutable_config): # GNU PGP can't handle paths longer than 108 characters (wtf!@#$) so we # have to make our own tmp_path with a shorter name than pytest's. # This comes up because tmp paths on macOS are already long-ish, and @@ -1432,7 +1432,7 @@ def mock_gnupghome(monkeypatch, tmp_path): short_name_tmpdir = tempfile.mkdtemp() # Redirect bootstrap root before gpg.init() so each xdist worker writes # bootstrap config to its own isolated directory. - monkeypatch.setattr(spack.paths, "default_user_bootstrap_path", str(tmp_path / "bootstrap")) + mutable_config.set("bootstrap:root", str(tmp_path / "bootstrap-test")) try: spack.util.gpg.init() except spack.util.gpg.SpackGPGError: From dded4496a24d331c0478d3f70eb6da483c9d8b7b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Mar 2026 15:33:55 -0800 Subject: [PATCH 316/506] include.yaml section cannot refer to location config vars --- etc/spack/include.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/spack/include.yaml b/etc/spack/include.yaml index a4c04b94616405..b8ab4ebfeb3b8b 100644 --- a/etc/spack/include.yaml +++ b/etc/spack/include.yaml @@ -2,7 +2,7 @@ include: # user configuration scope - name: "user" path_override_env_var: SPACK_USER_CONFIG_PATH - path: "$spack_home/.config/spack/" + path: "~/.config/spack/" optional: true prefer_modify: true when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' From 2fa5b9e7bde4d2f7e2e9d4b567e48411d253f9ed Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Mar 2026 15:53:06 -0800 Subject: [PATCH 317/506] Update docs to not explicitly list old format: the instructions that follow will cover this anyway --- lib/spack/docs/chain.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/docs/chain.rst b/lib/spack/docs/chain.rst index 0bab7ab8d371d4..ec572e7968047f 100644 --- a/lib/spack/docs/chain.rst +++ b/lib/spack/docs/chain.rst @@ -19,7 +19,7 @@ To register the other Spack instance, you can add it as an entry to ``upstreams. install-tree-1: install_tree: /path/to/another/spack/install/tree install-tree-2: - install_tree: /path/to/old/spack/opt/spack + install_tree: /path/to/yet/another/spack/install/tree The ``install_tree`` must point to a directory containing a Spack install tree, as defined in :ref:`config.yaml `. For a given instance of Spack, you can determine the directory to add here by running ``spack location --install-root`` in that instance, or with ``spack config get config``. From 62eaa71fcda929b81178022ae3661982f97eb52d Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Mar 2026 16:15:41 -0800 Subject: [PATCH 318/506] update docs to be more specific about exact data_home; update test --- lib/spack/docs/configuration.rst | 6 +++--- lib/spack/spack/test/paths.py | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index ecc8eda2067bb5..d02c2b82b68b3a 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -551,10 +551,10 @@ Furthermore each is controlled by a fall-through scheme. For example ``$data_home`` evaluates to one of the following (highest-priority first): #. ``SPACK_DATA_HOME`` env var if that is set -#. ``SPACK_HOME`` env var +#. Under ``SPACK_HOME`` env var; for ``$data_home``, it is this plus ``.local/share/spack`` #. ``config:locations:data`` -#. ``config:locations:home`` -#. ``XDG_DATA_HOME`` env var if that is set +#. Under ``config:locations:home``; for ``$data_home`` it is this plus ``.local/share/spack`` +#. ``XDG_DATA_HOME/spack`` if XDG_DATA_HOME is set #. Under the default for ``XDG_DATA_HOME``: ``~/.local/share/spack`` Of particular interest is where the environments and installs are placed by Spack, because these can take up a lot of space. diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 1bda7298b6c51a..82525db8461eca 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -138,11 +138,11 @@ def paths_base_nonempty_old_install(): def _unconditional_path_override_checks(tmp_path, base_paths_generator): # "config:locations:home" variable overrides the above (even if there # are no installs there and there are installs in the old location) - spack_home_prefix = _ensure_dir(tmp_path / "spack-home") - spack.config.set("config:locations:home", spack_home_prefix) + spack_home_cfg_prefix = _ensure_dir(tmp_path / "spack-home2") + spack.config.set("config:locations:home", spack_home_cfg_prefix) p2 = SpackPaths(base_paths_generator()) assert p2.default_install_location == str( - pathlib.Path(spack_home_prefix) / ".local" / "share" / "spack" / "installs" + pathlib.Path(spack_home_cfg_prefix) / ".local" / "share" / "spack" / "installs" ) assert not p2.bypassed_old_installs_warning(_show=False) @@ -152,6 +152,15 @@ def _unconditional_path_override_checks(tmp_path, base_paths_generator): p3 = SpackPaths(base_paths_generator()) assert p3.default_install_location == str(pathlib.Path(spack_data_prefix) / "installs") + # SPACK_HOME env variable overrides the above (even if there + # are no installs there and there are installs in the old location) + spack_home_env_prefix = _ensure_dir(tmp_path / "spack-home1") + os.environ["SPACK_HOME"] = spack_home_env_prefix + p1 = SpackPaths(base_paths_generator()) + assert p1.default_install_location == str( + pathlib.Path(spack_home_env_prefix) / ".local" / "share" / "spack" / "installs" + ) + # Check that $SPACK_DATA_HOME overrides all the above spack_data_home = _ensure_dir(tmp_path / "spack_data_home") os.environ["SPACK_DATA_HOME"] = spack_data_home From bbdc45b171ae645d7e378d9f5433b0cfc2ca26cb Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Mar 2026 16:49:45 -0800 Subject: [PATCH 319/506] grammar in docs --- lib/spack/docs/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index d02c2b82b68b3a..715f3d6c174259 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -563,7 +563,7 @@ Older installs of spack placed these within ``$spack``, and fallback scheme in t * ``$default_install_root``: the location where installs go by default. Overridden by ``config:install_tree:root``. - Prefers ``$data_home/installs``, but if there are no installs there but there are installs in the old location ``$spack/opt/spack``, then the old location will be used. + Prefers ``$data_home/installs``, but if there are no installs there and there are installs in the old location ``$spack/opt/spack``, then the old location will be used. * ``$default_envs_root``: the location where environments are managed by default. Overridden by ``config:environments_root``. Prefers ``$data_home/envs`` but if there are no envs there and there are envs in the old location ``$spack/var/spack/environments``, then the old location will be used. From e538173e8b90dd67dc80a7af0450199e4e5095ae Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Mar 2026 17:34:58 -0800 Subject: [PATCH 320/506] partial reorg of resolve_a_home --- lib/spack/spack/paths.py | 44 ++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 350dba4d6b8a47..af2fc6e57f6aef 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -11,6 +11,7 @@ import os import pathlib from enum import Enum +from functools import partial import spack.config as config import spack.llnl.util.tty as tty @@ -244,7 +245,7 @@ def __getattr__(self, name): raise AttributeError(name) return getattr(base, name) - def resolve_a_home(self, env_vars, xdg_var, config_var, home_rel): + def resolve_a_home(self, spack_vars, xdg_var, config_var, home_rel): """ Data stored by spack is split into state, data, and cache components. This function can resolve where each of these components should be @@ -263,32 +264,6 @@ def resolve_a_home(self, env_vars, xdg_var, config_var, home_rel): """ disable_env = config.get("config:locations:disable_env", False) - def spack_env_check(): - if disable_env: - return - if isinstance(env_vars, str): - x = [env_vars] - else: - x = env_vars - for n in x: - if n in os.environ: - return os.environ[n], Provenance.SPACK_ENV - - def xdg_env_check(): - if disable_env: - return - if xdg_var in os.environ: - return os.path.join(os.environ[xdg_var], "spack"), Provenance.XDG_VAR - - def spack_home_env_check(): - if disable_env: - return - if "SPACK_HOME" in os.environ: - return ( - os.path.join(os.environ["SPACK_HOME"], home_rel, "spack"), - Provenance.SPACK_HOME_ENV, - ) - def cfg_check(): val = config.get(f"config:locations:{config_var}", None) if val: @@ -299,6 +274,21 @@ def spack_home_cfg_check(): if h: return os.path.join(h, home_rel, "spack"), Provenance.CONFIG_HOME_VAR + def env_check(env_vars, provenance, rel=None): + if disable_env: + return + + rel = rel or "" + + for v in env_vars: + if v in os.environ: + return os.path.join(os.environ[v], rel), provenance + + spack_vars = [spack_vars] if isinstance(spack_vars, str) else spack_vars + spack_env_check = partial(env_check, spack_vars, Provenance.SPACK_ENV) + spack_home_env_check = partial(env_check, ["SPACK_HOME"], Provenance.SPACK_HOME_ENV, rel=os.path.join(home_rel, "spack")) + xdg_env_check = partial(env_check, [xdg_var], Provenance.XDG_VAR, rel="spack") + for check in [ spack_env_check, spack_home_env_check, From 83f7ce43d77f6ebea9986717764a07b7f3ad4323 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Mar 2026 17:38:06 -0800 Subject: [PATCH 321/506] avoid appending trailing slash --- lib/spack/spack/paths.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index af2fc6e57f6aef..b5daef905107d1 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -278,11 +278,14 @@ def env_check(env_vars, provenance, rel=None): if disable_env: return - rel = rel or "" + if rel: + full_path = lambda x: os.path.join(x, rel) + else: + full_path = lambda x: x for v in env_vars: if v in os.environ: - return os.path.join(os.environ[v], rel), provenance + return full_path(os.environ[v]), provenance spack_vars = [spack_vars] if isinstance(spack_vars, str) else spack_vars spack_env_check = partial(env_check, spack_vars, Provenance.SPACK_ENV) From 5ff784cf4cbafca7f87bdc4a0c5bc411e3bcdfbc Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Mar 2026 17:39:15 -0800 Subject: [PATCH 322/506] auto style --- lib/spack/spack/paths.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index b5daef905107d1..2cb3652b148618 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -289,7 +289,12 @@ def env_check(env_vars, provenance, rel=None): spack_vars = [spack_vars] if isinstance(spack_vars, str) else spack_vars spack_env_check = partial(env_check, spack_vars, Provenance.SPACK_ENV) - spack_home_env_check = partial(env_check, ["SPACK_HOME"], Provenance.SPACK_HOME_ENV, rel=os.path.join(home_rel, "spack")) + spack_home_env_check = partial( + env_check, + ["SPACK_HOME"], + Provenance.SPACK_HOME_ENV, + rel=os.path.join(home_rel, "spack"), + ) xdg_env_check = partial(env_check, [xdg_var], Provenance.XDG_VAR, rel="spack") for check in [ From eda78754ba6d04979e62d776171dc586e834dcad Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Mar 2026 17:40:39 -0800 Subject: [PATCH 323/506] update docstr --- lib/spack/spack/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 2cb3652b148618..6c0d92b9a6f0ef 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -252,7 +252,7 @@ def resolve_a_home(self, spack_vars, xdg_var, config_var, home_rel): stored. Args: - env_vars: spack-specific environment variables that indicate the + spack_vars: spack-specific environment variables that indicate the component location. Can be a list or a single variable. If this is a list, earlier elements have precedence. xdg_var: the XDG-based environment variable that indicates the From d931bd42670065e9efea68ce95497f00616f724b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Mar 2026 17:54:44 -0800 Subject: [PATCH 324/506] also make cfg checks based off of a reused function, vs separately defined functions --- lib/spack/spack/paths.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 6c0d92b9a6f0ef..8cd6d3d051df92 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -264,28 +264,23 @@ def resolve_a_home(self, spack_vars, xdg_var, config_var, home_rel): """ disable_env = config.get("config:locations:disable_env", False) - def cfg_check(): - val = config.get(f"config:locations:{config_var}", None) - if val: - return val, Provenance.CONFIG_VAR + append_rel = lambda base, rel: str(pathlib.Path(base) / (rel or "")) - def spack_home_cfg_check(): - h = config.get("config:locations:home", None) - if h: - return os.path.join(h, home_rel, "spack"), Provenance.CONFIG_HOME_VAR + def cfg_check(path, provenance, rel=None): + found = config.get(path, None) + if found: + return append_rel(found, rel), provenance + + spack_cfg_check = partial(cfg_check, f"config:locations:{config_var}", Provenance.CONFIG_VAR) + spack_home_cfg_check = partial(cfg_check, "config:locations:home", Provenance.CONFIG_HOME_VAR, rel=os.path.join(home_rel, "spack")) def env_check(env_vars, provenance, rel=None): if disable_env: return - if rel: - full_path = lambda x: os.path.join(x, rel) - else: - full_path = lambda x: x - for v in env_vars: if v in os.environ: - return full_path(os.environ[v]), provenance + return append_rel(os.environ[v], rel), provenance spack_vars = [spack_vars] if isinstance(spack_vars, str) else spack_vars spack_env_check = partial(env_check, spack_vars, Provenance.SPACK_ENV) @@ -300,7 +295,7 @@ def env_check(env_vars, provenance, rel=None): for check in [ spack_env_check, spack_home_env_check, - cfg_check, + spack_cfg_check, spack_home_cfg_check, xdg_env_check, ]: From 126287ddf07bf213de0024785f41194d37f539a6 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 5 Mar 2026 17:56:39 -0800 Subject: [PATCH 325/506] auto style --- lib/spack/spack/paths.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 8cd6d3d051df92..122e70ec237def 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -271,8 +271,15 @@ def cfg_check(path, provenance, rel=None): if found: return append_rel(found, rel), provenance - spack_cfg_check = partial(cfg_check, f"config:locations:{config_var}", Provenance.CONFIG_VAR) - spack_home_cfg_check = partial(cfg_check, "config:locations:home", Provenance.CONFIG_HOME_VAR, rel=os.path.join(home_rel, "spack")) + spack_cfg_check = partial( + cfg_check, f"config:locations:{config_var}", Provenance.CONFIG_VAR + ) + spack_home_cfg_check = partial( + cfg_check, + "config:locations:home", + Provenance.CONFIG_HOME_VAR, + rel=os.path.join(home_rel, "spack"), + ) def env_check(env_vars, provenance, rel=None): if disable_env: From 248e53623c2c43a3e7d2c2a085300e850f931bd7 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Mar 2026 16:14:53 -0800 Subject: [PATCH 326/506] rm site-admin scope - I think include: config section can address the needs this was intended to support --- lib/spack/spack/config.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 42be4ba4dea1a5..9b563b4d9e3c69 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1422,21 +1422,6 @@ def create_incremental() -> Generator[Configuration, None, None]: # line scopes go above this. configuration_paths = [("spack", os.path.join(spack.paths_base.etc_path))] - # Site admin scope has two uses: (a) admins can share config with one - # another, but not with end users (b) pip/apt-installed spack can - # change the default install root - site_admin_path = os.path.join(spack.paths_base.etc_path, "site-admin") - site_admin_accessible = False - try: - if os.path.isdir(site_admin_path): - os.listdir(site_admin_path) - site_admin_accessible = True - except PermissionError: - pass - if site_admin_accessible: - # TODO: list this as an available scope, but not a writable one - configuration_paths.append(("site-admin", site_admin_path)) - # Python packages can register configuration scopes via entry_points configuration_paths.extend(config_paths_from_entry_points()) From 2cd64883a9c8c1a5f38e2d2037a2c0d07c70f0a5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Mar 2026 18:51:29 -0800 Subject: [PATCH 327/506] allow path vars to reference other path vars in locations config --- lib/spack/spack/paths.py | 9 ++---- lib/spack/spack/paths_base.py | 7 +++++ lib/spack/spack/test/paths.py | 17 ++++++++++++ lib/spack/spack/util/path.py | 52 +++++++++++++++++++++++++---------- 4 files changed, 65 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 122e70ec237def..055200eb068877 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -16,7 +16,6 @@ import spack.config as config import spack.llnl.util.tty as tty import spack.paths_base as paths_base -import spack.util.hash as hash def dir_is_occupied(x, except_for=None): @@ -72,11 +71,6 @@ def __init__(self, base): ] ) - #: Not a location itself, but used for when Spack instances - #: share the same cache base directory for caches that should - #: not be shared between those instances. - self.spack_instance_id = hash.b32_hash(base.prefix)[:7] - @property def state_home(self): if not self._state_home: @@ -269,6 +263,9 @@ def resolve_a_home(self, spack_vars, xdg_var, config_var, home_rel): def cfg_check(path, provenance, rel=None): found = config.get(path, None) if found: + import spack.util.path + with spack.util.path.limited_paths(): + found = spack.util.path.canonicalize_path(found) return append_rel(found, rel), provenance spack_cfg_check = partial( diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index 0030b77a31e95c..71657de55a75f2 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -7,6 +7,7 @@ from enum import Enum from pathlib import PurePath +import spack.util.hash as hash import spack.llnl.util.filesystem @@ -88,6 +89,11 @@ def __init__(self, _prefix=None): os.getenv("SPACK_SYSTEM_CONFIG_PATH") or os.sep + os.path.join("etc", "spack") ) + #: Not a location itself, but used for when Spack instances + #: share the same cache base directory for caches that should + #: not be shared between those instances. + self.spack_instance_id = hash.b32_hash(self.prefix)[:7] + @property def env_based_state_home(self): """Spack has config-based logic for choosing a home for most state, but @@ -127,6 +133,7 @@ def env_based_state_home(self): mock_gpg_keys_path = locations.mock_gpg_keys_path system_config_path = locations.system_config_path user_config_path = locations.user_config_path +spack_instance_id = locations.spack_instance_id #: Recorded directory where spack command was originally invoked diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 82525db8461eca..4a1f5436f9473f 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -10,6 +10,7 @@ import spack.config import spack.paths_base import spack.subprocess_context +import spack.paths from spack.paths import SpackPaths from spack.paths_base import SpackPathsBase @@ -267,6 +268,22 @@ def test_user_cache_path_is_default_when_env_var_is_empty(tmp_path, set_home): ) +def test_location_vars_that_use_other_location_vars(tmp_path, set_home, mutable_config, monkeypatch): + homedir = _ensure_dir(tmp_path / "test-home") + set_home(homedir) + + basedir = _ensure_dir(tmp_path / "spack-root") + + mutable_config.set("config", {"locations": {"home": "$spack/home"}}) + + p1 = SpackPaths(SpackPathsBase(str(basedir))) + # This is a bit strange but resolution of the config variable involves accessing + # the module, so we need to monkeypatch that + monkeypatch.setattr(spack.paths_base, "locations", p1.base) + install_rel = pathlib.Path(".local") / "share" / "spack" / "installs" + assert p1.default_install_location == str(pathlib.Path(basedir) / "home" / install_rel) + + class SetAnXdgVarAndReadDataHome: """Access an XDG-dependent variable from spack.paths as quickly as possible. diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 6cdb80d21b6d02..41384c6e4bdab4 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -7,6 +7,7 @@ TODO: this is really part of spack.config. Consolidate it. """ import contextlib +from enum import Enum import getpass import os import pathlib @@ -52,31 +53,37 @@ def get_user(): NOMATCH = object() +class ResolutionContext(Enum): + IN_PATHS = 1 + ABOVE_PATHS = 2 + + +_resolution_context = ResolutionContext.ABOVE_PATHS + + +@contextlib.contextmanager +def limited_paths(): + global _resolution_context + _resolution_context = ResolutionContext.IN_PATHS + yield + _resolution_context = ResolutionContext.ABOVE_PATHS + + # Substitutions to perform def replacements(): # break circular imports import spack import spack.environment as ev - import spack.paths + import spack.paths_base arch = architecture() - paths = spack.paths.locations - - return { - "spack": lambda: spack.paths_base.prefix, + replace = { + "spack": lambda: spack.paths_base.locations.prefix, "user": lambda: get_user(), "tempdir": lambda: tempfile.gettempdir(), - "user_cache_path": lambda: paths.user_cache_path, - "default_install_root": lambda: paths.default_install_location, - "default_envs_root": lambda: paths.default_envs_path, - "modules_base": lambda: paths.modules_base, - "data_home": lambda: paths.data_home, - "cache_home": lambda: paths.cache_home, - "state_home": lambda: paths.state_home, - "spack_home": lambda: paths.spack_home, - "spack_instance_id": lambda: paths.spack_instance_id, + "spack_instance_id": lambda: spack.paths_base.locations.spack_instance_id, "architecture": lambda: arch, "arch": lambda: arch, "platform": lambda: arch.platform, @@ -89,6 +96,23 @@ def replacements(): "spack_short_version": lambda: spack.get_short_version(), } + global _resolution_context + if _resolution_context == ResolutionContext.ABOVE_PATHS: + import spack.paths + paths = spack.paths.locations + replace.update({ + "user_cache_path": lambda: paths.user_cache_path, + "default_install_root": lambda: paths.default_install_location, + "default_envs_root": lambda: paths.default_envs_path, + "modules_base": lambda: paths.modules_base, + "data_home": lambda: paths.data_home, + "cache_home": lambda: paths.cache_home, + "state_home": lambda: paths.state_home, + "spack_home": lambda: paths.spack_home, + }) + + return replace + # This is intended to be longer than the part of the install path # spack generates from the root path we give it. Included in the From 354a49aefee80f75105c15e377d43919b041de3e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 6 Mar 2026 18:54:44 -0800 Subject: [PATCH 328/506] style edit --- lib/spack/spack/paths.py | 1 + lib/spack/spack/paths_base.py | 2 +- lib/spack/spack/test/paths.py | 6 ++++-- lib/spack/spack/util/path.py | 27 ++++++++++++++------------- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 055200eb068877..97db9d8f12cacd 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -264,6 +264,7 @@ def cfg_check(path, provenance, rel=None): found = config.get(path, None) if found: import spack.util.path + with spack.util.path.limited_paths(): found = spack.util.path.canonicalize_path(found) return append_rel(found, rel), provenance diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index 71657de55a75f2..ec3c265674066e 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -7,8 +7,8 @@ from enum import Enum from pathlib import PurePath -import spack.util.hash as hash import spack.llnl.util.filesystem +import spack.util.hash as hash class XDG_vars(Enum): diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 4a1f5436f9473f..b336bfc19566ab 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -8,9 +8,9 @@ import pytest import spack.config +import spack.paths import spack.paths_base import spack.subprocess_context -import spack.paths from spack.paths import SpackPaths from spack.paths_base import SpackPathsBase @@ -268,7 +268,9 @@ def test_user_cache_path_is_default_when_env_var_is_empty(tmp_path, set_home): ) -def test_location_vars_that_use_other_location_vars(tmp_path, set_home, mutable_config, monkeypatch): +def test_location_vars_that_use_other_location_vars( + tmp_path, set_home, mutable_config, monkeypatch +): homedir = _ensure_dir(tmp_path / "test-home") set_home(homedir) diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 41384c6e4bdab4..c3cb61a610a9a6 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -7,7 +7,6 @@ TODO: this is really part of spack.config. Consolidate it. """ import contextlib -from enum import Enum import getpass import os import pathlib @@ -16,6 +15,7 @@ import sys import tempfile from datetime import date +from enum import Enum from typing import Optional import spack.llnl.util.tty as tty @@ -74,7 +74,6 @@ def replacements(): # break circular imports import spack import spack.environment as ev - import spack.paths_base arch = architecture() @@ -96,20 +95,22 @@ def replacements(): "spack_short_version": lambda: spack.get_short_version(), } - global _resolution_context if _resolution_context == ResolutionContext.ABOVE_PATHS: import spack.paths + paths = spack.paths.locations - replace.update({ - "user_cache_path": lambda: paths.user_cache_path, - "default_install_root": lambda: paths.default_install_location, - "default_envs_root": lambda: paths.default_envs_path, - "modules_base": lambda: paths.modules_base, - "data_home": lambda: paths.data_home, - "cache_home": lambda: paths.cache_home, - "state_home": lambda: paths.state_home, - "spack_home": lambda: paths.spack_home, - }) + replace.update( + { + "user_cache_path": lambda: paths.user_cache_path, + "default_install_root": lambda: paths.default_install_location, + "default_envs_root": lambda: paths.default_envs_path, + "modules_base": lambda: paths.modules_base, + "data_home": lambda: paths.data_home, + "cache_home": lambda: paths.cache_home, + "state_home": lambda: paths.state_home, + "spack_home": lambda: paths.spack_home, + } + ) return replace From 4c3e74b11878b9d73b44a40933c6ff6f894bbc97 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Mar 2026 11:44:49 -0700 Subject: [PATCH 329/506] rm context manager (still somewhat messy, but easier to understand) --- lib/spack/spack/paths.py | 3 +-- lib/spack/spack/util/path.py | 36 +++++++++++------------------------- 2 files changed, 12 insertions(+), 27 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 97db9d8f12cacd..4dc1f3196a10aa 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -265,8 +265,7 @@ def cfg_check(path, provenance, rel=None): if found: import spack.util.path - with spack.util.path.limited_paths(): - found = spack.util.path.canonicalize_path(found) + found = spack.util.path.canonicalize_path(found, _use_config=False) return append_rel(found, rel), provenance spack_cfg_check = partial( diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 1e90d2f607123b..0568d5aae1dc3a 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -15,7 +15,6 @@ import sys import tempfile from datetime import date -from enum import Enum from typing import Optional import spack.llnl.util.tty as tty @@ -53,24 +52,8 @@ def get_user(): NOMATCH = object() -class ResolutionContext(Enum): - IN_PATHS = 1 - ABOVE_PATHS = 2 - - -_resolution_context = ResolutionContext.ABOVE_PATHS - - -@contextlib.contextmanager -def limited_paths(): - global _resolution_context - _resolution_context = ResolutionContext.IN_PATHS - yield - _resolution_context = ResolutionContext.ABOVE_PATHS - - # Substitutions to perform -def replacements(): +def replacements(_use_config=True): # break circular imports import spack import spack.environment as ev @@ -95,7 +78,7 @@ def replacements(): "spack_short_version": lambda: spack.get_short_version(), } - if _resolution_context == ResolutionContext.ABOVE_PATHS: + if _use_config: import spack.paths paths = spack.paths.locations @@ -187,7 +170,7 @@ def get_system_path_max(): return sys_max_path_length -def substitute_config_variables(path): +def substitute_config_variables(path, _use_config): """Substitute placeholders into paths. Spack allows paths in configs to have some placeholders, as follows: @@ -218,7 +201,7 @@ def substitute_config_variables(path): replaced if there is an active environment, and should only be used in environment yaml files. """ - _replacements = replacements() + _replacements = replacements(_use_config=_use_config) # Look up replacements def repl(match): @@ -231,9 +214,9 @@ def repl(match): return re.sub(r"(\$\w+\b|\$\{\w+\})", repl, path) -def substitute_path_variables(path): +def substitute_path_variables(path, _use_config=True): """Substitute config vars, expand environment vars, expand user home.""" - path = substitute_config_variables(path) + path = substitute_config_variables(path, _use_config=_use_config) path = os.path.expandvars(path) path = os.path.expanduser(path) return path @@ -278,7 +261,7 @@ def add_padding(path, length): return os.path.join(path, padding) -def canonicalize_path(path: str, default_wd: Optional[str] = None) -> str: +def canonicalize_path(path: str, default_wd: Optional[str] = None, _use_config=True) -> str: """Same as substitute_path_variables, but also take absolute path. If the string is a yaml object with file annotations, make absolute paths @@ -291,6 +274,9 @@ def canonicalize_path(path: str, default_wd: Optional[str] = None) -> str: Returns: An absolute path or non-file URL with path variable substitution """ + # _use_config: whether or not to use config to resolve spack config + # variables. This should be false if this function is called in a context that + # is defining one of these variables. import urllib.parse import urllib.request @@ -301,7 +287,7 @@ def canonicalize_path(path: str, default_wd: Optional[str] = None) -> str: filename = os.path.dirname(path._start_mark.name) # type: ignore[attr-defined] assert path._start_mark.name == path._end_mark.name # type: ignore[attr-defined] - path = substitute_path_variables(path) + path = substitute_path_variables(path, _use_config=_use_config) # Ensure properly process a Windows path win_path = pathlib.PureWindowsPath(path) From fd19344585ead93fa39243d348baddacbcd44734 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Mar 2026 13:53:46 -0700 Subject: [PATCH 330/506] add explicit failure for attempts to define data_home in terms of spack_home --- lib/spack/spack/paths.py | 6 +++++- lib/spack/spack/test/paths.py | 6 ++++++ lib/spack/spack/util/path.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 4dc1f3196a10aa..c38da001266ab8 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -265,7 +265,11 @@ def cfg_check(path, provenance, rel=None): if found: import spack.util.path - found = spack.util.path.canonicalize_path(found, _use_config=False) + try: + found = spack.util.path.canonicalize_path(found, _use_config=False) + except spack.util.path.ResolutionContextError as e: + msg = f"{path} is defined in terms of another location ({e.var})" + raise ValueError(msg) return append_rel(found, rel), provenance spack_cfg_check = partial( diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index b336bfc19566ab..fd8d3f47e9d6ef 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -285,6 +285,12 @@ def test_location_vars_that_use_other_location_vars( install_rel = pathlib.Path(".local") / "share" / "spack" / "installs" assert p1.default_install_location == str(pathlib.Path(basedir) / "home" / install_rel) + p2 = SpackPaths(SpackPathsBase(str(basedir))) + monkeypatch.setattr(spack.paths_base, "locations", p2.base) + mutable_config.set("config", {"locations": {"home": "$spack/home", "data": "$spack_home"}}) + with pytest.raises(ValueError, match=r"config\:locations\:data.*spack_home"): + p2.default_install_location + class SetAnXdgVarAndReadDataHome: """Access an XDG-dependent variable from spack.paths as quickly as diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 0568d5aae1dc3a..894f3be718cd23 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -52,6 +52,21 @@ def get_user(): NOMATCH = object() +class ResolutionContextError(ValueError): + def __init__(self, var): + msg = f"Cannot resolve {var}" + super().__init__(msg) + self.var = var + + +class CannotResolve: + def __init__(self, var): + self.var = var + + def __call__(self): + raise ResolutionContextError(self.var) + + # Substitutions to perform def replacements(_use_config=True): # break circular imports @@ -94,6 +109,22 @@ def replacements(_use_config=True): "spack_home": lambda: paths.spack_home, } ) + else: + error_resolution = lambda x: CannotResolve(x) + mappings = list( + (x, error_resolution(x)) + for x in [ + "user_cache_path", + "default_install_root", + "default_envs_root", + "modules_base", + "data_home", + "cache_home", + "state_home", + "spack_home", + ] + ) + replace.update(dict(mappings)) return replace From 7d72b8a8b41a35d54ab550f3523c682138838808 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Mar 2026 14:31:34 -0700 Subject: [PATCH 331/506] intermediate: it works to resolve config vars in terms of other config vars, but the canonicalization needs to account for this --- lib/spack/spack/paths.py | 2 +- lib/spack/spack/test/paths.py | 4 ++-- lib/spack/spack/util/path.py | 17 +++++++++++------ 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index c38da001266ab8..53064f5061e5bf 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -266,7 +266,7 @@ def cfg_check(path, provenance, rel=None): import spack.util.path try: - found = spack.util.path.canonicalize_path(found, _use_config=False) + found = spack.util.path.canonicalize_path(found, _use_config=True, _recursive=True) except spack.util.path.ResolutionContextError as e: msg = f"{path} is defined in terms of another location ({e.var})" raise ValueError(msg) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index fd8d3f47e9d6ef..4f0997e3ac8f33 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -285,11 +285,11 @@ def test_location_vars_that_use_other_location_vars( install_rel = pathlib.Path(".local") / "share" / "spack" / "installs" assert p1.default_install_location == str(pathlib.Path(basedir) / "home" / install_rel) + # Now try defining a $data_home -> $spack_home -> $spack p2 = SpackPaths(SpackPathsBase(str(basedir))) monkeypatch.setattr(spack.paths_base, "locations", p2.base) mutable_config.set("config", {"locations": {"home": "$spack/home", "data": "$spack_home"}}) - with pytest.raises(ValueError, match=r"config\:locations\:data.*spack_home"): - p2.default_install_location + assert p2.default_install_location == str(pathlib.Path(basedir) / "home" / "installs") class SetAnXdgVarAndReadDataHome: diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 894f3be718cd23..5491a46dcfe76f 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -245,11 +245,16 @@ def repl(match): return re.sub(r"(\$\w+\b|\$\{\w+\})", repl, path) -def substitute_path_variables(path, _use_config=True): +def substitute_path_variables(path, _use_config=True, _recursive=False): """Substitute config vars, expand environment vars, expand user home.""" - path = substitute_config_variables(path, _use_config=_use_config) - path = os.path.expandvars(path) - path = os.path.expanduser(path) + prev = path + while True: + path = substitute_config_variables(path, _use_config=_use_config) + path = os.path.expandvars(path) + path = os.path.expanduser(path) + if (not _recursive) or (path == prev): + break + prev = path return path @@ -292,7 +297,7 @@ def add_padding(path, length): return os.path.join(path, padding) -def canonicalize_path(path: str, default_wd: Optional[str] = None, _use_config=True) -> str: +def canonicalize_path(path: str, default_wd: Optional[str] = None, _use_config=True, _recursive=False) -> str: """Same as substitute_path_variables, but also take absolute path. If the string is a yaml object with file annotations, make absolute paths @@ -318,7 +323,7 @@ def canonicalize_path(path: str, default_wd: Optional[str] = None, _use_config=T filename = os.path.dirname(path._start_mark.name) # type: ignore[attr-defined] assert path._start_mark.name == path._end_mark.name # type: ignore[attr-defined] - path = substitute_path_variables(path, _use_config=_use_config) + path = substitute_path_variables(path, _use_config=_use_config, _recursive=_recursive) # Ensure properly process a Windows path win_path = pathlib.PureWindowsPath(path) From b17e9457348b69ccf7c6b1dd0683fd9c85e64edd Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Mar 2026 14:38:12 -0700 Subject: [PATCH 332/506] remove logic for disallowing resolution of certain config vars in the context of resolving other config vars --- lib/spack/spack/paths.py | 2 +- lib/spack/spack/util/path.py | 59 ++++++++++-------------------------- 2 files changed, 17 insertions(+), 44 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 53064f5061e5bf..808b4067b7bd54 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -266,7 +266,7 @@ def cfg_check(path, provenance, rel=None): import spack.util.path try: - found = spack.util.path.canonicalize_path(found, _use_config=True, _recursive=True) + found = spack.util.path.canonicalize_path(found, _recursive=True) except spack.util.path.ResolutionContextError as e: msg = f"{path} is defined in terms of another location ({e.var})" raise ValueError(msg) diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 5491a46dcfe76f..1d13b0c3bfbdcb 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -68,10 +68,11 @@ def __call__(self): # Substitutions to perform -def replacements(_use_config=True): +def replacements(): # break circular imports import spack import spack.environment as ev + import spack.paths import spack.paths_base arch = architecture() @@ -91,41 +92,16 @@ def replacements(_use_config=True): "date": lambda: date.today().strftime("%Y-%m-%d"), "env": lambda: ev.active_environment().path if ev.active_environment() else NOMATCH, "spack_short_version": lambda: spack.get_short_version(), + "user_cache_path": lambda: spack.paths.locations.user_cache_path, + "default_install_root": lambda: spack.paths.locations.default_install_location, + "default_envs_root": lambda: spack.paths.locations.default_envs_path, + "modules_base": lambda: spack.paths.locations.modules_base, + "data_home": lambda: spack.paths.locations.data_home, + "cache_home": lambda: spack.paths.locations.cache_home, + "state_home": lambda: spack.paths.locations.state_home, + "spack_home": lambda: spack.paths.locations.spack_home, } - if _use_config: - import spack.paths - - paths = spack.paths.locations - replace.update( - { - "user_cache_path": lambda: paths.user_cache_path, - "default_install_root": lambda: paths.default_install_location, - "default_envs_root": lambda: paths.default_envs_path, - "modules_base": lambda: paths.modules_base, - "data_home": lambda: paths.data_home, - "cache_home": lambda: paths.cache_home, - "state_home": lambda: paths.state_home, - "spack_home": lambda: paths.spack_home, - } - ) - else: - error_resolution = lambda x: CannotResolve(x) - mappings = list( - (x, error_resolution(x)) - for x in [ - "user_cache_path", - "default_install_root", - "default_envs_root", - "modules_base", - "data_home", - "cache_home", - "state_home", - "spack_home", - ] - ) - replace.update(dict(mappings)) - return replace @@ -201,7 +177,7 @@ def get_system_path_max(): return sys_max_path_length -def substitute_config_variables(path, _use_config): +def substitute_config_variables(path): """Substitute placeholders into paths. Spack allows paths in configs to have some placeholders, as follows: @@ -232,7 +208,7 @@ def substitute_config_variables(path, _use_config): replaced if there is an active environment, and should only be used in environment yaml files. """ - _replacements = replacements(_use_config=_use_config) + _replacements = replacements() # Look up replacements def repl(match): @@ -245,11 +221,11 @@ def repl(match): return re.sub(r"(\$\w+\b|\$\{\w+\})", repl, path) -def substitute_path_variables(path, _use_config=True, _recursive=False): +def substitute_path_variables(path, _recursive=False): """Substitute config vars, expand environment vars, expand user home.""" prev = path while True: - path = substitute_config_variables(path, _use_config=_use_config) + path = substitute_config_variables(path) path = os.path.expandvars(path) path = os.path.expanduser(path) if (not _recursive) or (path == prev): @@ -297,7 +273,7 @@ def add_padding(path, length): return os.path.join(path, padding) -def canonicalize_path(path: str, default_wd: Optional[str] = None, _use_config=True, _recursive=False) -> str: +def canonicalize_path(path: str, default_wd: Optional[str] = None, _recursive=False) -> str: """Same as substitute_path_variables, but also take absolute path. If the string is a yaml object with file annotations, make absolute paths @@ -310,9 +286,6 @@ def canonicalize_path(path: str, default_wd: Optional[str] = None, _use_config=T Returns: An absolute path or non-file URL with path variable substitution """ - # _use_config: whether or not to use config to resolve spack config - # variables. This should be false if this function is called in a context that - # is defining one of these variables. import urllib.parse import urllib.request @@ -323,7 +296,7 @@ def canonicalize_path(path: str, default_wd: Optional[str] = None, _use_config=T filename = os.path.dirname(path._start_mark.name) # type: ignore[attr-defined] assert path._start_mark.name == path._end_mark.name # type: ignore[attr-defined] - path = substitute_path_variables(path, _use_config=_use_config, _recursive=_recursive) + path = substitute_path_variables(path, _recursive=_recursive) # Ensure properly process a Windows path win_path = pathlib.PureWindowsPath(path) From 0ca6771e3f0aeb2bf6b988361bfcd38254233e60 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Mar 2026 14:49:07 -0700 Subject: [PATCH 333/506] Revert "remove logic for disallowing resolution of certain config vars in the context of resolving other config vars" This reverts commit b17e9457348b69ccf7c6b1dd0683fd9c85e64edd. --- lib/spack/spack/paths.py | 2 +- lib/spack/spack/util/path.py | 59 ++++++++++++++++++++++++++---------- 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 808b4067b7bd54..53064f5061e5bf 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -266,7 +266,7 @@ def cfg_check(path, provenance, rel=None): import spack.util.path try: - found = spack.util.path.canonicalize_path(found, _recursive=True) + found = spack.util.path.canonicalize_path(found, _use_config=True, _recursive=True) except spack.util.path.ResolutionContextError as e: msg = f"{path} is defined in terms of another location ({e.var})" raise ValueError(msg) diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 1d13b0c3bfbdcb..5491a46dcfe76f 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -68,11 +68,10 @@ def __call__(self): # Substitutions to perform -def replacements(): +def replacements(_use_config=True): # break circular imports import spack import spack.environment as ev - import spack.paths import spack.paths_base arch = architecture() @@ -92,16 +91,41 @@ def replacements(): "date": lambda: date.today().strftime("%Y-%m-%d"), "env": lambda: ev.active_environment().path if ev.active_environment() else NOMATCH, "spack_short_version": lambda: spack.get_short_version(), - "user_cache_path": lambda: spack.paths.locations.user_cache_path, - "default_install_root": lambda: spack.paths.locations.default_install_location, - "default_envs_root": lambda: spack.paths.locations.default_envs_path, - "modules_base": lambda: spack.paths.locations.modules_base, - "data_home": lambda: spack.paths.locations.data_home, - "cache_home": lambda: spack.paths.locations.cache_home, - "state_home": lambda: spack.paths.locations.state_home, - "spack_home": lambda: spack.paths.locations.spack_home, } + if _use_config: + import spack.paths + + paths = spack.paths.locations + replace.update( + { + "user_cache_path": lambda: paths.user_cache_path, + "default_install_root": lambda: paths.default_install_location, + "default_envs_root": lambda: paths.default_envs_path, + "modules_base": lambda: paths.modules_base, + "data_home": lambda: paths.data_home, + "cache_home": lambda: paths.cache_home, + "state_home": lambda: paths.state_home, + "spack_home": lambda: paths.spack_home, + } + ) + else: + error_resolution = lambda x: CannotResolve(x) + mappings = list( + (x, error_resolution(x)) + for x in [ + "user_cache_path", + "default_install_root", + "default_envs_root", + "modules_base", + "data_home", + "cache_home", + "state_home", + "spack_home", + ] + ) + replace.update(dict(mappings)) + return replace @@ -177,7 +201,7 @@ def get_system_path_max(): return sys_max_path_length -def substitute_config_variables(path): +def substitute_config_variables(path, _use_config): """Substitute placeholders into paths. Spack allows paths in configs to have some placeholders, as follows: @@ -208,7 +232,7 @@ def substitute_config_variables(path): replaced if there is an active environment, and should only be used in environment yaml files. """ - _replacements = replacements() + _replacements = replacements(_use_config=_use_config) # Look up replacements def repl(match): @@ -221,11 +245,11 @@ def repl(match): return re.sub(r"(\$\w+\b|\$\{\w+\})", repl, path) -def substitute_path_variables(path, _recursive=False): +def substitute_path_variables(path, _use_config=True, _recursive=False): """Substitute config vars, expand environment vars, expand user home.""" prev = path while True: - path = substitute_config_variables(path) + path = substitute_config_variables(path, _use_config=_use_config) path = os.path.expandvars(path) path = os.path.expanduser(path) if (not _recursive) or (path == prev): @@ -273,7 +297,7 @@ def add_padding(path, length): return os.path.join(path, padding) -def canonicalize_path(path: str, default_wd: Optional[str] = None, _recursive=False) -> str: +def canonicalize_path(path: str, default_wd: Optional[str] = None, _use_config=True, _recursive=False) -> str: """Same as substitute_path_variables, but also take absolute path. If the string is a yaml object with file annotations, make absolute paths @@ -286,6 +310,9 @@ def canonicalize_path(path: str, default_wd: Optional[str] = None, _recursive=Fa Returns: An absolute path or non-file URL with path variable substitution """ + # _use_config: whether or not to use config to resolve spack config + # variables. This should be false if this function is called in a context that + # is defining one of these variables. import urllib.parse import urllib.request @@ -296,7 +323,7 @@ def canonicalize_path(path: str, default_wd: Optional[str] = None, _recursive=Fa filename = os.path.dirname(path._start_mark.name) # type: ignore[attr-defined] assert path._start_mark.name == path._end_mark.name # type: ignore[attr-defined] - path = substitute_path_variables(path, _recursive=_recursive) + path = substitute_path_variables(path, _use_config=_use_config, _recursive=_recursive) # Ensure properly process a Windows path win_path = pathlib.PureWindowsPath(path) From 35af6e276da45e8fe848898385d37088233f0515 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Mar 2026 15:28:25 -0700 Subject: [PATCH 334/506] include config cannot refer to data_home, but data_home can for example refer to spack_home --- lib/spack/spack/config.py | 14 ++++++++++++-- lib/spack/spack/paths.py | 6 +----- lib/spack/spack/test/config.py | 14 ++++++++++++++ 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 1be78024503532..973234c63e8b7f 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1066,8 +1066,18 @@ def _scope( # But ensure that name is unique if there are multiple paths. if not self.name or len(getattr(self, "paths", [])) > 1: parent_path = pathlib.Path(getattr(parent_scope, "path", "")) - real_path = pathlib.Path(spack.util.path.substitute_path_variables(path)) - + try: + real_path = pathlib.Path(spack.util.path.substitute_path_variables(path, _use_config=False)) + except spack.util.path.ResolutionContextError as e: + if self.name: + context = self.name + elif parent_scope: + context = parent_scope.name + else: + context = "(no context)" + msg = (f"Included scope is defined in terms of config variable ({e.var}): {path}" + + f"\nContext: {context}") + raise ValueError(msg) try: included_name = real_path.relative_to(parent_path) except ValueError: diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 53064f5061e5bf..062235851cd47c 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -265,11 +265,7 @@ def cfg_check(path, provenance, rel=None): if found: import spack.util.path - try: - found = spack.util.path.canonicalize_path(found, _use_config=True, _recursive=True) - except spack.util.path.ResolutionContextError as e: - msg = f"{path} is defined in terms of another location ({e.var})" - raise ValueError(msg) + found = spack.util.path.canonicalize_path(found, _recursive=True) return append_rel(found, rel), provenance spack_cfg_check = partial( diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 023f608886f1ad..89a42133162b11 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1646,6 +1646,20 @@ def test_included_optional_include_scopes(): spack.config.OptionalInclude({}).scopes(spack.config.ConfigScope("fail")) +def test_included_path_forbidden_var( + tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, +): + include = spack.config.included_path("$data_home/test") + parent_scope = mock_low_high_config.scopes["low"] + with pytest.raises(ValueError, match="defined in terms of.*data_home"): + include.scopes(parent_scope) + + # Note: defining a name actually bypasses any form of config + # variable resolution + # entry = {"name": "foo", "path": "$data_home/test"} + # include = spack.config.included_path(entry) + + def test_included_path_string( tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, capfd ): From 8d27520d80f00b86ca071cb77fc447ef3aac6162 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Mar 2026 15:32:26 -0700 Subject: [PATCH 335/506] auto style --- lib/spack/spack/config.py | 10 +++++++--- lib/spack/spack/test/config.py | 4 +--- lib/spack/spack/util/path.py | 4 +++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 973234c63e8b7f..23cbb35405f354 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1067,7 +1067,9 @@ def _scope( if not self.name or len(getattr(self, "paths", [])) > 1: parent_path = pathlib.Path(getattr(parent_scope, "path", "")) try: - real_path = pathlib.Path(spack.util.path.substitute_path_variables(path, _use_config=False)) + real_path = pathlib.Path( + spack.util.path.substitute_path_variables(path, _use_config=False) + ) except spack.util.path.ResolutionContextError as e: if self.name: context = self.name @@ -1075,8 +1077,10 @@ def _scope( context = parent_scope.name else: context = "(no context)" - msg = (f"Included scope is defined in terms of config variable ({e.var}): {path}" - + f"\nContext: {context}") + msg = ( + f"Included scope is defined in terms of config variable ({e.var}): {path}" + + f"\nContext: {context}" + ) raise ValueError(msg) try: included_name = real_path.relative_to(parent_path) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 89a42133162b11..39587fe9fe8cad 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1646,9 +1646,7 @@ def test_included_optional_include_scopes(): spack.config.OptionalInclude({}).scopes(spack.config.ConfigScope("fail")) -def test_included_path_forbidden_var( - tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, -): +def test_included_path_forbidden_var(tmp_path: pathlib.Path, mock_low_high_config, ensure_debug): include = spack.config.included_path("$data_home/test") parent_scope = mock_low_high_config.scopes["low"] with pytest.raises(ValueError, match="defined in terms of.*data_home"): diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 5491a46dcfe76f..0401a0554cdb4d 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -297,7 +297,9 @@ def add_padding(path, length): return os.path.join(path, padding) -def canonicalize_path(path: str, default_wd: Optional[str] = None, _use_config=True, _recursive=False) -> str: +def canonicalize_path( + path: str, default_wd: Optional[str] = None, _use_config=True, _recursive=False +) -> str: """Same as substitute_path_variables, but also take absolute path. If the string is a yaml object with file annotations, make absolute paths From b8fd52543bf779144ec33b09942a565354b21c49 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Mar 2026 16:21:26 -0700 Subject: [PATCH 336/506] env fixture cleanup based on review suggestions --- lib/spack/spack/test/paths.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 4f0997e3ac8f33..51b24ee1f7e95f 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -20,9 +20,8 @@ def _ensure_dir(pathlike): return str(pathlike) -@pytest.fixture(scope="module", autouse=True) -def clear_env_vars(): - saved = os.environ.copy() +@pytest.fixture(autouse=True) +def clear_env_vars(working_env): spack.paths_base._unset_xdg_vars(os.environ) for x in [ "SPACK_USER_CACHE_PATH", @@ -32,8 +31,6 @@ def clear_env_vars(): "SPACK_CACHE_HOME", ]: os.environ.pop(x, None) - yield - os.environ.update(saved) @pytest.fixture @@ -42,7 +39,6 @@ def _set_home(val): # Clear some env vars that can interfere w/ expanduser(~) on Windows os.environ.pop("USERPROFILE", None) os.environ.pop("HOMEDRIVE", None) - os.environ.pop("HOMEPATH", None) os.environ["HOMEPATH"] = val # For expanduser on Linux From 0df85feac51dd999b1064655718e47e3e716884f Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Mar 2026 17:19:47 -0700 Subject: [PATCH 337/506] do not expose default_license_dir as module-level var in spack.paths --- lib/spack/spack/paths.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 062235851cd47c..9152a525dd9cff 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -385,7 +385,6 @@ def freeze(): hooks_path = locations.hooks_path share_path = locations.share_path etc_path = locations.etc_path -default_license_dir = locations.default_license_dir var_path = locations.var_path test_repos_path = locations.test_repos_path mock_packages_path = locations.mock_packages_path From 10b0efef1ec145c25d8ea4a7d7647416b2e78ec0 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Mar 2026 19:12:37 -0700 Subject: [PATCH 338/506] rm default_license_dir because it is only referenced once in the code (and once in the tests) --- lib/spack/spack/config.py | 2 +- lib/spack/spack/paths_base.py | 2 -- lib/spack/spack/test/config.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 23cbb35405f354..d71c8d1f4cdd20 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -118,7 +118,7 @@ "dirty": False, "build_jobs": min(16, cpus_available()), "build_stage": "$tempdir/spack-stage", - "license_dir": spack.paths_base.default_license_dir, + "license_dir": os.path.join(spack.paths_base.etc_path, "licenses"), }, "concretizer": {"externals": {"completion": "default_variants"}}, } diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index ec3c265674066e..d42a550f43c378 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -62,7 +62,6 @@ def __init__(self, _prefix=None): self.hooks_path = os.path.join(self.module_path, "hooks") self.share_path = os.path.join(self.prefix, "share", "spack") self.etc_path = os.path.join(self.prefix, "etc", "spack") - self.default_license_dir = os.path.join(self.etc_path, "licenses") self.var_path = os.path.join(self.prefix, "var", "spack") # $spack/var/spack is generally read-only. Older instances may @@ -125,7 +124,6 @@ def env_based_state_home(self): hooks_path = locations.hooks_path share_path = locations.share_path etc_path = locations.etc_path -default_license_dir = locations.default_license_dir var_path = locations.var_path test_repos_path = locations.test_repos_path mock_packages_path = locations.mock_packages_path diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 39587fe9fe8cad..3d6feb32c553d9 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1158,7 +1158,7 @@ def test_bad_path_double_override(config): def test_license_dir_config(mutable_config, mock_packages, tmp_path): """Ensure license directory is customizable""" - expected_dir = spack_paths.default_license_dir + expected_dir = os.path.join(spack_paths.etc_path, "licenses") assert spack.config.get("config:license_dir") == expected_dir assert spack.package_base.PackageBase.global_license_dir == expected_dir assert spack.repo.PATH.get_pkg_class("pkg-a").global_license_dir == expected_dir From 3df188074cde7a83837f4d26188954c303b55fdc Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Mar 2026 19:43:21 -0700 Subject: [PATCH 339/506] license dir is for writing, not reading, it needs to be in data_home --- etc/spack/defaults/base/config.yaml | 2 +- lib/spack/spack/config.py | 2 +- lib/spack/spack/test/config.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index 5dfdc099f0436a..e93193bcd6f051 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -35,7 +35,7 @@ config: - $spack/share/spack/templates # Directory where licenses should be located - license_dir: $spack/etc/spack/licenses + license_dir: $data_home/licenses # Temporary locations Spack can try to use for builds. # diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index d71c8d1f4cdd20..620c0507da487a 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -118,7 +118,7 @@ "dirty": False, "build_jobs": min(16, cpus_available()), "build_stage": "$tempdir/spack-stage", - "license_dir": os.path.join(spack.paths_base.etc_path, "licenses"), + "license_dir": os.path.join("$data_home", "licenses"), }, "concretizer": {"externals": {"completion": "default_variants"}}, } diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 3d6feb32c553d9..49161af46e0d2b 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1158,7 +1158,7 @@ def test_bad_path_double_override(config): def test_license_dir_config(mutable_config, mock_packages, tmp_path): """Ensure license directory is customizable""" - expected_dir = os.path.join(spack_paths.etc_path, "licenses") + expected_dir = os.path.join("$data_home", "licenses") assert spack.config.get("config:license_dir") == expected_dir assert spack.package_base.PackageBase.global_license_dir == expected_dir assert spack.repo.PATH.get_pkg_class("pkg-a").global_license_dir == expected_dir From 5984c0538fe223d152f267a0f1271ff602d0b8fc Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Mar 2026 19:54:09 -0700 Subject: [PATCH 340/506] move test to use fixtures that help with paths mocking --- lib/spack/spack/test/config.py | 16 ---------------- lib/spack/spack/test/paths.py | 26 ++++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 49161af46e0d2b..1d525f45b79561 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -19,9 +19,7 @@ import spack.environment as ev import spack.error import spack.llnl.util.filesystem as fs -import spack.package_base import spack.platforms -import spack.repo import spack.schema.compilers import spack.schema.config import spack.schema.env @@ -1156,20 +1154,6 @@ def test_bad_path_double_override(config): pass -def test_license_dir_config(mutable_config, mock_packages, tmp_path): - """Ensure license directory is customizable""" - expected_dir = os.path.join("$data_home", "licenses") - assert spack.config.get("config:license_dir") == expected_dir - assert spack.package_base.PackageBase.global_license_dir == expected_dir - assert spack.repo.PATH.get_pkg_class("pkg-a").global_license_dir == expected_dir - - abs_path = str(tmp_path / "foo" / "bar" / "baz") - spack.config.set("config:license_dir", abs_path) - assert spack.config.get("config:license_dir") == abs_path - assert spack.package_base.PackageBase.global_license_dir == abs_path - assert spack.repo.PATH.get_pkg_class("pkg-a").global_license_dir == abs_path - - @pytest.mark.regression("22547") def test_single_file_scope_cache_clearing(env_yaml): scope = spack.config.SingleFileScope( diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 51b24ee1f7e95f..110529e55e278d 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -288,6 +288,32 @@ def test_location_vars_that_use_other_location_vars( assert p2.default_install_location == str(pathlib.Path(basedir) / "home" / "installs") +def test_license_dir_config(mutable_config, mock_packages, tmp_path, monkeypatch, set_home): + """Ensure license directory is customizable""" + import spack.config + import spack.package_base + import spack.repo + + basedir = _ensure_dir(tmp_path / "spack-root") + homedir = _ensure_dir(tmp_path / "base-prefix") + set_home(homedir) + + p1 = SpackPaths(SpackPathsBase(str(basedir))) + monkeypatch.setattr(spack.paths, "locations", p1) + + default_cfg_val = os.path.join("$data_home", "licenses") + resolved_dir = str(pathlib.Path(homedir) / ".local" / "share" / "spack" / "licenses") + assert spack.config.get("config:license_dir") == default_cfg_val + assert spack.package_base.PackageBase.global_license_dir == resolved_dir + assert spack.repo.PATH.get_pkg_class("pkg-a").global_license_dir == resolved_dir + + abs_path = str(tmp_path / "foo" / "bar" / "baz") + spack.config.set("config:license_dir", abs_path) + assert spack.config.get("config:license_dir") == abs_path + assert spack.package_base.PackageBase.global_license_dir == abs_path + assert spack.repo.PATH.get_pkg_class("pkg-a").global_license_dir == abs_path + + class SetAnXdgVarAndReadDataHome: """Access an XDG-dependent variable from spack.paths as quickly as possible. From 19b88225c7a327eae19b48a7d598f8008de306db Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Mar 2026 20:12:37 -0700 Subject: [PATCH 341/506] 'inline' default_test_path --- etc/spack/defaults/base/config.yaml | 2 +- lib/spack/spack/install_test.py | 2 +- lib/spack/spack/paths.py | 5 ----- lib/spack/spack/paths_base.py | 5 ++--- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index e93193bcd6f051..cee2be79e54a23 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -72,7 +72,7 @@ config: # Directory in which to run tests and store test results. # Tests will be stored in directories named by date/time and package # name/hash. - test_stage: $user_cache_path/test + test_stage: $state_home/test # Cache directory for already downloaded source tarballs and archived diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py index a54a68d9d2df7a..ce377e42cd27b5 100644 --- a/lib/spack/spack/install_test.py +++ b/lib/spack/spack/install_test.py @@ -97,7 +97,7 @@ def get_test_stage_dir() -> str: absolute path to the configured test stage root or, if none, the default test stage path """ return spack.util.path.canonicalize_path( - spack.config.get("config:test_stage", paths.default_test_path) + spack.config.get("config:test_stage", "$state_home/test") ) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 9152a525dd9cff..b9c79e1203b238 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -176,11 +176,6 @@ def reports_path(self): #: junit, cdash, etc. reports about builds return os.path.join(self.state_home, "reports") - @property - def default_test_path(self): - #: installation test (spack test) output - return os.path.join(self.state_home, "test") - @property def default_monitor_path(self): #: spack monitor analysis directories diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index d42a550f43c378..542efbd44516cf 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -24,9 +24,8 @@ class XDG_overrides(Enum): # This is for tests that want to clean the environment of XDG_ variables that -# affect spack behavior (and the corresponding SPACK_ overrides). Note that -# these vars will affect .default_test_path for the running instance, but -# the unit tests will not see the env vars +# affect spack behavior. Note that this will not influence install_test.py's +# view of config:test_stage def _unset_xdg_vars(env): for xdg_var in itertools.chain(XDG_vars, XDG_overrides): env.pop(xdg_var.value, None) From 534667f420e37167881c0cccb7e6c9fe02490979 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Mar 2026 20:22:12 -0700 Subject: [PATCH 342/506] likewise 'inline' default_misc_cache_path --- etc/spack/defaults/base/config.yaml | 4 +++- lib/spack/spack/caches.py | 2 +- lib/spack/spack/paths.py | 9 --------- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index cee2be79e54a23..f29664876757aa 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -91,7 +91,9 @@ config: # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` - # TODO: see note in paths.py - default_misc_cache_path + # TODO: restore when .ci/gitlab/configs/ci.yaml in spack-packages is + # updated to tolerate this relocation of misc caches + # misc_cache: $state_home/$spack_instance_id/cache misc_cache: $state_home/cache diff --git a/lib/spack/spack/caches.py b/lib/spack/spack/caches.py index 889fe17ff6d870..c816fa31eb5a43 100644 --- a/lib/spack/spack/caches.py +++ b/lib/spack/spack/caches.py @@ -20,7 +20,7 @@ def misc_cache_location(): Currently the ``MISC_CACHE`` stores indexes for virtual dependency providers and for which packages provide which tags. """ - path = spack.config.get("config:misc_cache", paths.default_misc_cache_path) + path = spack.config.get("config:misc_cache", "$state_home/cache") return spack.util.path.canonicalize_path(path) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index b9c79e1203b238..6c0d5ed55a49ba 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -216,15 +216,6 @@ def gpg_keys_path(self): self._data_home_provenance, ) - @property - def default_misc_cache_path(self): - #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) - #: overridden by `config:misc_cache` - # TODO: restore when .ci/gitlab/configs/ci.yaml in spack-packages is - # updated to tolerate this relocation of misc caches - # return os.path.join(self.state_home, self.spack_instance_id, "cache") - return os.path.join(self.state_home, "cache") - def __getattr__(self, name): # Things that aren't sensitive to import cycles can import the # paths module and access all items from paths_base From a11e6b9f7a20b606426c27586a0fe0f71e632d27 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 9 Mar 2026 20:26:08 -0700 Subject: [PATCH 343/506] rm unused imports on account of inlining --- lib/spack/spack/caches.py | 1 - lib/spack/spack/install_test.py | 1 - 2 files changed, 2 deletions(-) diff --git a/lib/spack/spack/caches.py b/lib/spack/spack/caches.py index c816fa31eb5a43..a7b8333e526817 100644 --- a/lib/spack/spack/caches.py +++ b/lib/spack/spack/caches.py @@ -11,7 +11,6 @@ import spack.util.file_cache import spack.util.path from spack.llnl.util.filesystem import mkdirp -from spack.paths import locations as paths def misc_cache_location(): diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py index ce377e42cd27b5..f27b9b7b6db18f 100644 --- a/lib/spack/spack/install_test.py +++ b/lib/spack/spack/install_test.py @@ -30,7 +30,6 @@ from spack.llnl.string import plural from spack.llnl.util.lang import nullcontext from spack.llnl.util.tty.color import colorize -from spack.paths import locations as paths from spack.spec import Spec from spack.util.prefix import Prefix From ca40af3f5c14de38445eef004a71139001a4b094 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 10 Mar 2026 15:21:55 -0700 Subject: [PATCH 344/506] simplify recursive config variable expansion Signed-off-by: Gregory Becker --- lib/spack/spack/config.py | 41 +++++++++++++------- lib/spack/spack/paths.py | 2 +- lib/spack/spack/util/path.py | 75 ++++++++++++------------------------ 3 files changed, 53 insertions(+), 65 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 620c0507da487a..ba9b700f5de6ea 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -136,6 +136,22 @@ #: safeguard for recursive includes -- maximum include depth MAX_RECURSIVE_INCLUDES = 100 +#: configurable config vars -- these cannot be used by include paths +#: nor by other paths that can affect config values +CONFIGURABLE_VARS = ( + "config_home", + "state_home", + "cache_home", + "data_home", + "spack_home", + "modules_base", + "default_envs_root", + "default_install_root", + "user_cache_path", +) +_CVARS_RE = "|".join(CONFIGURABLE_VARS) +CONFIGURABLE_VARS_REGEX = r"(\$(" + _CVARS_RE + r")\b)|(\$\{(" + _CVARS_RE + r")\})" + def _include_cache_location(): """Location to cache included configuration files.""" @@ -1065,23 +1081,20 @@ def _scope( # But ensure that name is unique if there are multiple paths. if not self.name or len(getattr(self, "paths", [])) > 1: - parent_path = pathlib.Path(getattr(parent_scope, "path", "")) - try: - real_path = pathlib.Path( - spack.util.path.substitute_path_variables(path, _use_config=False) - ) - except spack.util.path.ResolutionContextError as e: - if self.name: - context = self.name - elif parent_scope: - context = parent_scope.name - else: - context = "(no context)" + banned_var = re.match(CONFIGURABLE_VARS_REGEX, path) + if banned_var: + context = self.name or (parent_scope.name if parent_scope else "(no context)") msg = ( - f"Included scope is defined in terms of config variable ({e.var}): {path}" - + f"\nContext: {context}" + "Included scope is defined in terms of prohibited config variable." + f" ({banned_var.group(0)}): {path}" + f"\n Context: {context}" + "\n\n Include config paths may not refer to configurable config variables." ) raise ValueError(msg) + + real_path = pathlib.Path(spack.util.path.substitute_path_variables(path)) + parent_path = pathlib.Path(getattr(parent_scope, "path", "")) + try: included_name = real_path.relative_to(parent_path) except ValueError: diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 6c0d5ed55a49ba..34e84f34e7ed04 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -251,7 +251,7 @@ def cfg_check(path, provenance, rel=None): if found: import spack.util.path - found = spack.util.path.canonicalize_path(found, _recursive=True) + found = spack.util.path.canonicalize_path(found) return append_rel(found, rel), provenance spack_cfg_check = partial( diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 0401a0554cdb4d..d73ac134fa6930 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -68,12 +68,14 @@ def __call__(self): # Substitutions to perform -def replacements(_use_config=True): +def replacements(): # break circular imports import spack import spack.environment as ev import spack.paths_base + import spack.paths + paths = spack.paths.locations arch = architecture() replace = { @@ -91,41 +93,16 @@ def replacements(_use_config=True): "date": lambda: date.today().strftime("%Y-%m-%d"), "env": lambda: ev.active_environment().path if ev.active_environment() else NOMATCH, "spack_short_version": lambda: spack.get_short_version(), + "user_cache_path": lambda: paths.user_cache_path, + "default_install_root": lambda: paths.default_install_location, + "default_envs_root": lambda: paths.default_envs_path, + "modules_base": lambda: paths.modules_base, + "data_home": lambda: paths.data_home, + "cache_home": lambda: paths.cache_home, + "state_home": lambda: paths.state_home, + "spack_home": lambda: paths.spack_home, } - if _use_config: - import spack.paths - - paths = spack.paths.locations - replace.update( - { - "user_cache_path": lambda: paths.user_cache_path, - "default_install_root": lambda: paths.default_install_location, - "default_envs_root": lambda: paths.default_envs_path, - "modules_base": lambda: paths.modules_base, - "data_home": lambda: paths.data_home, - "cache_home": lambda: paths.cache_home, - "state_home": lambda: paths.state_home, - "spack_home": lambda: paths.spack_home, - } - ) - else: - error_resolution = lambda x: CannotResolve(x) - mappings = list( - (x, error_resolution(x)) - for x in [ - "user_cache_path", - "default_install_root", - "default_envs_root", - "modules_base", - "data_home", - "cache_home", - "state_home", - "spack_home", - ] - ) - replace.update(dict(mappings)) - return replace @@ -201,7 +178,7 @@ def get_system_path_max(): return sys_max_path_length -def substitute_config_variables(path, _use_config): +def substitute_config_variables(path): """Substitute placeholders into paths. Spack allows paths in configs to have some placeholders, as follows: @@ -232,7 +209,7 @@ def substitute_config_variables(path, _use_config): replaced if there is an active environment, and should only be used in environment yaml files. """ - _replacements = replacements(_use_config=_use_config) + _replacements = replacements() # Look up replacements def repl(match): @@ -242,19 +219,19 @@ def repl(match): return m if repl is NOMATCH else str(repl) # Replace $var or ${var}. - return re.sub(r"(\$\w+\b|\$\{\w+\})", repl, path) + # Iterate in case values refer to other config variables + new_path = re.sub(r"(\$\w+\b|\$\{\w+\})", repl, path) + while path != new_path: + path = new_path + new_path = re.sub(r"(\$\w+\b|\$\{\w+\})", repl, path) + return new_path -def substitute_path_variables(path, _use_config=True, _recursive=False): +def substitute_path_variables(path): """Substitute config vars, expand environment vars, expand user home.""" - prev = path - while True: - path = substitute_config_variables(path, _use_config=_use_config) - path = os.path.expandvars(path) - path = os.path.expanduser(path) - if (not _recursive) or (path == prev): - break - prev = path + path = substitute_config_variables(path) + path = os.path.expandvars(path) + path = os.path.expanduser(path) return path @@ -297,9 +274,7 @@ def add_padding(path, length): return os.path.join(path, padding) -def canonicalize_path( - path: str, default_wd: Optional[str] = None, _use_config=True, _recursive=False -) -> str: +def canonicalize_path(path: str, default_wd: Optional[str] = None) -> str: """Same as substitute_path_variables, but also take absolute path. If the string is a yaml object with file annotations, make absolute paths @@ -325,7 +300,7 @@ def canonicalize_path( filename = os.path.dirname(path._start_mark.name) # type: ignore[attr-defined] assert path._start_mark.name == path._end_mark.name # type: ignore[attr-defined] - path = substitute_path_variables(path, _use_config=_use_config, _recursive=_recursive) + path = substitute_path_variables(path) # Ensure properly process a Windows path win_path = pathlib.PureWindowsPath(path) From 891df5ba5d1285f9db7be5af3017ffda2a4d5670 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 10 Mar 2026 22:29:48 -0700 Subject: [PATCH 345/506] style edit --- lib/spack/spack/util/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index d73ac134fa6930..cd7ad6f9cddf2a 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -72,8 +72,8 @@ def replacements(): # break circular imports import spack import spack.environment as ev - import spack.paths_base import spack.paths + import spack.paths_base paths = spack.paths.locations arch = architecture() From fb28c1b5a03cef002d6805c56e8795640e6f4212 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 10 Mar 2026 22:35:31 -0700 Subject: [PATCH 346/506] move misc cache back into subdir based on spack instance id --- etc/spack/defaults/base/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index f29664876757aa..018f020b3f3dbd 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -94,7 +94,7 @@ config: # TODO: restore when .ci/gitlab/configs/ci.yaml in spack-packages is # updated to tolerate this relocation of misc caches # misc_cache: $state_home/$spack_instance_id/cache - misc_cache: $state_home/cache + misc_cache: $state_home/$spack_instance_id/cache # Abort downloads after this many seconds if not data is received. From 712a58b67094237881c074db0749a75fb6144041 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 11 Mar 2026 11:09:16 -0700 Subject: [PATCH 347/506] apply analogous changes from subprocess_context to replicated logic in new_installer --- lib/spack/spack/new_installer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/new_installer.py b/lib/spack/spack/new_installer.py index d6357cf22d0ebb..ff9b36268ccf10 100644 --- a/lib/spack/spack/new_installer.py +++ b/lib/spack/spack/new_installer.py @@ -280,7 +280,7 @@ def __init__(self): self.config = spack.config.CONFIG.ensure_unwrapped() self.store = spack.store.STORE self.monkey_patches = spack.subprocess_context.TestPatches.create() - self.spack_working_dir = spack.paths.spack_working_dir + self.spack_working_dir = spack.paths_base.spack_working_dir def restore(self): if multiprocessing.get_start_method() == "fork": @@ -288,7 +288,7 @@ def restore(self): spack.store.STORE = self.store spack.config.CONFIG = self.config self.monkey_patches.restore() - spack.paths.spack_working_dir = self.spack_working_dir + spack.paths_base.spack_working_dir = self.spack_working_dir class PrefixPivoter: From 44213f89ec51fa92384be97c7c40f080153d705a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 12 Mar 2026 10:46:29 -0700 Subject: [PATCH 348/506] remove resolved TODO comment --- etc/spack/defaults/base/config.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index 018f020b3f3dbd..41923cd3e60054 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -91,9 +91,6 @@ config: # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` - # TODO: restore when .ci/gitlab/configs/ci.yaml in spack-packages is - # updated to tolerate this relocation of misc caches - # misc_cache: $state_home/$spack_instance_id/cache misc_cache: $state_home/$spack_instance_id/cache From 00ca5e24a5fca9de402ca01e354328714c3968fd Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 12 Mar 2026 11:26:21 -0700 Subject: [PATCH 349/506] explicitly describe spack_home; bring up that config files are an exception --- lib/spack/docs/configuration.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 715f3d6c174259..27dd372694c83d 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -533,18 +533,19 @@ These special variables are substituted first, so any environment variables with Spack-specific variables controlling data location ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Files generated and stored by spack are organized roughly into three categories: +Files generated and used by spack are organized roughly into four categories: * Persistent, large quantities of data (e.g. installs and environments) * Temporary (or assumed temporary) large quantities of data (e.g. stages for installs) * Persistent caches/indices used by spack to speed up its commands (small quantities of data) +* The configuration files themselves The corresponding variables that describe where this data is placed are: * ``$data_home`` * ``$cache_home`` -* ``$state_home`` -* ``$user_cache_path``: legacy variable, equivalent to ``$state_home`` +* ``$state_home`` (also known as ``$user_cache_path``) +* Config file locations are an exception: they can only be controlled with :ref:`environment variables ` or with :ref:`include.yaml ` You can refer to these variables when configuring locations for stages, misc cache, etc. Furthermore each is controlled by a fall-through scheme. @@ -557,8 +558,10 @@ For example ``$data_home`` evaluates to one of the following (highest-priority f #. ``XDG_DATA_HOME/spack`` if XDG_DATA_HOME is set #. Under the default for ``XDG_DATA_HOME``: ``~/.local/share/spack`` +``config:locations:home`` / ``SPACK_HOME`` can be used to control all 3 of ``data_home``, ``cache_home``, and ``state_home``. + Of particular interest is where the environments and installs are placed by Spack, because these can take up a lot of space. -Generally speaking these are controlled by ``$data_home``. +These are controlled by ``$data_home``. Older installs of spack placed these within ``$spack``, and fallback scheme in these cases is augmented to prefer these old locations if no data is detected in the corresponding new locations: * ``$default_install_root``: the location where installs go by default. @@ -704,10 +707,6 @@ Spack provides three environment variables that allow you to override or opt out * ``SPACK_DISABLE_LOCAL_CONFIG``: Set this environment variable to completely disable **all** configurations from the system and user directories. Spack will then only consider its own defaults and ``site`` configuration locations. -And one that allows you to move the default cache location: - -* ``SPACK_USER_CACHE_PATH``: Override the default path to use for user data (misc_cache, tests, reports, etc.). - With these settings, if you want to isolate Spack in a CI environment, you can do this: .. code-block:: console From 0a3ae3f0870ec08ab49f206ad6270fcecff73cf8 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 12 Mar 2026 11:51:45 -0700 Subject: [PATCH 350/506] explain that build_stage isnt usually controlled by cache_home --- lib/spack/docs/configuration.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 27dd372694c83d..2609bb0178a8f3 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -548,7 +548,12 @@ The corresponding variables that describe where this data is placed are: * Config file locations are an exception: they can only be controlled with :ref:`environment variables ` or with :ref:`include.yaml ` You can refer to these variables when configuring locations for stages, misc cache, etc. -Furthermore each is controlled by a fall-through scheme. + +Each of these variables are the *default* (fallback) for data in their category: more-specific data in that category may have config that overrides these defaults. +For example while build stages would reasonably be placed in ``$cache_home``, Spack's default configuration sets ``config:build_stage`` to the user's tempdir. +Any configuration controlling location that is more-specific than the above variables will always take precedence (e.g. ``config:install_tree:root``). + +Each of these variables can be set with config or with environment variables. For example ``$data_home`` evaluates to one of the following (highest-priority first): #. ``SPACK_DATA_HOME`` env var if that is set From dedb7f917d0b76eacad0755a7687c3ec1b5b15ae Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 12 Mar 2026 13:28:15 -0700 Subject: [PATCH 351/506] add section about avoiding home dir --- lib/spack/docs/package_fundamentals.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lib/spack/docs/package_fundamentals.rst b/lib/spack/docs/package_fundamentals.rst index 8817c316354710..09c0bf3e9ee852 100644 --- a/lib/spack/docs/package_fundamentals.rst +++ b/lib/spack/docs/package_fundamentals.rst @@ -116,6 +116,27 @@ And if you *only* wanted to see packages that provide MPI-2, you would add a ver Notice that the package versions that provide insufficient MPI versions are now filtered out. +Controlling where spack writes data +----------------------------------- + +A fresh checkout of spack will not write anything into the `$spack`` prefix; instead, all data is placed under the user's home directory. +You can control this in the following ways: + +* Redirect everything with environment variables: set ``SPACK_HOME`` and one of ``SPACK_USER_CONFIG_PATH`` or ``SPACK_DISABLE_LOCAL_CONFIG=1`` +* Or redirect everything with config: + * set ``config:locations:home`` + * Update the ``user`` config scope with ``spack config --scope=spack edit include`` +* Or redirect installs, environments, and cached downloads (everything that takes up significant space) by setting ``SPACK_DATA_HOME`` +* Or use finer-grained configuration settings, for example: + * ``config:install_tree:root`` to control where installs go + * ``config:build_stage`` to control where builds are staged + +For more on this, see: + +* :ref:`Variables controlling data location ` +* :ref:`include.yaml ` +* :ref:`config.yaml ` + Installing and Uninstalling --------------------------- From b5ac32db6efe66c6217e3793595b9d8bb855e562 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 12 Mar 2026 14:25:36 -0700 Subject: [PATCH 352/506] list formatting --- lib/spack/docs/package_fundamentals.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/spack/docs/package_fundamentals.rst b/lib/spack/docs/package_fundamentals.rst index 09c0bf3e9ee852..fe2ed2a34cce60 100644 --- a/lib/spack/docs/package_fundamentals.rst +++ b/lib/spack/docs/package_fundamentals.rst @@ -124,10 +124,12 @@ You can control this in the following ways: * Redirect everything with environment variables: set ``SPACK_HOME`` and one of ``SPACK_USER_CONFIG_PATH`` or ``SPACK_DISABLE_LOCAL_CONFIG=1`` * Or redirect everything with config: + * set ``config:locations:home`` * Update the ``user`` config scope with ``spack config --scope=spack edit include`` * Or redirect installs, environments, and cached downloads (everything that takes up significant space) by setting ``SPACK_DATA_HOME`` * Or use finer-grained configuration settings, for example: + * ``config:install_tree:root`` to control where installs go * ``config:build_stage`` to control where builds are staged From c5a5085a4bf940f4e0e9ca20e5100cd355b47010 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 24 Mar 2026 11:08:30 -0700 Subject: [PATCH 353/506] missed a new reference --- lib/spack/spack/test/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index c0a4d95d81a9c3..73c291f51bc38b 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1694,7 +1694,7 @@ def test_included_path_substitution(): # check a straight path substitution entry = {"path": "$user_cache_path/path/to/config.yaml"} include = spack.config.included_path(entry) - assert spack.paths.user_cache_path in include.path + assert spack_paths.user_cache_path in include.path # check path through an environment variable path = "/path/to/project/packages.yaml" From be939fa5249660cca0c0369a3de033e4d2add3bc Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 24 Mar 2026 14:03:19 -0700 Subject: [PATCH 354/506] include path processing changed --- lib/spack/spack/config.py | 54 ++++++++++++++++++---------------- lib/spack/spack/test/config.py | 9 ++---- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index a1e81a8bc1096d..60389aec60bbb1 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1036,6 +1036,22 @@ def override( assert scope is overrides +def substitute_include_path(path, context): + # circular dependencies + import spack.util.path + + banned_var = re.match(CONFIGURABLE_VARS_REGEX, path) + if banned_var: + msg = ( + "Included scope is defined in terms of prohibited config variable." + f" ({banned_var.group(0)}): {path}" + f"\n Context: {context}" + "\n\n Include config paths may not refer to configurable config variables." + ) + raise ValueError(msg) + return spack.util.path.substitute_path_variables(path) + + #: Class for the relevance of an optional path conditioned on a limited #: python code that evaluates to a boolean and or explicit specification #: as optional. @@ -1070,9 +1086,6 @@ def _scope( Raises: ValueError: the required configuration path does not exist """ - # circular dependencies - import spack.util.path - # Ignore included concrete environment files (i.e., ``spack.lock``) # since they are not normal configuration (scope) files and their # processing is handled when the environment is processed. @@ -1091,18 +1104,8 @@ def _scope( # But ensure that name is unique if there are multiple paths. if not self.name or len(getattr(self, "paths", [])) > 1: - banned_var = re.match(CONFIGURABLE_VARS_REGEX, path) - if banned_var: - context = self.name or (parent_scope.name if parent_scope else "(no context)") - msg = ( - "Included scope is defined in terms of prohibited config variable." - f" ({banned_var.group(0)}): {path}" - f"\n Context: {context}" - "\n\n Include config paths may not refer to configurable config variables." - ) - raise ValueError(msg) - - real_path = pathlib.Path(spack.util.path.substitute_path_variables(path)) + context = self.name or (parent_scope.name if parent_scope else "(no context)") + real_path = pathlib.Path(substitute_include_path(path, context)) parent_path = pathlib.Path(getattr(parent_scope, "path", "")) try: @@ -1192,16 +1195,16 @@ class IncludePath(OptionalInclude): destination: Optional[str] def __init__(self, entry: dict): - # circular dependencies - import spack.util.path - super().__init__(entry) path_override_env_var = entry.get("path_override_env_var", "") if path_override_env_var and path_override_env_var in os.environ: path = os.environ[path_override_env_var] else: path = entry.get("path", "") - self.path = spack.util.path.substitute_path_variables(path) + + context_prefix = f"({self.name}) " if self.name else "" + context = f"{context_prefix}{path}" + self.path = substitute_include_path(path, context) self.sha256 = entry.get("sha256", "") self.destination = None @@ -1271,17 +1274,16 @@ class GitIncludePaths(OptionalInclude): destination: Optional[str] def __init__(self, entry: dict): - # circular dependencies - import spack.util.path - super().__init__(entry) - self.git = spack.util.path.substitute_path_variables(entry.get("git", "")) + context = self.name or entry.get("git", "") or "(Git include)" + self.git = substitute_include_path(entry.get("git", ""), context) self.branch = entry.get("branch", "") self.commit = entry.get("commit", "") self.tag = entry.get("tag", "") - self._paths = [ - spack.util.path.substitute_path_variables(path) for path in entry.get("paths", []) - ] + self._paths = [] + for path in entry.get("paths", []): + sub_context = f"{context} - {path}" + self._paths.append(substitute_include_path(path, sub_context)) self.destination = None if not self.branch and not self.commit and not self.tag: diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 73c291f51bc38b..8f26f86fa27214 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1631,9 +1631,9 @@ def test_included_optional_include_scopes(): def test_included_path_forbidden_var(tmp_path: pathlib.Path, mock_low_high_config, ensure_debug): - include = spack.config.included_path("$data_home/test") - parent_scope = mock_low_high_config.scopes["low"] with pytest.raises(ValueError, match="defined in terms of.*data_home"): + include = spack.config.included_path("$data_home/test") + parent_scope = mock_low_high_config.scopes["low"] include.scopes(parent_scope) # Note: defining a name actually bypasses any form of config @@ -1691,11 +1691,6 @@ def test_included_path_string_no_parent_path( def test_included_path_substitution(): - # check a straight path substitution - entry = {"path": "$user_cache_path/path/to/config.yaml"} - include = spack.config.included_path(entry) - assert spack_paths.user_cache_path in include.path - # check path through an environment variable path = "/path/to/project/packages.yaml" os.environ["SPACK_TEST_PATH_SUB"] = path From 124071e2446be09d67006e5e09e074cf36d1910b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 24 Mar 2026 14:09:07 -0700 Subject: [PATCH 355/506] logic paths seem less fragile for IncludePath config var resolution --- lib/spack/spack/test/config.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 8f26f86fa27214..11d637a07d31c3 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1636,11 +1636,6 @@ def test_included_path_forbidden_var(tmp_path: pathlib.Path, mock_low_high_confi parent_scope = mock_low_high_config.scopes["low"] include.scopes(parent_scope) - # Note: defining a name actually bypasses any form of config - # variable resolution - # entry = {"name": "foo", "path": "$data_home/test"} - # include = spack.config.included_path(entry) - def test_included_path_string( tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, capfd From eb064102e82379ce7254cb4d65713e9384545fa5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 26 Mar 2026 18:11:10 -0700 Subject: [PATCH 356/506] rm vestigial comments/code --- lib/spack/spack/test/conftest.py | 4 ---- lib/spack/spack/util/path.py | 3 --- 2 files changed, 7 deletions(-) diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 21a1365a229ad5..31d36147530a8d 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -2503,10 +2503,6 @@ def _write(config, data, scope): return _write -def _include_cache_root(): - return join_path(str(tempfile.mkdtemp()), "user_cache", "includes") - - @pytest.fixture() def wrapper_dir(install_mockery): """Installs the compiler wrapper and returns the prefix where the script is installed.""" diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 9788929d28b36e..cea110b4885034 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -290,9 +290,6 @@ def canonicalize_path(path: str, default_wd: Optional[str] = None) -> str: Returns: An absolute path or non-file URL with path variable substitution """ - # _use_config: whether or not to use config to resolve spack config - # variables. This should be false if this function is called in a context that - # is defining one of these variables. import urllib.parse import urllib.request From 12f7113fefd27e96dd875b4e22cce44e26f663a8 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 26 Mar 2026 18:24:44 -0700 Subject: [PATCH 357/506] correct tests with merge --- lib/spack/spack/paths.py | 10 ++++++++++ lib/spack/spack/test/config.py | 8 +++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 34e84f34e7ed04..8894152af68c33 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -115,6 +115,16 @@ def spack_home(self): def user_cache_path(self): return self.state_home + from contextlib import contextmanager + + @contextmanager + def redirect_state_home(self, x): + old = self._state_home + self._state_home = x + yield + if old: + self._state_home = old + @property def default_install_location(self): return self._decide_old_or_new_location( diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 7a584d05c77b49..2488df8db8434d 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1924,10 +1924,12 @@ def test_included_path_git_temp_dest(mock_low_high_config): assert dest_dir == temp_dir, pre + rest -def test_included_path_git_errs(tmp_path: pathlib.Path, mock_low_high_config, monkeypatch): - # TODO (47615): I think this will fail - monkeypatch.setattr(spack.paths, "user_cache_path", str(tmp_path)) +@pytest.fixture +def redirect_user_cache(tmp_path: pathlib.Path): + spack_paths.redirect_state_home(tmp_path) + +def test_included_path_git_errs(mock_low_high_config, monkeypatch, redirect_user_cache): paths = ["concretizer.yaml"] entry = { "git": "https://example.com/linux/configs.git", From 0cc62436036c4822c715eed21bba4743171481ab Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 27 Mar 2026 10:27:59 -0700 Subject: [PATCH 358/506] rm module-level vars from spack.paths --- lib/spack/spack/package.py | 4 ++- lib/spack/spack/paths.py | 29 ------------------- .../packages/url_list_test/package.py | 8 ++--- 3 files changed, 7 insertions(+), 34 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 80a0c240e0d327..b1f0486e888153 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -125,7 +125,7 @@ zsh_completion_path, ) from spack.package_test import compare_output, compare_output_file, compile_c_and_execute -from spack.paths import spack_script +import spack.paths from spack.phase_callbacks import run_after, run_before from spack.platforms import host as host_platform from spack.spec import Spec @@ -187,6 +187,8 @@ #: Alias for :func:`os.symlink` (with certain Windows-specific changes) symlink = symlink +spack_script = spack.paths.locations.spack_script + # Not an import alias because black and isort disagree about style create_builder = spack.builder.create diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 8894152af68c33..f25e317631ff87 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -359,32 +359,3 @@ def freeze(): locations.state_home locations.data_home locations.cache_home - - -# At least one builtin spack package expects that spack.paths is -# importable and that it has spack_script as a module-level attribute. -# Some test packages expect other paths (like test_path) -prefix = locations.prefix -spack_root = locations.spack_root -bin_path = locations.bin_path -spack_script = locations.spack_script -sbang_script = locations.sbang_script -lib_path = locations.lib_path -external_path = locations.external_path -module_path = locations.module_path -vendor_path = locations.vendor_path -command_path = locations.command_path -platform_path = locations.platform_path -compilers_path = locations.compilers_path -operating_system_path = locations.operating_system_path -test_path = locations.test_path -hooks_path = locations.hooks_path -share_path = locations.share_path -etc_path = locations.etc_path -var_path = locations.var_path -test_repos_path = locations.test_repos_path -mock_packages_path = locations.mock_packages_path -mock_gpg_data_path = locations.mock_gpg_data_path -mock_gpg_keys_path = locations.mock_gpg_keys_path -system_config_path = locations.system_config_path -user_config_path = locations.user_config_path diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/url_list_test/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/url_list_test/package.py index 5fa78405eafdbb..ce8e39a2e71d19 100644 --- a/var/spack/test_repos/spack_repo/builtin_mock/packages/url_list_test/package.py +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/url_list_test/package.py @@ -4,7 +4,7 @@ from spack_repo.builtin_mock.build_systems.generic import Package -import spack.paths +from spack.paths import locations as spack_paths from spack.package import * from spack.util.url import path_to_file_url @@ -14,9 +14,9 @@ class UrlListTest(Package): homepage = "http://www.url-list-example.com" - web_data_path = join_path(spack.paths.test_path, "data", "web") - url = path_to_file_url(join_path(spack.paths.test_path, "data", "web") + "/foo-0.0.0.tar.gz") - list_url = path_to_file_url(join_path(spack.paths.test_path, "data", "web") + "/index.html") + web_data_path = join_path(spack_paths.test_path, "data", "web") + url = path_to_file_url(join_path(spack_paths.test_path, "data", "web") + "/foo-0.0.0.tar.gz") + list_url = path_to_file_url(join_path(spack_paths.test_path, "data", "web") + "/index.html") list_depth = 3 version("0.0.0", md5="00000000000000000000000000000000") From 3a47a5ebcdccce0eb95875709d55f9df876df181 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 27 Mar 2026 10:38:57 -0700 Subject: [PATCH 359/506] fix a few reference errors --- lib/spack/spack/package.py | 2 +- lib/spack/spack/paths.py | 3 +- lib/spack/spack/test/conftest.py | 29 +++++++++---------- .../packages/url_list_test/package.py | 2 +- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b1f0486e888153..f3d2217318164e 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -13,6 +13,7 @@ import spack.builder import spack.llnl.util.tty as _tty +import spack.paths from spack.archspec import microarchitecture_flags, microarchitecture_flags_from_target from spack.build_environment import ( MakeExecutable, @@ -125,7 +126,6 @@ zsh_completion_path, ) from spack.package_test import compare_output, compare_output_file, compile_c_and_execute -import spack.paths from spack.phase_callbacks import run_after, run_before from spack.platforms import host as host_platform from spack.spec import Spec diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index f25e317631ff87..9ad36eb8354488 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -10,6 +10,7 @@ """ import os import pathlib +from contextlib import contextmanager from enum import Enum from functools import partial @@ -115,8 +116,6 @@ def spack_home(self): def user_cache_path(self): return self.state_home - from contextlib import contextmanager - @contextmanager def redirect_state_home(self, x): old = self._state_home diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 31d36147530a8d..a13e3b1b3aa668 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -80,7 +80,6 @@ working_dir, ) from spack.main import SpackCommand -from spack.paths import locations as paths from spack.util.pattern import Bunch from spack.util.remote_file_cache import raw_github_gitlab_url @@ -136,7 +135,7 @@ def git(): # @pytest.fixture(scope="session") def last_two_git_commits(git): - spack_git_path = paths.prefix + spack_git_path = spack.paths_base.prefix with working_dir(spack_git_path): git_log_out = git("log", "-n", "2", output=str, error=os.devnull) @@ -334,25 +333,25 @@ def latest_commit(): os.makedirs(os.path.dirname(filename)) # add diff-test as a new package to the repository - shutil.copy2(f"{paths.test_path}/data/conftest/diff-test/package-0.txt", filename) + shutil.copy2(f"{spack.paths_base.test_path}/data/conftest/diff-test/package-0.txt", filename) git("add", filename) commit("diff-test: new package") commits.append(latest_commit()) # add v2.1.5 to diff-test - shutil.copy2(f"{paths.test_path}/data/conftest/diff-test/package-1.txt", filename) + shutil.copy2(f"{spack.paths_base.test_path}/data/conftest/diff-test/package-1.txt", filename) git("add", filename) commit("diff-test: add v2.1.5") commits.append(latest_commit()) # add v2.1.6 to diff-test - shutil.copy2(f"{paths.test_path}/data/conftest/diff-test/package-2.txt", filename) + shutil.copy2(f"{spack.paths_base.test_path}/data/conftest/diff-test/package-2.txt", filename) git("add", filename) commit("diff-test: add v2.1.6") commits.append(latest_commit()) # add v2.1.7 and v2.1.8 to diff-test - shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-3.txt", filename) + shutil.copy2(f"{spack.paths_base.test_path}/data/conftest/diff-test/package-3.txt", filename) git("add", filename) commit("diff-test: add v2.1.7 and v2.1.8") commits.append(latest_commit()) @@ -710,7 +709,7 @@ def _use_test_platform(test_platform): # @pytest.fixture(scope="session") def mock_packages_repo(): - yield spack.repo.from_path(paths.mock_packages_path) + yield spack.repo.from_path(spack.paths_base.mock_packages_path) def _pkg_install_fn(pkg, spec, prefix): @@ -757,7 +756,7 @@ def mock_packages(mock_packages_repo, mock_pkg_install, request): def mutable_mock_repo(mock_packages_repo, request): """Function-scoped mock packages, for tests that need to modify them.""" ensure_configuration_fixture_run_before(request) - mock_repo = spack.repo.from_path(paths.mock_packages_path) + mock_repo = spack.repo.from_path(spack.paths_base.mock_packages_path) with spack.repo.use_repositories(mock_repo) as mock_packages_repo: yield mock_packages_repo @@ -772,7 +771,7 @@ def __init__(self, root_directory: str) -> None: namespace = f"test_namespace_{RepoBuilder._counter}" repo_root = os.path.join(root_directory, namespace) os.makedirs(repo_root, exist_ok=True) - self.template_dirs = (os.path.join(paths.share_path, "templates"),) + self.template_dirs = (os.path.join(spack.paths_base.share_path, "templates"),) self.root, self.namespace = spack.repo.create_repo(repo_root, namespace) self.build_system_name = f"test_build_system_{self.namespace}" self._add_build_system() @@ -871,7 +870,7 @@ def default_config(): This ensures we can test the real default configuration without having tests fail when the user overrides the defaults that we test against.""" - defaults_path = os.path.join(paths.etc_path, "defaults") + defaults_path = os.path.join(spack.paths_base.etc_path, "defaults") if sys.platform == "win32": defaults_path = os.path.join(defaults_path, "windows") with spack.config.use_configuration(defaults_path) as defaults_config: @@ -884,7 +883,7 @@ def mock_uarch_json(tmp_path_factory: pytest.TempPathFactory): tmpdir = tmp_path_factory.mktemp("microarchitectures") uarch_json_source = ( - Path(paths.test_path) / "data" / "microarchitectures" / "microarchitectures.json" + Path(spack.paths_base.test_path) / "data" / "microarchitectures" / "microarchitectures.json" ) uarch_json_dest = tmpdir / "microarchitectures.json" shutil.copy2(uarch_json_source, uarch_json_dest) @@ -928,7 +927,7 @@ def configuration_dir(tmp_path_factory: pytest.TempPathFactory, linux_os): # /data/config has mock config yaml files in it # copy these to the site config. - test_config = Path(paths.test_path) / "data" / "config" + test_config = Path(spack.paths_base.test_path) / "data" / "config" shutil.copytree(test_config, tmp_path / "site") # Create temporary 'defaults', 'site' and 'user' folders @@ -1413,7 +1412,7 @@ def module_configuration(monkeypatch, request, mutable_config): # Key for specific settings relative to this module type writer_key = str(writer_mod.__name__).split(".")[-1] # Root folder for configuration - root_for_conf = os.path.join(paths.test_path, "data", "modules", writer_key) + root_for_conf = os.path.join(spack.paths_base.test_path, "data", "modules", writer_key) # ConfigUpdate, when called, will modify configuration, so we need to use # the mutable_config fixture @@ -2026,7 +2025,7 @@ def mock_clone_repo(tmp_path_factory: pytest.TempPathFactory): ) shutil.copytree( - os.path.join(paths.mock_packages_path, spack.repo.packages_dir_name), + os.path.join(spack.paths_base.mock_packages_path, spack.repo.packages_dir_name), os.path.join(str(repodir), spack.repo.packages_dir_name), ) @@ -2186,7 +2185,7 @@ def noncyclical_dir_structure(tmp_path: Path): @pytest.fixture(scope="function") def mock_config_data(): - config_data_dir = os.path.join(paths.test_path, "data", "config") + config_data_dir = os.path.join(spack.paths_base.test_path, "data", "config") return config_data_dir, os.listdir(config_data_dir) diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/url_list_test/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/url_list_test/package.py index ce8e39a2e71d19..9196e4f13e0d9f 100644 --- a/var/spack/test_repos/spack_repo/builtin_mock/packages/url_list_test/package.py +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/url_list_test/package.py @@ -4,8 +4,8 @@ from spack_repo.builtin_mock.build_systems.generic import Package -from spack.paths import locations as spack_paths from spack.package import * +from spack.paths import locations as spack_paths from spack.util.url import path_to_file_url From 0327525a1cf1aba10d4ee42d847c990cf6a729c5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 27 Mar 2026 11:28:14 -0700 Subject: [PATCH 360/506] rm paths_base module-level variables (other than .locations) --- lib/spack/spack/__init__.py | 4 +-- lib/spack/spack/cmd/__init__.py | 8 +++--- lib/spack/spack/cmd/common/__init__.py | 10 ++++---- lib/spack/spack/config.py | 6 ++--- lib/spack/spack/main.py | 4 +-- lib/spack/spack/paths_base.py | 26 ------------------- lib/spack/spack/test/conftest.py | 35 ++++++++++++-------------- lib/spack/spack/trace.py | 2 +- 8 files changed, 33 insertions(+), 62 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 6cc1f9e0975e75..5814058befeca2 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -43,7 +43,7 @@ def get_spack_commit() -> Optional[str]: Returns: (str or None) the commit sha if available, otherwise None """ - git_path = os.path.join(spack.paths_base.prefix, ".git") + git_path = os.path.join(spack.paths_base.locations.prefix, ".git") if not os.path.exists(git_path): return None @@ -53,7 +53,7 @@ def get_spack_commit() -> Optional[str]: rev = git( "-C", - spack.paths_base.prefix, + spack.paths_base.locations.prefix, "rev-parse", "HEAD", output=str, diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 8ddb9abfb99761..9543512165b828 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -20,8 +20,8 @@ import spack.extensions import spack.llnl.string import spack.llnl.util.tty as tty -import spack.paths -import spack.paths_base +from spack.paths import locations as paths +from spack.paths_base import locations as paths_base import spack.repo import spack.spec import spack.spec_parser @@ -88,7 +88,7 @@ def all_commands(): global _all_commands if _all_commands is None: _all_commands = [] - command_paths = [spack.paths_base.command_path] # Built-in commands + command_paths = [paths_base.command_path] # Built-in commands command_paths += spack.extensions.get_command_paths() # Extensions for path in command_paths: for file in os.listdir(path): @@ -580,7 +580,7 @@ def print_how_many_pkgs(specs, pkg_type="", suffix=""): def spack_is_git_repo(): """Ensure that this instance of Spack is a git clone.""" - return is_git_repo(spack.paths.prefix) + return is_git_repo(paths.prefix) def is_git_repo(path): diff --git a/lib/spack/spack/cmd/common/__init__.py b/lib/spack/spack/cmd/common/__init__.py index 8c29ae83ed2148..38471bea1ecbd4 100644 --- a/lib/spack/spack/cmd/common/__init__.py +++ b/lib/spack/spack/cmd/common/__init__.py @@ -25,19 +25,19 @@ def shell_init_instructions(cmd, equivalent): "To set up shell support, run the command below for your shell.", "", color.colorize("@*c{For bash/zsh/sh:}"), - " . %s/setup-env.sh" % spack.paths_base.share_path, + " . %s/setup-env.sh" % spack.paths_base.locations.share_path, "", color.colorize("@*c{For csh/tcsh:}"), - " source %s/setup-env.csh" % spack.paths_base.share_path, + " source %s/setup-env.csh" % spack.paths_base.locations.share_path, "", color.colorize("@*c{For fish:}"), - " source %s/setup-env.fish" % spack.paths_base.share_path, + " source %s/setup-env.fish" % spack.paths_base.locations.share_path, "", color.colorize("@*c{For Windows batch:}"), - " %s\\spack_cmd.bat" % spack.paths_base.bin_path, + " %s\\spack_cmd.bat" % spack.paths_base.locations.bin_path, "", color.colorize("@*c{For PowerShell:}"), - " %s\\setup-env.ps1" % spack.paths_base.share_path, + " %s\\setup-env.ps1" % spack.paths_base.locations.share_path, "", "Or, if you do not want to use shell support, run " + ("one of these" if shell_specific else "this") diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 4c083e52363a57..5ccb9ccea4d1e4 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -42,7 +42,7 @@ from spack.vendor import jsonschema import spack.error -import spack.paths_base +from spack.paths_base import locations as paths_base import spack.schema import spack.schema.bootstrap import spack.schema.cdash @@ -104,7 +104,7 @@ } #: Path to the main configuration scope -CONFIGURATION_DEFAULTS_PATH = ("defaults", os.path.join(spack.paths_base.etc_path, "defaults")) +CONFIGURATION_DEFAULTS_PATH = ("defaults", os.path.join(paths_base.etc_path, "defaults")) #: Hard-coded default values for some key configuration options. #: This ensures that Spack will still work even if config.yaml in @@ -1561,7 +1561,7 @@ def create_incremental() -> Generator[Configuration, None, None]: # Initial topmost scope is spack (the config scope in the spack instance). # It includes the user, site, and system scopes. Environments and command # line scopes go above this. - configuration_paths = [("spack", os.path.join(spack.paths_base.etc_path))] + configuration_paths = [("spack", os.path.join(paths_base.etc_path))] # Python packages can register configuration scopes via entry_points configuration_paths.extend(config_paths_from_entry_points()) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 481f31fcf9e8b4..107e0fd2baac57 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -604,7 +604,7 @@ def setup_main_options(args): # override lock configuration if passed on command line if args.locks is not None: if args.locks is False: - spack.util.lock.check_lock_safety(spack.paths_base.prefix) + spack.util.lock.check_lock_safety(spack.paths_base.locations.prefix) spack.config.set("config:locks", args.locks, scope="command_line") if args.mock: @@ -613,7 +613,7 @@ def setup_main_options(args): key = syaml.syaml_str("repos") key.override = True spack.config.CONFIG.scopes["command_line"].sections["repos"] = syaml.syaml_dict( - [(key, [spack.paths_base.mock_packages_path])] + [(key, [spack.paths_base.locations.mock_packages_path])] ) # If the user asked for it, don't check ssl certs. diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index 542efbd44516cf..0bef3164b407bc 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -106,32 +106,6 @@ def env_based_state_home(self): locations = SpackPathsBase() -prefix = locations.prefix -spack_root = locations.spack_root -bin_path = locations.bin_path -spack_script = locations.spack_script -sbang_script = locations.sbang_script -lib_path = locations.lib_path -external_path = locations.external_path -module_path = locations.module_path -vendor_path = locations.vendor_path -command_path = locations.command_path -platform_path = locations.platform_path -compilers_path = locations.compilers_path -operating_system_path = locations.operating_system_path -test_path = locations.test_path -hooks_path = locations.hooks_path -share_path = locations.share_path -etc_path = locations.etc_path -var_path = locations.var_path -test_repos_path = locations.test_repos_path -mock_packages_path = locations.mock_packages_path -mock_gpg_data_path = locations.mock_gpg_data_path -mock_gpg_keys_path = locations.mock_gpg_keys_path -system_config_path = locations.system_config_path -user_config_path = locations.user_config_path -spack_instance_id = locations.spack_instance_id - #: Recorded directory where spack command was originally invoked spack_working_dir = None diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index a13e3b1b3aa668..c7c18513e65799 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -135,7 +135,7 @@ def git(): # @pytest.fixture(scope="session") def last_two_git_commits(git): - spack_git_path = spack.paths_base.prefix + spack_git_path = spack.paths_base.locations.prefix with working_dir(spack_git_path): git_log_out = git("log", "-n", "2", output=str, error=os.devnull) @@ -154,11 +154,8 @@ def write_file(filename, contents): @pytest.fixture def override_path(monkeypatch): def _override(path_attr, new_path): - if hasattr(spack.paths_base, path_attr): - monkeypatch.setattr(spack.paths_base, path_attr, new_path) - - if hasattr(spack.paths, path_attr): - monkeypatch.setattr(spack.paths, path_attr, new_path) + if hasattr(spack.paths_base.locations, path_attr): + monkeypatch.setattr(spack.paths_base.locations, path_attr, new_path) if hasattr(spack.paths.locations, path_attr): monkeypatch.setattr(spack.paths.locations, path_attr, new_path) @@ -333,25 +330,25 @@ def latest_commit(): os.makedirs(os.path.dirname(filename)) # add diff-test as a new package to the repository - shutil.copy2(f"{spack.paths_base.test_path}/data/conftest/diff-test/package-0.txt", filename) + shutil.copy2(f"{spack.paths_base.locations.test_path}/data/conftest/diff-test/package-0.txt", filename) git("add", filename) commit("diff-test: new package") commits.append(latest_commit()) # add v2.1.5 to diff-test - shutil.copy2(f"{spack.paths_base.test_path}/data/conftest/diff-test/package-1.txt", filename) + shutil.copy2(f"{spack.paths_base.locations.test_path}/data/conftest/diff-test/package-1.txt", filename) git("add", filename) commit("diff-test: add v2.1.5") commits.append(latest_commit()) # add v2.1.6 to diff-test - shutil.copy2(f"{spack.paths_base.test_path}/data/conftest/diff-test/package-2.txt", filename) + shutil.copy2(f"{spack.paths_base.locations.test_path}/data/conftest/diff-test/package-2.txt", filename) git("add", filename) commit("diff-test: add v2.1.6") commits.append(latest_commit()) # add v2.1.7 and v2.1.8 to diff-test - shutil.copy2(f"{spack.paths_base.test_path}/data/conftest/diff-test/package-3.txt", filename) + shutil.copy2(f"{spack.paths_base.locations.test_path}/data/conftest/diff-test/package-3.txt", filename) git("add", filename) commit("diff-test: add v2.1.7 and v2.1.8") commits.append(latest_commit()) @@ -709,7 +706,7 @@ def _use_test_platform(test_platform): # @pytest.fixture(scope="session") def mock_packages_repo(): - yield spack.repo.from_path(spack.paths_base.mock_packages_path) + yield spack.repo.from_path(spack.paths_base.locations.mock_packages_path) def _pkg_install_fn(pkg, spec, prefix): @@ -756,7 +753,7 @@ def mock_packages(mock_packages_repo, mock_pkg_install, request): def mutable_mock_repo(mock_packages_repo, request): """Function-scoped mock packages, for tests that need to modify them.""" ensure_configuration_fixture_run_before(request) - mock_repo = spack.repo.from_path(spack.paths_base.mock_packages_path) + mock_repo = spack.repo.from_path(spack.paths_base.locations.mock_packages_path) with spack.repo.use_repositories(mock_repo) as mock_packages_repo: yield mock_packages_repo @@ -771,7 +768,7 @@ def __init__(self, root_directory: str) -> None: namespace = f"test_namespace_{RepoBuilder._counter}" repo_root = os.path.join(root_directory, namespace) os.makedirs(repo_root, exist_ok=True) - self.template_dirs = (os.path.join(spack.paths_base.share_path, "templates"),) + self.template_dirs = (os.path.join(spack.paths_base.locations.share_path, "templates"),) self.root, self.namespace = spack.repo.create_repo(repo_root, namespace) self.build_system_name = f"test_build_system_{self.namespace}" self._add_build_system() @@ -870,7 +867,7 @@ def default_config(): This ensures we can test the real default configuration without having tests fail when the user overrides the defaults that we test against.""" - defaults_path = os.path.join(spack.paths_base.etc_path, "defaults") + defaults_path = os.path.join(spack.paths_base.locations.etc_path, "defaults") if sys.platform == "win32": defaults_path = os.path.join(defaults_path, "windows") with spack.config.use_configuration(defaults_path) as defaults_config: @@ -883,7 +880,7 @@ def mock_uarch_json(tmp_path_factory: pytest.TempPathFactory): tmpdir = tmp_path_factory.mktemp("microarchitectures") uarch_json_source = ( - Path(spack.paths_base.test_path) / "data" / "microarchitectures" / "microarchitectures.json" + Path(spack.paths_base.locations.test_path) / "data" / "microarchitectures" / "microarchitectures.json" ) uarch_json_dest = tmpdir / "microarchitectures.json" shutil.copy2(uarch_json_source, uarch_json_dest) @@ -927,7 +924,7 @@ def configuration_dir(tmp_path_factory: pytest.TempPathFactory, linux_os): # /data/config has mock config yaml files in it # copy these to the site config. - test_config = Path(spack.paths_base.test_path) / "data" / "config" + test_config = Path(spack.paths_base.locations.test_path) / "data" / "config" shutil.copytree(test_config, tmp_path / "site") # Create temporary 'defaults', 'site' and 'user' folders @@ -1412,7 +1409,7 @@ def module_configuration(monkeypatch, request, mutable_config): # Key for specific settings relative to this module type writer_key = str(writer_mod.__name__).split(".")[-1] # Root folder for configuration - root_for_conf = os.path.join(spack.paths_base.test_path, "data", "modules", writer_key) + root_for_conf = os.path.join(spack.paths_base.locations.test_path, "data", "modules", writer_key) # ConfigUpdate, when called, will modify configuration, so we need to use # the mutable_config fixture @@ -2025,7 +2022,7 @@ def mock_clone_repo(tmp_path_factory: pytest.TempPathFactory): ) shutil.copytree( - os.path.join(spack.paths_base.mock_packages_path, spack.repo.packages_dir_name), + os.path.join(spack.paths_base.locations.mock_packages_path, spack.repo.packages_dir_name), os.path.join(str(repodir), spack.repo.packages_dir_name), ) @@ -2185,7 +2182,7 @@ def noncyclical_dir_structure(tmp_path: Path): @pytest.fixture(scope="function") def mock_config_data(): - config_data_dir = os.path.join(spack.paths_base.test_path, "data", "config") + config_data_dir = os.path.join(spack.paths_base.locations.test_path, "data", "config") return config_data_dir, os.listdir(config_data_dir) diff --git a/lib/spack/spack/trace.py b/lib/spack/spack/trace.py index d932c5247b9918..776c6ab513e153 100644 --- a/lib/spack/spack/trace.py +++ b/lib/spack/spack/trace.py @@ -5,7 +5,7 @@ import pathlib import warnings -import spack.paths_base as paths_base +from spack.paths_base import locations as paths_base def _most_recent_internal_call(): From b3276c4b01393ced0a7cdadf4dc83faddc02b54b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 27 Mar 2026 11:47:43 -0700 Subject: [PATCH 361/506] rm env_based_state_home (the only reference to it was _include_cache_location, which was recently been removed) --- lib/spack/spack/paths_base.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index 0bef3164b407bc..9b8df4323c0a84 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -92,18 +92,6 @@ def __init__(self, _prefix=None): #: not be shared between those instances. self.spack_instance_id = hash.b32_hash(self.prefix)[:7] - @property - def env_based_state_home(self): - """Spack has config-based logic for choosing a home for most state, but - this is specifically for caching state related to the config system - itself: it is based entirely on env vars and not on configuration - variables. It is not affected by `config:locations:disable_env`. - """ - override = lambda: os.environ.get(XDG_overrides.state_home.value) - xdg = lambda: os.environ.get(XDG_vars.state_home.value) - default = lambda: os.path.expanduser(os.path.join("~", ".state", "spack")) - return override() or xdg() or default() - locations = SpackPathsBase() From 53fa1dd2ed76004dcdfbe7eef8b70921e83c8a5b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 27 Mar 2026 12:00:52 -0700 Subject: [PATCH 362/506] style fix; reference error in paths_base.working_dir --- lib/spack/spack/cmd/__init__.py | 4 ++-- lib/spack/spack/config.py | 2 +- lib/spack/spack/paths_base.py | 4 ++-- lib/spack/spack/test/conftest.py | 29 +++++++++++++++++++++++------ 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 9543512165b828..46b95c1d00b8ea 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -20,8 +20,6 @@ import spack.extensions import spack.llnl.string import spack.llnl.util.tty as tty -from spack.paths import locations as paths -from spack.paths_base import locations as paths_base import spack.repo import spack.spec import spack.spec_parser @@ -34,6 +32,8 @@ from spack.llnl.util.lang import attr_setdefault, index_by from spack.llnl.util.tty.colify import colify from spack.llnl.util.tty.color import colorize +from spack.paths import locations as paths +from spack.paths_base import locations as paths_base from ..enums import InstallRecordStatus diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 5ccb9ccea4d1e4..96e5aa83c68b2f 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -42,7 +42,6 @@ from spack.vendor import jsonschema import spack.error -from spack.paths_base import locations as paths_base import spack.schema import spack.schema.bootstrap import spack.schema.cdash @@ -70,6 +69,7 @@ import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml from spack.llnl.util import filesystem, lang, tty +from spack.paths_base import locations as paths_base from spack.util.cpus import cpus_available from spack.util.spack_yaml import get_mark_from_yaml_data diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index 9b8df4323c0a84..b8adb4b98d9448 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -105,5 +105,5 @@ def set_working_dir(): try: spack_working_dir = os.getcwd() except OSError: - os.chdir(prefix) - spack_working_dir = prefix + os.chdir(locations.prefix) + spack_working_dir = locations.prefix diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index c7c18513e65799..195ba3999ffe35 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -330,25 +330,37 @@ def latest_commit(): os.makedirs(os.path.dirname(filename)) # add diff-test as a new package to the repository - shutil.copy2(f"{spack.paths_base.locations.test_path}/data/conftest/diff-test/package-0.txt", filename) + shutil.copy2( + f"{spack.paths_base.locations.test_path}/data/conftest/diff-test/package-0.txt", + filename, + ) git("add", filename) commit("diff-test: new package") commits.append(latest_commit()) # add v2.1.5 to diff-test - shutil.copy2(f"{spack.paths_base.locations.test_path}/data/conftest/diff-test/package-1.txt", filename) + shutil.copy2( + f"{spack.paths_base.locations.test_path}/data/conftest/diff-test/package-1.txt", + filename, + ) git("add", filename) commit("diff-test: add v2.1.5") commits.append(latest_commit()) # add v2.1.6 to diff-test - shutil.copy2(f"{spack.paths_base.locations.test_path}/data/conftest/diff-test/package-2.txt", filename) + shutil.copy2( + f"{spack.paths_base.locations.test_path}/data/conftest/diff-test/package-2.txt", + filename, + ) git("add", filename) commit("diff-test: add v2.1.6") commits.append(latest_commit()) # add v2.1.7 and v2.1.8 to diff-test - shutil.copy2(f"{spack.paths_base.locations.test_path}/data/conftest/diff-test/package-3.txt", filename) + shutil.copy2( + f"{spack.paths_base.locations.test_path}/data/conftest/diff-test/package-3.txt", + filename, + ) git("add", filename) commit("diff-test: add v2.1.7 and v2.1.8") commits.append(latest_commit()) @@ -880,7 +892,10 @@ def mock_uarch_json(tmp_path_factory: pytest.TempPathFactory): tmpdir = tmp_path_factory.mktemp("microarchitectures") uarch_json_source = ( - Path(spack.paths_base.locations.test_path) / "data" / "microarchitectures" / "microarchitectures.json" + Path(spack.paths_base.locations.test_path) + / "data" + / "microarchitectures" + / "microarchitectures.json" ) uarch_json_dest = tmpdir / "microarchitectures.json" shutil.copy2(uarch_json_source, uarch_json_dest) @@ -1409,7 +1424,9 @@ def module_configuration(monkeypatch, request, mutable_config): # Key for specific settings relative to this module type writer_key = str(writer_mod.__name__).split(".")[-1] # Root folder for configuration - root_for_conf = os.path.join(spack.paths_base.locations.test_path, "data", "modules", writer_key) + root_for_conf = os.path.join( + spack.paths_base.locations.test_path, "data", "modules", writer_key + ) # ConfigUpdate, when called, will modify configuration, so we need to use # the mutable_config fixture From 0337d7637932852d03ec803b83a563f2c10194d5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 27 Mar 2026 13:22:36 -0700 Subject: [PATCH 363/506] move xdg option to last parameter --- lib/spack/spack/paths.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 9ad36eb8354488..5f784b9046a014 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -77,9 +77,9 @@ def state_home(self): if not self._state_home: self._state_home, self._state_home_provenance = self.resolve_a_home( ["SPACK_STATE_HOME", "SPACK_USER_CACHE_PATH"], - "XDG_STATE_HOME", "state", SpackPaths.relative_state_home, + "XDG_STATE_HOME", ) return self._state_home @@ -87,7 +87,7 @@ def state_home(self): def cache_home(self): if not self._cache_home: self._cache_home, self._cache_home_provenance = self.resolve_a_home( - "SPACK_CACHE_HOME", "XDG_CACHE_HOME", "cache", SpackPaths.relative_cache_home + "SPACK_CACHE_HOME", "cache", SpackPaths.relative_cache_home, "XDG_CACHE_HOME" ) return self._cache_home @@ -95,7 +95,7 @@ def cache_home(self): def data_home(self): if not self._data_home: self._data_home, self._data_home_provenance = self.resolve_a_home( - "SPACK_DATA_HOME", "XDG_DATA_HOME", "data", SpackPaths.relative_data_home + "SPACK_DATA_HOME", "data", SpackPaths.relative_data_home, "XDG_DATA_HOME" ) return self._data_home @@ -234,7 +234,7 @@ def __getattr__(self, name): raise AttributeError(name) return getattr(base, name) - def resolve_a_home(self, spack_vars, xdg_var, config_var, home_rel): + def resolve_a_home(self, spack_vars, config_var, home_rel, xdg_var): """ Data stored by spack is split into state, data, and cache components. This function can resolve where each of these components should be @@ -244,12 +244,12 @@ def resolve_a_home(self, spack_vars, xdg_var, config_var, home_rel): spack_vars: spack-specific environment variables that indicate the component location. Can be a list or a single variable. If this is a list, earlier elements have precedence. - xdg_var: the XDG-based environment variable that indicates the - component location. config_var: the spack config variable that indicates the component location. home_rel: for $SPACK_HOME and config:locations:home, this relative path is appended to the result to get the component location. + xdg_var: the XDG-based environment variable that indicates the + component location. """ disable_env = config.get("config:locations:disable_env", False) From 050a3f92f1c0b680fa1f73f8ea4bd3ef2ff244be Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 27 Mar 2026 13:43:04 -0700 Subject: [PATCH 364/506] update docstring for core fall-through _home logic --- lib/spack/spack/paths.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 5f784b9046a014..ed10ddd405332b 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -236,9 +236,22 @@ def __getattr__(self, name): def resolve_a_home(self, spack_vars, config_var, home_rel, xdg_var): """ - Data stored by spack is split into state, data, and cache components. - This function can resolve where each of these components should be - stored. + Files stored by spack are split into state, data, and cache components. + Each of these categories has the same fall-through/prioritization path, + established by this function: + 1. ``SPACK_x_HOME``: for example if the ``SPACK_DATA_HOME`` env var is + set, it has the highest precedence. + 2. If the ``SPACK_HOME`` env variable is set, it can collect all of these + components together + 3. ``config:locations:x`` + 4. ``config:locations:home`` + 5. ``XDG_x_HOME``: e.g. if the ``XDG_DATA_HOME`` env var is set + 6. In the user's home directory, in the XDG default location for that + component. + + Note that configuration settings for specific data (e.g. + ``config:install_tree:root`` for where installs are placed) will take + precedence over any of this. Args: spack_vars: spack-specific environment variables that indicate the From f6a97868f6f91a9e110ee0d5b1949286d11de6c0 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 27 Mar 2026 13:59:36 -0700 Subject: [PATCH 365/506] rm unused function in test --- lib/spack/spack/test/paths.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 110529e55e278d..47b677a2248c37 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -333,10 +333,6 @@ def __call__(self): ), f"Expected {expected}\nGot {spack.paths.locations.default_install_location}" -def all_dirs_empty(a_dir): - return False - - def test_child_proc_sanity_xdg_based_paths(tmp_path, set_home, monkeypatch): # Unlike the other tests in this module, this is specifically testing # the behavior of the spack.paths module vs. (the more targeted testing From a1c94200efd55a9318819792bce54cf7a8b49cff Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 27 Mar 2026 14:21:29 -0700 Subject: [PATCH 366/506] when using set_home fixture, env should be restored --- lib/spack/spack/test/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 47b677a2248c37..2133ccf3850187 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -34,7 +34,7 @@ def clear_env_vars(working_env): @pytest.fixture -def set_home(): +def set_home(working_env): def _set_home(val): # Clear some env vars that can interfere w/ expanduser(~) on Windows os.environ.pop("USERPROFILE", None) From 7d114d6e82761e4bae59ee666183826744bc11cf Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 27 Mar 2026 14:47:39 -0700 Subject: [PATCH 367/506] refactor out some redundancy in test logic that unsets path env vars --- lib/spack/spack/paths.py | 23 +++++++++++++++++++++++ lib/spack/spack/paths_base.py | 22 ---------------------- lib/spack/spack/test/paths.py | 10 +--------- 3 files changed, 24 insertions(+), 31 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index ed10ddd405332b..dc5c372cd7a6ad 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -8,6 +8,7 @@ throughout Spack and should bring in a minimal number of external dependencies. """ +import itertools import os import pathlib from contextlib import contextmanager @@ -50,6 +51,28 @@ def unilateral_override(self): } +class XDG_vars(Enum): + state_home = "XDG_STATE_HOME" + data_home = "XDG_DATA_HOME" + cache_home = "XDG_CACHE_HOME" + + +class Spack_vars(Enum): + state_home = "SPACK_STATE_HOME" + data_home = "SPACK_DATA_HOME" + cache_home = "SPACK_CACHE_HOME" + user_cache_path = "SPACK_USER_CACHE_PATH" + home = "SPACK_HOME" + + +# This is for tests that want to clean the environment of XDG_ variables that +# affect spack behavior. Note that this will not influence install_test.py's +# view of config:test_stage +def _unset_path_vars(env): + for env_var in itertools.chain(XDG_vars, Spack_vars): + env.pop(env_var.value, None) + + class SpackPaths: relative_state_home = os.path.join(".local", "state") diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index b8adb4b98d9448..7106fb6763c8f9 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -2,35 +2,13 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import itertools import os -from enum import Enum from pathlib import PurePath import spack.llnl.util.filesystem import spack.util.hash as hash -class XDG_vars(Enum): - state_home = "XDG_STATE_HOME" - data_home = "XDG_DATA_HOME" - cache_home = "XDG_CACHE_HOME" - - -class XDG_overrides(Enum): - state_home = "SPACK_STATE_HOME" - data_home = "SPACK_DATA_HOME" - cache_home = "SPACK_CACHE_HOME" - - -# This is for tests that want to clean the environment of XDG_ variables that -# affect spack behavior. Note that this will not influence install_test.py's -# view of config:test_stage -def _unset_xdg_vars(env): - for xdg_var in itertools.chain(XDG_vars, XDG_overrides): - env.pop(xdg_var.value, None) - - class SpackPathsBase: def __init__(self, _prefix=None): #: This file lives in $prefix/lib/spack/spack/__file__ diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 2133ccf3850187..2d2561f7f78c06 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -22,15 +22,7 @@ def _ensure_dir(pathlike): @pytest.fixture(autouse=True) def clear_env_vars(working_env): - spack.paths_base._unset_xdg_vars(os.environ) - for x in [ - "SPACK_USER_CACHE_PATH", - "SPACK_HOME", - "SPACK_DATA_HOME", - "SPACK_STATE_HOME", - "SPACK_CACHE_HOME", - ]: - os.environ.pop(x, None) + spack.paths._unset_path_vars(os.environ) @pytest.fixture From 5fa53796d4dd6b0ed6c31e39c4bab6b92a04c72e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sun, 29 Mar 2026 18:32:56 -0700 Subject: [PATCH 368/506] path attributes are no longer duplicated between classes --- lib/spack/spack/test/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 195ba3999ffe35..250b9698f7cd6a 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -156,8 +156,7 @@ def override_path(monkeypatch): def _override(path_attr, new_path): if hasattr(spack.paths_base.locations, path_attr): monkeypatch.setattr(spack.paths_base.locations, path_attr, new_path) - - if hasattr(spack.paths.locations, path_attr): + elif hasattr(spack.paths.locations, path_attr): monkeypatch.setattr(spack.paths.locations, path_attr, new_path) return _override From 8cef75234914b632027b98e9a240cd4d473c748a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 30 Mar 2026 17:30:03 -0700 Subject: [PATCH 369/506] fix docstring formatting --- lib/spack/spack/paths.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index dc5c372cd7a6ad..d202d0616fb3d0 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -262,15 +262,16 @@ def resolve_a_home(self, spack_vars, config_var, home_rel, xdg_var): Files stored by spack are split into state, data, and cache components. Each of these categories has the same fall-through/prioritization path, established by this function: - 1. ``SPACK_x_HOME``: for example if the ``SPACK_DATA_HOME`` env var is - set, it has the highest precedence. - 2. If the ``SPACK_HOME`` env variable is set, it can collect all of these - components together - 3. ``config:locations:x`` - 4. ``config:locations:home`` - 5. ``XDG_x_HOME``: e.g. if the ``XDG_DATA_HOME`` env var is set - 6. In the user's home directory, in the XDG default location for that - component. + + 1. ``SPACK_x_HOME``: for example if the ``SPACK_DATA_HOME`` env var is + set, it has the highest precedence. + 2. If the ``SPACK_HOME`` env variable is set, it can collect all of these + components together + 3. ``config:locations:x`` + 4. ``config:locations:home`` + 5. ``XDG_x_HOME``: e.g. if the ``XDG_DATA_HOME`` env var is set + 6. In the user's home directory, in the XDG default location for that + component. Note that configuration settings for specific data (e.g. ``config:install_tree:root`` for where installs are placed) will take From 2a47c187394300f5e282b7c4ece283700ce49223 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 30 Mar 2026 19:17:41 -0700 Subject: [PATCH 370/506] clarify that generally spack.paths attributes cannot be directly overridden --- lib/spack/spack/paths.py | 24 ++++++++++++------------ lib/spack/spack/test/config.py | 5 ----- lib/spack/spack/test/conftest.py | 16 ++++++++++++---- lib/spack/spack/test/git_fetch.py | 2 +- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index d202d0616fb3d0..42b07cd9968975 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -139,14 +139,6 @@ def spack_home(self): def user_cache_path(self): return self.state_home - @contextmanager - def redirect_state_home(self, x): - old = self._state_home - self._state_home = x - yield - if old: - self._state_home = old - @property def default_install_location(self): return self._decide_old_or_new_location( @@ -220,10 +212,18 @@ def user_repos_cache_path(self): return self._user_repos_cache_path return os.path.join(self.state_home, "git_repos") - @user_repos_cache_path.setter - def user_repos_cache_path(self, val): - # setter for tests - self._user_repos_cache_path = val + @contextmanager + def redirect_state_home(self, x): + old = self._state_home + self._state_home = x + yield + self._state_home = old + + @contextmanager + def redirect_user_repos_cache_path(self, x): + self._user_repos_cache_path = x + yield + delattr(self, "_user_repos_cache_path") @property def package_repos_path(self): diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 2488df8db8434d..f518eed2f22cf8 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1924,11 +1924,6 @@ def test_included_path_git_temp_dest(mock_low_high_config): assert dest_dir == temp_dir, pre + rest -@pytest.fixture -def redirect_user_cache(tmp_path: pathlib.Path): - spack_paths.redirect_state_home(tmp_path) - - def test_included_path_git_errs(mock_low_high_config, monkeypatch, redirect_user_cache): paths = ["concretizer.yaml"] entry = { diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 250b9698f7cd6a..9ea558e54c63c7 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -157,20 +157,28 @@ def _override(path_attr, new_path): if hasattr(spack.paths_base.locations, path_attr): monkeypatch.setattr(spack.paths_base.locations, path_attr, new_path) elif hasattr(spack.paths.locations, path_attr): - monkeypatch.setattr(spack.paths.locations, path_attr, new_path) + raise ValueError( + "To redirect individual spack.paths attributes, use (or add)" + " an appropriate redirect to spack.paths and conftest" + ) return _override @pytest.fixture -def override_git_repos_cache_path(tmp_path: Path, monkeypatch, override_path): +def redirect_user_repos_cache_path(tmp_path: Path): tmp_git_path = tmp_path / "git-repo-cache-path-for-tests" tmp_git_path.mkdir() - override_path("user_repos_cache_path", str(tmp_git_path)) + spack.paths.locations.redirect_user_repos_cache_path(tmp_git_path) + + +@pytest.fixture +def redirect_user_cache(tmp_path: Path): + spack.paths.locations.redirect_state_home(tmp_path) @pytest.fixture -def mock_git_version_info(git, tmp_path: Path, override_git_repos_cache_path): +def mock_git_version_info(git, tmp_path: Path, redirect_user_repos_cache_path): """Create a mock git repo with known structure The structure of commits in this repo is as follows:: diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 0eccf2dab412f4..1607f09f721932 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -186,7 +186,7 @@ def test_adhoc_version_submodules( mutable_mock_repo, monkeypatch, mock_stage, - override_git_repos_cache_path, + redirect_user_repos_cache_path, ): t = mock_git_repository.checks["tag"] # Construct the package under test From 4ae876cc01fde5a24aa7fa4899d0658e66c4c83c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 30 Mar 2026 19:23:59 -0700 Subject: [PATCH 371/506] rename fixture; add explanation for lower-level redirect --- lib/spack/spack/paths.py | 2 ++ lib/spack/spack/test/config.py | 2 +- lib/spack/spack/test/conftest.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 42b07cd9968975..8186212d17a881 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -221,6 +221,8 @@ def redirect_state_home(self, x): @contextmanager def redirect_user_repos_cache_path(self, x): + # This is under state_home, but tests may want to selectively redirect + # only this attribute (e.g. to avoid regenerating provider cache) self._user_repos_cache_path = x yield delattr(self, "_user_repos_cache_path") diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index f518eed2f22cf8..2a9b2e7d1604ef 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1924,7 +1924,7 @@ def test_included_path_git_temp_dest(mock_low_high_config): assert dest_dir == temp_dir, pre + rest -def test_included_path_git_errs(mock_low_high_config, monkeypatch, redirect_user_cache): +def test_included_path_git_errs(mock_low_high_config, monkeypatch, redirect_state_home): paths = ["concretizer.yaml"] entry = { "git": "https://example.com/linux/configs.git", diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 9ea558e54c63c7..3184a7427eefba 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -173,7 +173,7 @@ def redirect_user_repos_cache_path(tmp_path: Path): @pytest.fixture -def redirect_user_cache(tmp_path: Path): +def redirect_state_home(tmp_path: Path): spack.paths.locations.redirect_state_home(tmp_path) From 602bcdee022221893acab84dab4ff56eb5153607 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 30 Mar 2026 19:44:46 -0700 Subject: [PATCH 372/506] update outdated reference --- lib/spack/spack/test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 3184a7427eefba..daaa1e2e37549e 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -284,7 +284,7 @@ def latest_commit(): @pytest.fixture -def mock_git_package_changes(git, tmp_path: Path, override_git_repos_cache_path, monkeypatch): +def mock_git_package_changes(git, tmp_path: Path, redirect_user_repos_cache_path): """Create a mock git repo with known structure of package edits The structure of commits in this repo is as follows:: From 6b88c537f3010f8f2c2c58047536d28e25d85beb Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 31 Mar 2026 16:25:42 -0700 Subject: [PATCH 373/506] add docs section about restoring old locations behavior after updating to 1.2 --- lib/spack/docs/index.rst | 1 + lib/spack/docs/updating_spack.rst | 41 +++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 lib/spack/docs/updating_spack.rst diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 2aaa4c6d0aaba4..b993502c03e550 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -56,6 +56,7 @@ If you're new to Spack and want to start using it, see :doc:`getting_started`, o package_fundamentals configuring_compilers environments_basics + updating_spack frequently_asked_questions getting_help diff --git a/lib/spack/docs/updating_spack.rst b/lib/spack/docs/updating_spack.rst new file mode 100644 index 00000000000000..fc1cfc109cc837 --- /dev/null +++ b/lib/spack/docs/updating_spack.rst @@ -0,0 +1,41 @@ +.. + Copyright Spack Project Developers. See COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. meta:: + :description lang=en: + If you update an existing Spack instance, this section explains what to look out for. + +Updating Spack +============== + +If you have an existing instance of spack and you update it with ``git pull``, this should generally work fine. +Spack's behavior can differ between minor releases and this section explains what you might have to update. + +The most-significant changes are documented in https://github.com/spack/spack/discussions/30634 + +1.2 +--- + +Moving all data written by Spack out of the spack prefix +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +See: https://github.com/spack/spack/pull/47615 + +One of the most significant changes is where Spack will put (and look for) installed packages: + +- If you don't clone a new instance of Spack, and only ``git pull`` into old instances, then each of those will continue to write into their own prefixes. +- If new instances are cloned but are configured to write outside of the default in ``~``, then old spack instances will continue to write into their own prefixes. +- If any new spack instance is cloned and writes into ``~`` (the new default in 1.2), then any old spack instance that updates to version 1.2 will start writing into ``~`` as well. + - If you want to force an old instance to continue to write into the old default after pulling 1.2, then you will need to set ``config:install_tree:root`` to point to that location (``$spack/opt/spack``). + - You can do this "ahead of time" as well: set ``config:install_tree:root`` before pulling 1.2. + +Other data stored in ``$spack`` also moves out in 1.2: + +- Environments, and you can restore old behavior with ``config:environments_root:$spack/var/spack/environments``. +- Modules, and you can restore old behavior by setting ``modules:default:roots:{tcl,lmod}`` to ``$spack/share/spack/{modules,lmod}`` +- GPG keys + - Default trusted keys for ``spack gpg init`` are stored in ``$spack/var/spack/gpg``. + If mixing old and new instances of spack ``spack gpg init --from=/var/spack/gpg`` if you placed any keys there (in other words, that directory is managed like any external store of GPG keys). + - You can use the old storage location for keys by setting the ``SPACK_GNUPGHOME`` environment variable to ``/opt/spack/gpg`` \ No newline at end of file From a888ef31f57363eab0198aba98bd647b8da9ef80 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 31 Mar 2026 16:44:27 -0700 Subject: [PATCH 374/506] proper list sub-bullet formatting --- lib/spack/docs/updating_spack.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/docs/updating_spack.rst b/lib/spack/docs/updating_spack.rst index fc1cfc109cc837..e46aee48d833e5 100644 --- a/lib/spack/docs/updating_spack.rst +++ b/lib/spack/docs/updating_spack.rst @@ -28,6 +28,7 @@ One of the most significant changes is where Spack will put (and look for) insta - If you don't clone a new instance of Spack, and only ``git pull`` into old instances, then each of those will continue to write into their own prefixes. - If new instances are cloned but are configured to write outside of the default in ``~``, then old spack instances will continue to write into their own prefixes. - If any new spack instance is cloned and writes into ``~`` (the new default in 1.2), then any old spack instance that updates to version 1.2 will start writing into ``~`` as well. + - If you want to force an old instance to continue to write into the old default after pulling 1.2, then you will need to set ``config:install_tree:root`` to point to that location (``$spack/opt/spack``). - You can do this "ahead of time" as well: set ``config:install_tree:root`` before pulling 1.2. @@ -36,6 +37,7 @@ Other data stored in ``$spack`` also moves out in 1.2: - Environments, and you can restore old behavior with ``config:environments_root:$spack/var/spack/environments``. - Modules, and you can restore old behavior by setting ``modules:default:roots:{tcl,lmod}`` to ``$spack/share/spack/{modules,lmod}`` - GPG keys + - Default trusted keys for ``spack gpg init`` are stored in ``$spack/var/spack/gpg``. If mixing old and new instances of spack ``spack gpg init --from=/var/spack/gpg`` if you placed any keys there (in other words, that directory is managed like any external store of GPG keys). - - You can use the old storage location for keys by setting the ``SPACK_GNUPGHOME`` environment variable to ``/opt/spack/gpg`` \ No newline at end of file + - You can use the old storage location for keys by setting the ``SPACK_GNUPGHOME`` environment variable to ``/opt/spack/gpg``. From 4a795d728b1c4e1a68d3cccc1c45d8628776c967 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 1 Apr 2026 12:18:23 -0700 Subject: [PATCH 375/506] move controlling-where-spack-writes-data from package-fundamentals to configuration doc section --- ...iguration.rst => configuration_basics.rst} | 33 ++++++++++++++++++- lib/spack/docs/index.rst | 2 +- lib/spack/docs/package_fundamentals.rst | 23 ------------- 3 files changed, 33 insertions(+), 25 deletions(-) rename lib/spack/docs/{configuration.rst => configuration_basics.rst} (95%) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration_basics.rst similarity index 95% rename from lib/spack/docs/configuration.rst rename to lib/spack/docs/configuration_basics.rst index 2609bb0178a8f3..611825ee30e6c0 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration_basics.rst @@ -9,8 +9,16 @@ .. _configuration: +Configuration Basics +==================== + +Spack is configured with YAML files. +You can customize Spack's behavior by modifying these files, and you can also tell it to look for files in custom locations. +This section covers how configuration is generally organized and processed. +The configuration is organized into categories with predefined schemas, and individual sections cover each of these categories. + Configuration Files -=================== +------------------- Spack has many configuration files. Here is a quick list of them, in case you want to skip directly to specific docs: @@ -26,6 +34,29 @@ Here is a quick list of them, in case you want to skip directly to specific docs You can also add any of these as inline configuration in the YAML manifest file (``spack.yaml``) describing an :ref:`environment `. +Controlling where spack writes data +----------------------------------- + +A fresh checkout of spack will not write anything into the `$spack`` prefix; instead, all data is placed under the user's home directory. +You can control this in the following ways: + +* Redirect everything with environment variables: set ``SPACK_HOME`` and one of ``SPACK_USER_CONFIG_PATH`` or ``SPACK_DISABLE_LOCAL_CONFIG=1`` +* Or redirect everything with config: + + * set ``config:locations:home`` + * Update the ``user`` config scope with ``spack config --scope=spack edit include`` +* Or redirect installs, environments, and cached downloads (everything that takes up significant space) by setting ``SPACK_DATA_HOME`` +* Or use finer-grained configuration settings, for example: + + * ``config:install_tree:root`` to control where installs go + * ``config:build_stage`` to control where builds are staged + +For more on this, see: + +* :ref:`Variables controlling data location ` +* :ref:`include.yaml ` +* :ref:`config.yaml ` + YAML Format ----------- diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index b993502c03e550..d42c746ff19b06 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -72,7 +72,7 @@ If you're new to Spack and want to start using it, see :doc:`getting_started`, o :maxdepth: 2 :caption: Configuration - configuration + configuration_basics config_yaml packages_yaml toolchains_yaml diff --git a/lib/spack/docs/package_fundamentals.rst b/lib/spack/docs/package_fundamentals.rst index fe2ed2a34cce60..8817c316354710 100644 --- a/lib/spack/docs/package_fundamentals.rst +++ b/lib/spack/docs/package_fundamentals.rst @@ -116,29 +116,6 @@ And if you *only* wanted to see packages that provide MPI-2, you would add a ver Notice that the package versions that provide insufficient MPI versions are now filtered out. -Controlling where spack writes data ------------------------------------ - -A fresh checkout of spack will not write anything into the `$spack`` prefix; instead, all data is placed under the user's home directory. -You can control this in the following ways: - -* Redirect everything with environment variables: set ``SPACK_HOME`` and one of ``SPACK_USER_CONFIG_PATH`` or ``SPACK_DISABLE_LOCAL_CONFIG=1`` -* Or redirect everything with config: - - * set ``config:locations:home`` - * Update the ``user`` config scope with ``spack config --scope=spack edit include`` -* Or redirect installs, environments, and cached downloads (everything that takes up significant space) by setting ``SPACK_DATA_HOME`` -* Or use finer-grained configuration settings, for example: - - * ``config:install_tree:root`` to control where installs go - * ``config:build_stage`` to control where builds are staged - -For more on this, see: - -* :ref:`Variables controlling data location ` -* :ref:`include.yaml ` -* :ref:`config.yaml ` - Installing and Uninstalling --------------------------- From 82ab0303a1dcd0be273ca5e545226554c994fc28 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 1 Apr 2026 13:33:34 -0700 Subject: [PATCH 376/506] updating-spack high-level docs section was removed (content was moved to 30634) Signed-off-by: Peter Scheibel --- lib/spack/docs/index.rst | 1 - lib/spack/docs/updating_spack.rst | 43 ------------------------------- 2 files changed, 44 deletions(-) delete mode 100644 lib/spack/docs/updating_spack.rst diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index b19bb37ea9f740..320f1d4a62abea 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -57,7 +57,6 @@ If you're new to Spack and want to start using it, see :doc:`getting_started`, o installing configuring_compilers environments_basics - updating_spack frequently_asked_questions getting_help diff --git a/lib/spack/docs/updating_spack.rst b/lib/spack/docs/updating_spack.rst deleted file mode 100644 index e46aee48d833e5..00000000000000 --- a/lib/spack/docs/updating_spack.rst +++ /dev/null @@ -1,43 +0,0 @@ -.. - Copyright Spack Project Developers. See COPYRIGHT file for details. - - SPDX-License-Identifier: (Apache-2.0 OR MIT) - -.. meta:: - :description lang=en: - If you update an existing Spack instance, this section explains what to look out for. - -Updating Spack -============== - -If you have an existing instance of spack and you update it with ``git pull``, this should generally work fine. -Spack's behavior can differ between minor releases and this section explains what you might have to update. - -The most-significant changes are documented in https://github.com/spack/spack/discussions/30634 - -1.2 ---- - -Moving all data written by Spack out of the spack prefix -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -See: https://github.com/spack/spack/pull/47615 - -One of the most significant changes is where Spack will put (and look for) installed packages: - -- If you don't clone a new instance of Spack, and only ``git pull`` into old instances, then each of those will continue to write into their own prefixes. -- If new instances are cloned but are configured to write outside of the default in ``~``, then old spack instances will continue to write into their own prefixes. -- If any new spack instance is cloned and writes into ``~`` (the new default in 1.2), then any old spack instance that updates to version 1.2 will start writing into ``~`` as well. - - - If you want to force an old instance to continue to write into the old default after pulling 1.2, then you will need to set ``config:install_tree:root`` to point to that location (``$spack/opt/spack``). - - You can do this "ahead of time" as well: set ``config:install_tree:root`` before pulling 1.2. - -Other data stored in ``$spack`` also moves out in 1.2: - -- Environments, and you can restore old behavior with ``config:environments_root:$spack/var/spack/environments``. -- Modules, and you can restore old behavior by setting ``modules:default:roots:{tcl,lmod}`` to ``$spack/share/spack/{modules,lmod}`` -- GPG keys - - - Default trusted keys for ``spack gpg init`` are stored in ``$spack/var/spack/gpg``. - If mixing old and new instances of spack ``spack gpg init --from=/var/spack/gpg`` if you placed any keys there (in other words, that directory is managed like any external store of GPG keys). - - You can use the old storage location for keys by setting the ``SPACK_GNUPGHOME`` environment variable to ``/opt/spack/gpg``. From 67b6c96a95b696301ae68c511631eef19eafd117 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 1 Apr 2026 13:39:26 -0700 Subject: [PATCH 377/506] merge accidentally re-added something that was removed Signed-off-by: Peter Scheibel --- lib/spack/spack/new_installer.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/spack/spack/new_installer.py b/lib/spack/spack/new_installer.py index 60d9bbe59fb84e..cfe6d3e3b8a480 100644 --- a/lib/spack/spack/new_installer.py +++ b/lib/spack/spack/new_installer.py @@ -326,10 +326,6 @@ def __init__(self): self.store = spack.store.STORE self.monkey_patches = spack.subprocess_context.TestPatches.create() self.spack_working_dir = spack.paths_base.spack_working_dir - # Avoid 8k stat calls in build process. The downside of this is the additional startup - # cost that blocks the parent process in `proc.start()`, but we avoid filesystem pressure. - # TODO: we don't need to send this if Spec.satisfies(...) etc does not depend on the repo. - self.repo_cache = spack.repo.FastPackageChecker._paths_cache def restore(self): if multiprocessing.get_start_method() == "fork": From 3ef4017f2e54bee09873efd6d66d064d5692e759 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 1 Apr 2026 13:51:23 -0700 Subject: [PATCH 378/506] warning message typo Signed-off-by: Peter Scheibel --- lib/spack/spack/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 8186212d17a881..ff28f23b5e420d 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -178,7 +178,7 @@ def bypassed_old_envs_warning(self, _show=True): " environment path resolution mechanism is active and determined that" f" {self.default_envs_path} is where it should look for and" " place new environments. You can suppress this warning by setting" - " config:install_tree:root, config:locations:home, config:locations:data," + " config:environments_root, config:locations:home, config:locations:data," " SPACK_DATA_HOME, or SPACK_HOME" ) if _show: From 57abed898895fff0f14c8286f91fc6c73fe9e3ba Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 1 Apr 2026 13:57:34 -0700 Subject: [PATCH 379/506] add bypass warning for gpg keys (like for installs and environments) Signed-off-by: Peter Scheibel --- lib/spack/spack/paths.py | 19 +++++++++++++++++++ lib/spack/spack/util/gpg.py | 5 ++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index ff28f23b5e420d..e51194e86e9f5c 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -186,6 +186,25 @@ def bypassed_old_envs_warning(self, _show=True): return msg return "" + def bypassed_old_gpg_warning(self, _show=True): + if ( + self.gpg_path != self.base.old_gpg_path + and dir_is_occupied(self.base.old_gpg_path) + and not self._data_home_provenance.unilateral_override() + ): + msg = ( + f"Detected GPG keys in {self.base.old_gpg_path}; Spack's default" + " gpg keys resolution mechanism is active and determined that" + f" {self.gpg_path} is where it should look for and" + " place new GPG keys. You can suppress this warning by setting" + " config:locations:home, config:locations:data," + " SPACK_GNUPGHOME, SPACK_DATA_HOME, or SPACK_HOME" + ) + if _show: + tty.warn(msg) + return msg + return "" + @property def default_envs_path(self): return self._decide_old_or_new_location( diff --git a/lib/spack/spack/util/gpg.py b/lib/spack/spack/util/gpg.py index 90e832797f6cbe..cf0425fd0e6af8 100644 --- a/lib/spack/spack/util/gpg.py +++ b/lib/spack/spack/util/gpg.py @@ -57,7 +57,10 @@ def init(gnupghome=None, force=False): return # Set the value of GNUPGHOME to be used in this module - GNUPGHOME = gnupghome or os.getenv("SPACK_GNUPGHOME") or paths.gpg_path + GNUPGHOME = gnupghome or os.getenv("SPACK_GNUPGHOME") + if not GNUPGHOME: + GNUPGHOME = paths.gpg_path + paths.bypassed_old_gpg_warning() # Set the executable objects for "gpg" and "gpgconf" with spack.bootstrap.ensure_bootstrap_configuration(): From 6c8e7cccb86dc7bb34b64c1b2884839dff7cf04c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 1 Apr 2026 14:45:36 -0700 Subject: [PATCH 380/506] minor refactor of bypass check Signed-off-by: Peter Scheibel --- lib/spack/spack/paths.py | 79 ++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 44 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index e51194e86e9f5c..22166b5b8c470e 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -149,57 +149,48 @@ def default_install_location(self): ) def bypassed_old_installs_warning(self, _show=True): - if ( - self.default_install_location != self.base.old_install_path - and dir_is_occupied(self.base.old_install_path) - and not self._data_home_provenance.unilateral_override() - ): - msg = ( - f"Detected installs in {self.base.old_install_path}; Spack's default" - " install path resolution mechanism is active and determined that" - f" {self.default_install_location} is where it should look for and" - " place new installs. You can suppress this warning by setting" - " config:install_tree:root, config:locations:home, config:locations:data," - " SPACK_DATA_HOME, or SPACK_HOME" - ) - if _show: - tty.warn(msg) - return msg - return "" + msg = ( + f"Detected installs in {self.base.old_install_path}; Spack's default" + " install path resolution mechanism is active and determined that" + f" {self.default_install_location} is where it should look for and" + " place new installs. You can suppress this warning by setting" + " config:install_tree:root, config:locations:home, config:locations:data," + " SPACK_DATA_HOME, or SPACK_HOME" + ) + return self._bypass_warning( + self.default_install_location, self.base.old_install_path, msg, _show=_show + ) def bypassed_old_envs_warning(self, _show=True): - if ( - self.default_envs_path != self.base.old_envs_path - and dir_is_occupied(self.base.old_envs_path) - and not self._data_home_provenance.unilateral_override() - ): - msg = ( - f"Detected environments in {self.base.old_envs_path}; Spack's default" - " environment path resolution mechanism is active and determined that" - f" {self.default_envs_path} is where it should look for and" - " place new environments. You can suppress this warning by setting" - " config:environments_root, config:locations:home, config:locations:data," - " SPACK_DATA_HOME, or SPACK_HOME" - ) - if _show: - tty.warn(msg) - return msg - return "" + msg = ( + f"Detected environments in {self.base.old_envs_path}; Spack's default" + " environment path resolution mechanism is active and determined that" + f" {self.default_envs_path} is where it should look for and" + " place new environments. You can suppress this warning by setting" + " config:environments_root, config:locations:home, config:locations:data," + " SPACK_DATA_HOME, or SPACK_HOME" + ) + return self._bypass_warning( + self.default_envs_path, self.base.old_envs_path, msg, _show=_show + ) def bypassed_old_gpg_warning(self, _show=True): + msg = ( + f"Detected GPG keys in {self.base.old_gpg_path}; Spack's default" + " gpg keys resolution mechanism is active and determined that" + f" {self.gpg_path} is where it should look for and" + " place new GPG keys. You can suppress this warning by setting" + " config:locations:home, config:locations:data," + " SPACK_GNUPGHOME, SPACK_DATA_HOME, or SPACK_HOME" + ) + return self._bypass_warning(self.gpg_path, self.base.old_gpg_path, msg, _show=_show) + + def _bypass_warning(self, chosen_path, old_path, msg, _show=True): if ( - self.gpg_path != self.base.old_gpg_path - and dir_is_occupied(self.base.old_gpg_path) + chosen_path != old_path + and dir_is_occupied(old_path) and not self._data_home_provenance.unilateral_override() ): - msg = ( - f"Detected GPG keys in {self.base.old_gpg_path}; Spack's default" - " gpg keys resolution mechanism is active and determined that" - f" {self.gpg_path} is where it should look for and" - " place new GPG keys. You can suppress this warning by setting" - " config:locations:home, config:locations:data," - " SPACK_GNUPGHOME, SPACK_DATA_HOME, or SPACK_HOME" - ) if _show: tty.warn(msg) return msg From c290d1b273ecbd8124a671ca0a33f926ea4825e2 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 1 Apr 2026 15:49:35 -0700 Subject: [PATCH 381/506] further consolidation Signed-off-by: Peter Scheibel --- lib/spack/spack/paths.py | 39 +++++++++++------------------------ lib/spack/spack/test/paths.py | 12 +++++------ 2 files changed, 17 insertions(+), 34 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 22166b5b8c470e..06d85581ce4055 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -149,43 +149,28 @@ def default_install_location(self): ) def bypassed_old_installs_warning(self, _show=True): - msg = ( - f"Detected installs in {self.base.old_install_path}; Spack's default" - " install path resolution mechanism is active and determined that" - f" {self.default_install_location} is where it should look for and" - " place new installs. You can suppress this warning by setting" - " config:install_tree:root, config:locations:home, config:locations:data," - " SPACK_DATA_HOME, or SPACK_HOME" - ) + cfg_settings = ["config:install_tree:root", "config:locations:home", "config:locations:data", "SPACK_DATA_HOME", "SPACK_HOME"] return self._bypass_warning( - self.default_install_location, self.base.old_install_path, msg, _show=_show + self.default_install_location, self.base.old_install_path, "installs", cfg_settings, _show=_show ) def bypassed_old_envs_warning(self, _show=True): - msg = ( - f"Detected environments in {self.base.old_envs_path}; Spack's default" - " environment path resolution mechanism is active and determined that" - f" {self.default_envs_path} is where it should look for and" - " place new environments. You can suppress this warning by setting" - " config:environments_root, config:locations:home, config:locations:data," - " SPACK_DATA_HOME, or SPACK_HOME" - ) + cfg_settings = ["config:environments_root", "config:locations:home", "config:locations:data", "SPACK_DATA_HOME", "SPACK_HOME"] return self._bypass_warning( - self.default_envs_path, self.base.old_envs_path, msg, _show=_show + self.default_envs_path, self.base.old_envs_path, "environments", cfg_settings, _show=_show ) def bypassed_old_gpg_warning(self, _show=True): + cfg_settings = ["SPACK_GNUPGHOME", "config:locations:home", "config:locations:data", "SPACK_DATA_HOME", "SPACK_HOME"] + return self._bypass_warning(self.gpg_path, self.base.old_gpg_path, "GPG keys", cfg_settings, _show=_show) + + def _bypass_warning(self, chosen_path, old_path, data_category, cfg_settings, _show=True, ): msg = ( - f"Detected GPG keys in {self.base.old_gpg_path}; Spack's default" - " gpg keys resolution mechanism is active and determined that" - f" {self.gpg_path} is where it should look for and" - " place new GPG keys. You can suppress this warning by setting" - " config:locations:home, config:locations:data," - " SPACK_GNUPGHOME, SPACK_DATA_HOME, or SPACK_HOME" + f"Bypassing data for {data_category} existing in: {old_path}" + f"\nIn favor of: {chosen_path}" + "\nYou can explicitly designate a location (and suppress this warning)" + f" by setting one of: {', '.join(cfg_settings)}" ) - return self._bypass_warning(self.gpg_path, self.base.old_gpg_path, msg, _show=_show) - - def _bypass_warning(self, chosen_path, old_path, msg, _show=True): if ( chosen_path != old_path and dir_is_occupied(old_path) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 2d2561f7f78c06..35bc33eb172db2 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -98,10 +98,9 @@ def paths_base_nonempty_old_install(): ) (pathlib.Path(new_default_installs_dir) / "afile").touch() assert p1.default_install_location == new_default_installs_dir - assert ( - f"{new_default_installs_dir} is where it should look" - in p1.bypassed_old_installs_warning(_show=False) - ) + warning_msg1 = p1.bypassed_old_installs_warning(_show=False) + assert f"Bypassing data for installs existing in: {p1.base.old_install_path}" in warning_msg1 + assert f"In favor of: {new_default_installs_dir}" in warning_msg1 spack.config.set("config:locations", {}) @@ -113,9 +112,8 @@ def paths_base_nonempty_old_install(): p4 = SpackPaths(paths_base_nonempty_old_install()) xdg_installs_location = _ensure_dir(pathlib.Path(xdg_data_home) / "spack" / "installs") assert p4.default_install_location == str(xdg_installs_location) - assert f"{xdg_installs_location} is where it should look" in p4.bypassed_old_installs_warning( - _show=False - ) + warning_msg4 = p4.bypassed_old_installs_warning(_show=False) + assert f"In favor of: {xdg_installs_location}" in warning_msg4 # (sanity) XDG_DATA_HOME still overrides when there is something in it (pathlib.Path(xdg_installs_location) / "afile").touch() From fe91d00246c8a585dbec68a8925e1f07238cc5e9 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 1 Apr 2026 15:56:25 -0700 Subject: [PATCH 382/506] more consolidation and auto style Signed-off-by: Peter Scheibel --- lib/spack/spack/paths.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 06d85581ce4055..41412543e8601f 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -149,27 +149,44 @@ def default_install_location(self): ) def bypassed_old_installs_warning(self, _show=True): - cfg_settings = ["config:install_tree:root", "config:locations:home", "config:locations:data", "SPACK_DATA_HOME", "SPACK_HOME"] + cfg_settings = ["config:install_tree:root"] return self._bypass_warning( - self.default_install_location, self.base.old_install_path, "installs", cfg_settings, _show=_show + self.default_install_location, + self.base.old_install_path, + "installs", + cfg_settings, + _show=_show, ) def bypassed_old_envs_warning(self, _show=True): - cfg_settings = ["config:environments_root", "config:locations:home", "config:locations:data", "SPACK_DATA_HOME", "SPACK_HOME"] + cfg_settings = ["config:environments_root"] return self._bypass_warning( - self.default_envs_path, self.base.old_envs_path, "environments", cfg_settings, _show=_show + self.default_envs_path, + self.base.old_envs_path, + "environments", + cfg_settings, + _show=_show, ) def bypassed_old_gpg_warning(self, _show=True): - cfg_settings = ["SPACK_GNUPGHOME", "config:locations:home", "config:locations:data", "SPACK_DATA_HOME", "SPACK_HOME"] - return self._bypass_warning(self.gpg_path, self.base.old_gpg_path, "GPG keys", cfg_settings, _show=_show) + cfg_settings = ["SPACK_GNUPGHOME"] + return self._bypass_warning( + self.gpg_path, self.base.old_gpg_path, "GPG keys", cfg_settings, _show=_show + ) - def _bypass_warning(self, chosen_path, old_path, data_category, cfg_settings, _show=True, ): + def _bypass_warning(self, chosen_path, old_path, data_category, cfg_settings, _show=True): + generic_cfg = [ + "config:locations:home", + "config:locations:data", + "SPACK_DATA_HOME", + "SPACK_HOME", + ] msg = ( f"Bypassing data for {data_category} existing in: {old_path}" f"\nIn favor of: {chosen_path}" "\nYou can explicitly designate a location (and suppress this warning)" f" by setting one of: {', '.join(cfg_settings)}" + f"\nOr use a catch-all setting: {', '.join(generic_cfg)}" ) if ( chosen_path != old_path From cd3874d616045368bd686c2ef04347a6c594ff7d Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 1 Apr 2026 15:59:48 -0700 Subject: [PATCH 383/506] minor tweak to warning Signed-off-by: Peter Scheibel --- lib/spack/spack/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 41412543e8601f..914fb07b15cde1 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -185,7 +185,7 @@ def _bypass_warning(self, chosen_path, old_path, data_category, cfg_settings, _s f"Bypassing data for {data_category} existing in: {old_path}" f"\nIn favor of: {chosen_path}" "\nYou can explicitly designate a location (and suppress this warning)" - f" by setting one of: {', '.join(cfg_settings)}" + f" by setting: {', '.join(cfg_settings)}" f"\nOr use a catch-all setting: {', '.join(generic_cfg)}" ) if ( From 71d03435fe0d289c7cd36ef57002d1043ea18b64 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 1 Apr 2026 16:32:11 -0700 Subject: [PATCH 384/506] doc reorg --- lib/spack/docs/configuration_basics.rst | 28 ++---------------- lib/spack/docs/index.rst | 1 + lib/spack/docs/where_spack_writes_data.rst | 33 ++++++++++++++++++++++ 3 files changed, 36 insertions(+), 26 deletions(-) create mode 100644 lib/spack/docs/where_spack_writes_data.rst diff --git a/lib/spack/docs/configuration_basics.rst b/lib/spack/docs/configuration_basics.rst index 611825ee30e6c0..0f3c48b24a1c3f 100644 --- a/lib/spack/docs/configuration_basics.rst +++ b/lib/spack/docs/configuration_basics.rst @@ -12,10 +12,9 @@ Configuration Basics ==================== -Spack is configured with YAML files. +Spack's behavior can be customized with YAML files that it searches for in specific locations, and also by setting environment variables. You can customize Spack's behavior by modifying these files, and you can also tell it to look for files in custom locations. -This section covers how configuration is generally organized and processed. -The configuration is organized into categories with predefined schemas, and individual sections cover each of these categories. +This section covers how the configuration system generally works (e.g. how precedence is established when two files configure different values for the same setting), and later sections cover specific configuration settings (e.g. how to control where installs are placed). Configuration Files ------------------- @@ -34,29 +33,6 @@ Here is a quick list of them, in case you want to skip directly to specific docs You can also add any of these as inline configuration in the YAML manifest file (``spack.yaml``) describing an :ref:`environment `. -Controlling where spack writes data ------------------------------------ - -A fresh checkout of spack will not write anything into the `$spack`` prefix; instead, all data is placed under the user's home directory. -You can control this in the following ways: - -* Redirect everything with environment variables: set ``SPACK_HOME`` and one of ``SPACK_USER_CONFIG_PATH`` or ``SPACK_DISABLE_LOCAL_CONFIG=1`` -* Or redirect everything with config: - - * set ``config:locations:home`` - * Update the ``user`` config scope with ``spack config --scope=spack edit include`` -* Or redirect installs, environments, and cached downloads (everything that takes up significant space) by setting ``SPACK_DATA_HOME`` -* Or use finer-grained configuration settings, for example: - - * ``config:install_tree:root`` to control where installs go - * ``config:build_stage`` to control where builds are staged - -For more on this, see: - -* :ref:`Variables controlling data location ` -* :ref:`include.yaml ` -* :ref:`config.yaml ` - YAML Format ----------- diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 320f1d4a62abea..b52f593b054300 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -73,6 +73,7 @@ If you're new to Spack and want to start using it, see :doc:`getting_started`, o :caption: Configuration configuration_basics + where_spack_writes_data config_yaml packages_yaml toolchains_yaml diff --git a/lib/spack/docs/where_spack_writes_data.rst b/lib/spack/docs/where_spack_writes_data.rst new file mode 100644 index 00000000000000..f40a9c56e6fe6e --- /dev/null +++ b/lib/spack/docs/where_spack_writes_data.rst @@ -0,0 +1,33 @@ +.. + Copyright Spack Project Developers. See COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. meta:: + :description lang=en: + Learn how to control where Spack generates files. + +.. _where_spack_writes_data: + +Controlling where spack writes data +=================================== + +A fresh checkout of spack will not write anything into the `$spack`` prefix; instead, all data is placed under the user's home directory. +You can control this in the following ways: + +* Redirect everything with environment variables: set ``SPACK_HOME`` and one of ``SPACK_USER_CONFIG_PATH`` or ``SPACK_DISABLE_LOCAL_CONFIG=1`` +* Or redirect everything with config: + + * set ``config:locations:home`` + * Update the ``user`` config scope with ``spack config --scope=spack edit include`` +* Or redirect installs, environments, and cached downloads (everything that takes up significant space) by setting ``SPACK_DATA_HOME`` +* Or use finer-grained configuration settings, for example: + + * ``config:install_tree:root`` to control where installs go + * ``config:build_stage`` to control where builds are staged + +For more on this, see: + +* :ref:`Variables controlling data location ` +* :ref:`include.yaml ` +* :ref:`config.yaml ` \ No newline at end of file From 486e39e6111b798d195bf7cc197993701787dd08 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 1 Apr 2026 17:57:14 -0700 Subject: [PATCH 385/506] docs typo Signed-off-by: Peter Scheibel --- lib/spack/docs/where_spack_writes_data.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/docs/where_spack_writes_data.rst b/lib/spack/docs/where_spack_writes_data.rst index f40a9c56e6fe6e..697bab386fc78b 100644 --- a/lib/spack/docs/where_spack_writes_data.rst +++ b/lib/spack/docs/where_spack_writes_data.rst @@ -12,7 +12,7 @@ Controlling where spack writes data =================================== -A fresh checkout of spack will not write anything into the `$spack`` prefix; instead, all data is placed under the user's home directory. +A fresh checkout of spack will not write anything into the ``$spack`` prefix; instead, all data is placed under the user's home directory. You can control this in the following ways: * Redirect everything with environment variables: set ``SPACK_HOME`` and one of ``SPACK_USER_CONFIG_PATH`` or ``SPACK_DISABLE_LOCAL_CONFIG=1`` From fa98d5eb605ff1d9c6d4a539724f39950621b03e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 2 Apr 2026 22:42:36 -0700 Subject: [PATCH 386/506] move set_home into conftest Signed-off-by: Peter Scheibel --- lib/spack/spack/test/conftest.py | 14 ++++++++++++++ lib/spack/spack/test/paths.py | 14 -------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index daaa1e2e37549e..3791e5eee2622b 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -177,6 +177,20 @@ def redirect_state_home(tmp_path: Path): spack.paths.locations.redirect_state_home(tmp_path) +@pytest.fixture +def set_home(working_env): + def _set_home(val): + # Clear some env vars that can interfere w/ expanduser(~) on Windows + os.environ.pop("USERPROFILE", None) + os.environ.pop("HOMEDRIVE", None) + os.environ["HOMEPATH"] = val + + # For expanduser on Linux + os.environ["HOME"] = val + + yield _set_home + + @pytest.fixture def mock_git_version_info(git, tmp_path: Path, redirect_user_repos_cache_path): """Create a mock git repo with known structure diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 35bc33eb172db2..d238b761381d52 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -25,20 +25,6 @@ def clear_env_vars(working_env): spack.paths._unset_path_vars(os.environ) -@pytest.fixture -def set_home(working_env): - def _set_home(val): - # Clear some env vars that can interfere w/ expanduser(~) on Windows - os.environ.pop("USERPROFILE", None) - os.environ.pop("HOMEDRIVE", None) - os.environ["HOMEPATH"] = val - - # For expanduser on Linux - os.environ["HOME"] = val - - yield _set_home - - def test_install_location(working_env, tmp_path, mutable_config, set_home): # If prior default install dir inside spack prefix does not # exist, place installs in $HOME From f7be065ef1c1eba9f98f769549bb69d078c3184e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 2 Apr 2026 13:26:36 -0700 Subject: [PATCH 387/506] isolate cmd (and test) Signed-off-by: Peter Scheibel --- lib/spack/spack/cmd/isolate.py | 161 +++++++++++++++++++++++++++ lib/spack/spack/test/cmd/isolate.py | 164 ++++++++++++++++++++++++++++ share/spack/spack-completion.bash | 11 +- share/spack/spack-completion.fish | 13 +++ 4 files changed, 348 insertions(+), 1 deletion(-) create mode 100644 lib/spack/spack/cmd/isolate.py create mode 100644 lib/spack/spack/test/cmd/isolate.py diff --git a/lib/spack/spack/cmd/isolate.py b/lib/spack/spack/cmd/isolate.py new file mode 100644 index 00000000000000..fcdf290ad36d98 --- /dev/null +++ b/lib/spack/spack/cmd/isolate.py @@ -0,0 +1,161 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import argparse +from pathlib import Path + +import spack +import spack.config +import spack.llnl.util.tty as tty +import spack.paths_base +import spack.util.path +import spack.util.spack_yaml as syaml +from spack.cmd.common import arguments + +description = "force spack to only use/write data from/to an isolated prefix" +section = "config" +level = "long" + + +def setup_parser(subparser: argparse.ArgumentParser) -> None: + subparser.add_argument( + "--force-scopes", action="store_true", help="remove all config scopes in ~" + ) + subparser.add_argument( + "--force-home", action="store_true", help="override home even if user setting is detected" + ) + subparser.add_argument("--force-all", action="store_true", help="set all 'force' options") + # TODO: add an --undo option: delete config:locations:home and restore + # default config scopes in the include: section of the spack scope + arguments.add_common_arguments(subparser, ["specs"]) + + +class Force: + + SCOPES = 1 + HOME = 2 + + options = {SCOPES: "--force-scopes", HOME: "--force-home"} + + def __init__(self, args): + self.all = args.force_all + self.active = {Force.SCOPES: args.force_scopes, Force.HOME: args.force_home} + + def go_ahead(self, category, context): + do_it = self.all or self.active[category] + if do_it: + tty.debug(f"{context}\nOverriding because{Force.options[category]} is set") + else: + tty.warn(f"{context}\nYou can override with {Force.options[category]}") + return do_it + + +def isolate(parser, args): + # TODO: warn if installed specs are detected? If we move config:locations:home + # then these will be "lost" + + force = Force(args) + + spack_root = Path(spack.paths_base.locations.prefix) + + def is_in_spack_prefix(path): + resolved = Path(spack.util.path.canonicalize_path(path)).resolve() + return path == spack_root or spack_root in resolved.parents + + def same_path(x, y): + return Path(x).resolve(strict=False) == Path(y).resolve(strict=False) + + current_install_root = spack.config.get("config:install_tree:root") + current_env_root = spack.config.get("config:environments_root") + + if current_install_root and current_install_root != "$default_install_root": + tty.warn( + f"config:install_tree:root is set to {current_install_root}," + " the install tree will not be relocated because this setting" + " has precedence." + ) + + if current_env_root and current_env_root != "$default_envs_root": + tty.warn( + f"config:environments_root is set to {current_env_root}, " + " environments will not be relocated because this setting" + " has precedence." + ) + + current_home = spack.config.get("config:locations:home") + change_home = True + if current_home: + if is_in_spack_prefix(current_home): + change_home = False + tty.debug(f"config:locations:home is inside $spack: {current_home}\nkeeping it") + else: + msg = f"config:locations:home is outside of $spack: {current_home}" + if not force.go_ahead(Force.HOME, msg): + # By default for a new clone of Spack, config:locations:home is not + # set. So if we're here, it implies we will be changing something + # that was set by the user. + change_home = False + + # else: home is unset (typical) - we will set it to relocate everything + # except for config (and then redirect config) + + if change_home: + config_spack = spack.config.get("config", scope="spack") + locations = config_spack.setdefault("locations", {}) + locations["home"] = "$spack/all-data" + + include_spack = spack.config.get("include", scope="spack") or [] + + # Take included scopes defined in the "spack" scope and remove the + # user/system scopes. Look for other scopes defined here that exist + # outside the spack prefix; --force will remove those scopes but + # otherwise this command will just print a warning + visited = set() + new_include_spack = syaml.syaml_list() + for scope_def in include_spack: + scope_name = scope_def.get("name", None) + path = scope_def.get("path") + if scope_name: + scope_id = f"{scope_name} - {path}" + else: + scope_id = f"(unnamed) - {path}" + + if scope_name == "user": + path = spack.util.path.canonicalize_path(path) + if same_path(path, spack.util.path.canonicalize_path("~/.config/spack")): + tty.debug("Removing default user scope") + elif not force.go_ahead(Force.SCOPES, "user scope is not in default path"): + new_include_spack.append(scope_def) + visited.add("user") + elif scope_name == "system": + if same_path(path, "/etc/spack"): + tty.debug("Removing default system scope") + elif not force.go_ahead(Force.SCOPES, "system scope is not in default path"): + new_include_spack.append(scope_def) + visited.add("system") + else: + if not is_in_spack_prefix(path): + if not force.go_ahead( + Force.SCOPES, f"User-defined scope is not in spack prefix: {scope_id}" + ): + new_include_spack.append(scope_def) + else: + new_include_spack.append(scope_def) + + for expect_remove in ["user", "system"]: + if expect_remove not in visited: + tty.warn( + f"Expected to find (and remove): {expect_remove}" + "\nbut it wasn't present (suggests prior run of `spack isolate`, or)" + " manual editing)" + ) + + if change_home: + spack.config.set("config", config_spack, scope="spack") + # TODO: the "spack" config scope is version-controlled in the + # spack repo, so this change can be dropped or committed by + # accident (e.g. if the user runs `git checkout` or + # `git commit -a` without being aware of the effects of this + # command. + spack.config.set("include", new_include_spack, scope="spack") diff --git a/lib/spack/spack/test/cmd/isolate.py b/lib/spack/spack/test/cmd/isolate.py new file mode 100644 index 00000000000000..f9f416e0e8448d --- /dev/null +++ b/lib/spack/spack/test/cmd/isolate.py @@ -0,0 +1,164 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import shutil +from pathlib import Path + +import pytest + +import spack.config +import spack.paths_base +import spack.util.spack_yaml as syaml +from spack.main import SpackCommand +from spack.paths import SpackPaths +from spack.paths import locations as paths +from spack.paths_base import SpackPathsBase + +isolate = SpackCommand("isolate") + + +@pytest.fixture() +def mock_spack_scope(tmp_path_factory, mutable_config): + higher_scope_dir = tmp_path_factory.mktemp("higher") + spack_scope_dir = tmp_path_factory.mktemp("test-spack-scope") + lower_scope_dir = tmp_path_factory.mktemp("higher") + + # Copy the real spack scope includes into a test location, so that + # we can check that this command modifies it. + src_file = Path(paths.prefix) / "etc" / "spack" / "include.yaml" + cpy_dst = str(Path(spack_scope_dir) / "include.yaml") + shutil.copyfile(src_file, cpy_dst) + + scopes = [ + spack.config.DirectoryConfigScope("lower", str(lower_scope_dir)), + spack.config.DirectoryConfigScope("spack", str(spack_scope_dir)), + spack.config.DirectoryConfigScope("higher", str(higher_scope_dir)), + ] + + with spack.config.use_configuration(*scopes) as config: + yield config + + +def includes_as_a_dict(): + includes = spack.config.get("include") + return dict((scope["name"], scope) for scope in includes) + + +def test_isolate_removes_scopes(mock_spack_scope): + # In default case, spack has two scopes that point to locations + # outside of spack. `spack isolate` should remove both if they + # haven't been modified. + + assert not spack.config.get("config:locations:home") + initial_cfg = includes_as_a_dict() + assert "user" in initial_cfg + assert "system" in initial_cfg + + spack.config.get("include") + output = isolate() + + # Warnings would be printed if user modifications are detected. + # In this test we haven't made any, so there shouldn't be any + # warnings. + assert "Warning:" not in output + + after_cfg = includes_as_a_dict() + assert "user" not in after_cfg + assert "system" not in after_cfg + + +def test_isolate_retains_scope_when_not_default(mock_spack_scope): + cfg_as_dict = includes_as_a_dict() + non_default_path = "/something/else/not/the/default" + cfg_as_dict["user"]["path"] = non_default_path + + spack.config.set("include", syaml.syaml_list(cfg_as_dict.values()), scope="spack") + isolate() + + after_cfg = includes_as_a_dict() + assert "system" not in after_cfg + assert after_cfg["user"]["path"] == non_default_path + + +@pytest.fixture +def redirect_base_paths(tmp_path_factory, monkeypatch): + base_prefix = str(tmp_path_factory.mktemp("base-test")) + pb = SpackPathsBase(base_prefix) + # spack.util.path.canonicalize_path uses paths_base.locations + # so this should generally be monkeypatched when testing anything + # that modifies path vars + monkeypatch.setattr(spack.paths_base, "locations", pb) + yield pb + + +def test_isolate_redirect_installs( + mock_spack_scope, set_home, tmp_path_factory, redirect_base_paths +): + # in the default case where a user clones a new spack instance, it will + # write into ~. Check that `spack isolate` redirects installs (for example) + # into $spack + base = redirect_base_paths + home_prefix = str(tmp_path_factory.mktemp("home-test")) + set_home(home_prefix) + + p1 = SpackPaths(base) + assert p1.default_install_location == str( + Path(home_prefix) / ".local" / "share" / "spack" / "installs" + ) + + isolate() + + p2 = SpackPaths(base) + assert p2.default_install_location == str( + Path(p2.base.prefix) / "all-data" / ".local" / "share" / "spack" / "installs" + ) + + +def test_isolate_cfg_points_outside_spack( + mock_spack_scope, tmp_path_factory, mutable_config, redirect_base_paths +): + # check that `spack isolate` will rewrite config:locations:home + # when it points outside of $spack + base = redirect_base_paths + dir_outside_spack = str(tmp_path_factory.mktemp("outside-dir")) + + spack.config.set("config:locations", {"home": dir_outside_spack}, scope="spack") + + initial_expected = str(Path(dir_outside_spack) / ".local" / "share" / "spack" / "installs") + p1 = SpackPaths(base) + assert p1.default_install_location == initial_expected + + # First try to isolate without --force-... - nothing should happen + cmd_output = isolate() + assert "config:locations:home is outside of $spack" in cmd_output + assert "You can override with --force-home" in cmd_output + p2 = SpackPaths(base) + assert p2.default_install_location == initial_expected + + isolate("--force-home") + p3 = SpackPaths(base) + assert p3.default_install_location == str( + Path(base.prefix) / "all-data" / ".local" / "share" / "spack" / "installs" + ) + + +def test_isolate_unaffected_by_higher_scope( + mock_spack_scope, tmp_path_factory, mutable_config, redirect_base_paths +): + # set config:locations:home in a higher-precedence scope: make sure + # that `spack isolate` does not interfere + base = redirect_base_paths + dir_outside_spack = str(tmp_path_factory.mktemp("outside-dir")) + + spack.config.set("config", {"locations": {"home": dir_outside_spack}}, scope="higher") + + p1 = SpackPaths(base) + expected_path = str(Path(dir_outside_spack) / ".local" / "share" / "spack" / "installs") + assert p1.default_install_location == expected_path + + isolate() + + p2 = SpackPaths(base) + # `spack isolate` does not modify scopes other than the 'spack' scope + assert p2.default_install_location == expected_path diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 39a7c8c42d1c65..c0306edd88e94b 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -396,7 +396,7 @@ _spack() { then SPACK_COMPREPLY="--color -v --verbose -k --insecure -b --bootstrap -V --version -h --help -H --all-help -c --config -C --config-scope -e --env -D --env-dir -E --no-env --use-env-repo -d --debug -t --backtrace --pdb --timestamp -m --mock --print-shell-vars --stacktrace --warn-writes-into-spack -l --enable-locks -L --disable-locks -p --profile --profile-file --sorted-profile --lines" else - SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse logs maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" + SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install isolate license list load location log-parse logs maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" fi } @@ -1368,6 +1368,15 @@ _spack_install() { fi } +_spack_isolate() { + if $list_options + then + SPACK_COMPREPLY="-h --help --force-scopes --force-home --force-all" + else + _all_packages + fi +} + _spack_license() { if $list_options then diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 2a192a0cdb0b18..43d26c55de814e 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -389,6 +389,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a graph -d 'generat complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a help -d 'get help on spack and its commands' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a info -d 'get detailed information on a particular package' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a install -d 'build and install packages' +complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a isolate -d 'force spack to only use/write data from/to an isolated prefix' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a license -d 'list and check license headers on files in spack' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a list -d 'list and search available packages' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a load -d 'add package to the user environment' @@ -2201,6 +2202,18 @@ complete -c spack -n '__fish_spack_using_command install' -l fresh-roots -l reus complete -c spack -n '__fish_spack_using_command install' -l deprecated -f -a config_deprecated complete -c spack -n '__fish_spack_using_command install' -l deprecated -d 'allow concretizer to select deprecated versions' +# spack isolate +set -g __fish_spack_optspecs_spack_isolate h/help force-scopes force-home force-all +complete -c spack -n '__fish_spack_using_command_pos_remainder 0 isolate' -f -k -a '(__fish_spack_specs)' +complete -c spack -n '__fish_spack_using_command isolate' -s h -l help -f -a help +complete -c spack -n '__fish_spack_using_command isolate' -s h -l help -d 'show this help message and exit' +complete -c spack -n '__fish_spack_using_command isolate' -l force-scopes -f -a force_scopes +complete -c spack -n '__fish_spack_using_command isolate' -l force-scopes -d 'remove all config scopes in ~' +complete -c spack -n '__fish_spack_using_command isolate' -l force-home -f -a force_home +complete -c spack -n '__fish_spack_using_command isolate' -l force-home -d 'override home even if user setting is detected' +complete -c spack -n '__fish_spack_using_command isolate' -l force-all -f -a force_all +complete -c spack -n '__fish_spack_using_command isolate' -l force-all -d 'set all '"'"'force'"'"' options' + # spack license set -g __fish_spack_optspecs_spack_license h/help root= complete -c spack -n '__fish_spack_using_command_pos 0 license' -f -a list-files -d 'list files in spack that should have license headers' From dd3cad2237f32e89d3e3a7ac7f1c5d0a400de55f Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 3 Apr 2026 16:09:12 -0700 Subject: [PATCH 388/506] default resolution of config var when config is missing should match default config Signed-off-by: Peter Scheibel --- lib/spack/spack/caches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/caches.py b/lib/spack/spack/caches.py index a7b8333e526817..33b214560961c0 100644 --- a/lib/spack/spack/caches.py +++ b/lib/spack/spack/caches.py @@ -19,7 +19,7 @@ def misc_cache_location(): Currently the ``MISC_CACHE`` stores indexes for virtual dependency providers and for which packages provide which tags. """ - path = spack.config.get("config:misc_cache", "$state_home/cache") + path = spack.config.get("config:misc_cache", "$state_home/$spack_instance_id/cache") return spack.util.path.canonicalize_path(path) From 9695c86599456bd7f30bbb5fc5cd29b80c3b42cc Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 09:34:14 -0700 Subject: [PATCH 389/506] use spack query function vs. hardcoding value Signed-off-by: Peter Scheibel --- .github/workflows/bootstrap.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index 300aa1d1a5577d..6731e2b74295c1 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -48,7 +48,7 @@ jobs: spack bootstrap disable github-actions-v2 spack bootstrap disable github-actions-v0.6 spack solve zlib - tree ~/.local/state/spack/bootstrap/store + tree $(spack bootstrap root) clingo-sources: if: github.repository == 'spack/spack' @@ -75,7 +75,7 @@ jobs: spack bootstrap disable github-actions-v0.6 export PATH="$(brew --prefix bison)/bin:$(brew --prefix cmake)/bin:$PATH" spack solve zlib - tree ~/.local/state/spack/bootstrap/store + tree $(spack bootstrap root) gnupg-sources: if: github.repository == 'spack/spack' @@ -104,7 +104,7 @@ jobs: spack bootstrap disable github-actions-v2 spack bootstrap disable github-actions-v0.6 spack gpg list - tree ~/.local/state/spack/bootstrap/store + tree $(spack bootstrap root) from-binaries: if: github.repository == 'spack/spack' @@ -152,13 +152,13 @@ jobs: fi spack solve zlib done - tree ~/.local/state/spack/bootstrap/store + tree $(spack bootstrap root) - name: Bootstrap GnuPG run: | . share/spack/setup-env.sh spack config add config:installer:new spack gpg list - tree ~/.local/state/spack/bootstrap/store + tree $(spack bootstrap root) windows: if: github.repository == 'spack/spack' @@ -182,10 +182,10 @@ jobs: spack bootstrap disable github-actions-v0.6 spack -d solve zlib ./share/spack/qa/validate_last_exit.ps1 - tree $env:userprofile/.local/state/spack/bootstrap/store + tree $(spack bootstrap root) - name: Bootstrap GnuPG run: | ./share/spack/setup-env.ps1 spack -d gpg list ./share/spack/qa/validate_last_exit.ps1 - tree $env:userprofile/.local/state/spack/bootstrap/store + tree $(spack bootstrap root) From 1e3a67df707e8a342ca267c584c130e709246f70 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 09:37:39 -0700 Subject: [PATCH 390/506] rm outdated line (was outdated before this PR) Signed-off-by: Peter Scheibel --- lib/spack/docs/developer_guide.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index 0391311a8f58a8..f5d5a778d95540 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -66,7 +66,6 @@ So that you can familiarize yourself with the project, we will start with a high etc/ spack/ <- Spack config files. - Can be overridden by files in ~/.config/spack. var/ spack/ From 422f86b0e090278485238c167983b3594e528de0 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 09:39:11 -0700 Subject: [PATCH 391/506] add reference to XDG Signed-off-by: Peter Scheibel --- lib/spack/docs/where_spack_writes_data.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/spack/docs/where_spack_writes_data.rst b/lib/spack/docs/where_spack_writes_data.rst index 697bab386fc78b..982002ef8e3c53 100644 --- a/lib/spack/docs/where_spack_writes_data.rst +++ b/lib/spack/docs/where_spack_writes_data.rst @@ -26,6 +26,8 @@ You can control this in the following ways: * ``config:install_tree:root`` to control where installs go * ``config:build_stage`` to control where builds are staged +In the absence of any Spack-specific settings, Spack will respect [XDG](https://specifications.freedesktop.org/basedir/latest/) environment variables controlling the home directory for specific types of data. + For more on this, see: * :ref:`Variables controlling data location ` From 31cd1c10d4839e2f605a50a996f4e4b259bbeeac Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 09:45:29 -0700 Subject: [PATCH 392/506] use state_home over user_cache_path Signed-off-by: Peter Scheibel --- lib/spack/spack/solver/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/solver/core.py b/lib/spack/spack/solver/core.py index cacd8d3808e9b5..8c6bb24f31ae3e 100644 --- a/lib/spack/spack/solver/core.py +++ b/lib/spack/spack/solver/core.py @@ -155,7 +155,7 @@ def _ensure_clingo_or_raise(clingo_mod: ModuleType) -> None: if ( pathlib.Path( sup.canonicalize_path( - spack.config.CONFIG.get("bootstrap:root", "$user_cache_path/bootstrap") + spack.config.CONFIG.get("bootstrap:root", "$state_home/bootstrap") ) ) in pathlib.Path(clingo_mod.__file__).parents From e7b1653d88109dafaa538b03b864dd2bc321d381 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 09:50:13 -0700 Subject: [PATCH 393/506] avoid mutable_config fixture for test that does not modify config Signed-off-by: Peter Scheibel --- lib/spack/spack/test/cmd/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index 24f725de4270cb..36f7f5089cffe0 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -270,7 +270,7 @@ def test_update_completion_arg(shell, tmp_path: pathlib.Path, monkeypatch): # Note: this test is never expected to be supported on Windows @pytest.mark.not_on_windows("Shell completion script generator fails on windows") @pytest.mark.parametrize("shell", ["bash", "fish"]) -def test_updated_completion_scripts(shell, tmp_path: pathlib.Path, mutable_config): +def test_updated_completion_scripts(shell, tmp_path: pathlib.Path, config): """Make sure our shell tab completion scripts remain up-to-date.""" width = 72 From fb4936d5430f8110dcf0a1aec4562c556d6631aa Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 10:00:13 -0700 Subject: [PATCH 394/506] better wording Signed-off-by: Peter Scheibel --- lib/spack/docs/configuration_basics.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/docs/configuration_basics.rst b/lib/spack/docs/configuration_basics.rst index 0f3c48b24a1c3f..6b3fed06d090ac 100644 --- a/lib/spack/docs/configuration_basics.rst +++ b/lib/spack/docs/configuration_basics.rst @@ -564,9 +564,9 @@ Each of these variables can be set with config or with environment variables. For example ``$data_home`` evaluates to one of the following (highest-priority first): #. ``SPACK_DATA_HOME`` env var if that is set -#. Under ``SPACK_HOME`` env var; for ``$data_home``, it is this plus ``.local/share/spack`` +#. Under ``SPACK_HOME`` env var; for ``$data_home``, it is ``$SPACK_HOME/.local/share/spack`` #. ``config:locations:data`` -#. Under ``config:locations:home``; for ``$data_home`` it is this plus ``.local/share/spack`` +#. Under ``config:locations:home``; for ``$data_home`` it is ``$spack_home/.local/share/spack`` #. ``XDG_DATA_HOME/spack`` if XDG_DATA_HOME is set #. Under the default for ``XDG_DATA_HOME``: ``~/.local/share/spack`` From 0e0ca1cf7ed5ceee89d896660bbc97dd21228daf Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 10:11:02 -0700 Subject: [PATCH 395/506] path is 'environments' now, not 'envs' Signed-off-by: Peter Scheibel --- lib/spack/docs/environments.rst | 8 ++++---- lib/spack/spack/paths.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index 507cd9d5801196..9fc6401021a370 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -63,14 +63,14 @@ An environment is created by: $ spack env create myenv -The directory ``$data_home/envs/myenv`` is created to manage the environment. +The directory ``$data_home/environments/myenv`` is created to manage the environment. .. note:: - By default, all managed environments are stored in the ``$data_home/envs`` folder. + By default, all managed environments are stored in the ``$data_home/environments`` folder. This location can be changed by setting the ``environments_root`` variable in ``config.yaml``. -Spack creates the file ``spack.yaml``, hidden directory ``.spack-env``, and ``spack.lock`` file under ``$data_home/envs/myenv``. +Spack creates the file ``spack.yaml``, hidden directory ``.spack-env``, and ``spack.lock`` file under ``$data_home/environments/myenv``. User interaction occurs through the ``spack.yaml`` file and the Spack commands that affect it. Metadata and, by default, the view are stored in the ``.spack-env`` directory. When the environment is concretized, Spack creates the ``spack.lock`` file with the fully configured specs and dependencies for the environment. @@ -240,7 +240,7 @@ The same rule applies to the ``install`` and ``uninstall`` commands. $ spack install zlib@1.2.8 [+] yfc7epf zlib@1.2.8 ~/spack/opt/spack/linux-rhel7-broadwell/gcc-8.1.0/zlib-1.2.8-yfc7epf57nsfn2gn4notccaiyxha6z7x (12s) - ==> Updating view at ~/.local/share/spack/envs/myenv/.spack-env/view + ==> Updating view at ~/.local/share/spack/environments/myenv/.spack-env/view $ spack find ==> In environment myenv diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 914fb07b15cde1..66048eb500690d 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -202,8 +202,8 @@ def _bypass_warning(self, chosen_path, old_path, data_category, cfg_settings, _s def default_envs_path(self): return self._decide_old_or_new_location( self.base.old_envs_path, - os.path.join(self.data_home, "envs"), - os.path.join(self.default_data_home, "envs"), + os.path.join(self.data_home, "environments"), + os.path.join(self.default_data_home, "environments"), self._data_home_provenance, ) From 39432be77cfa732eded3b04155bfcacf9efc12db Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 10:32:14 -0700 Subject: [PATCH 396/506] rm vestigial code (was introduced and later made so in this PR) Signed-off-by: Peter Scheibel --- lib/spack/spack/util/path.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index cea110b4885034..c9f1856e9cc3f9 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -52,21 +52,6 @@ def get_user(): NOMATCH = object() -class ResolutionContextError(ValueError): - def __init__(self, var): - msg = f"Cannot resolve {var}" - super().__init__(msg) - self.var = var - - -class CannotResolve: - def __init__(self, var): - self.var = var - - def __call__(self): - raise ResolutionContextError(self.var) - - # Substitutions to perform def replacements(): # break circular imports From ae86e231778e7d9d749e19729d43608593876418 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 10:37:54 -0700 Subject: [PATCH 397/506] add reference in docs Signed-off-by: Peter Scheibel --- lib/spack/docs/config_yaml.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index faf77bba25ac96..e3600f07533cb6 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -129,7 +129,7 @@ The ``locations`` section includes variables that control where spack stores dat * ``locations:cache`` * ``locations:home`` -The additional flag setting ``locations:disable_env`` will prevent environment variables from influencing these locations. +The additional flag setting ``locations:disable_env`` will prevent the environment variables described in :ref:`config-file-data-variables` from influencing these locations. ``verify_ssl`` -------------------- From 9088b3b4aa1029d4da3a91c9032c4c9b91da736b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 10:49:54 -0700 Subject: [PATCH 398/506] better method name Signed-off-by: Peter Scheibel --- lib/spack/spack/environment/environment.py | 2 +- lib/spack/spack/paths.py | 14 +++++++------- lib/spack/spack/store.py | 2 +- lib/spack/spack/test/paths.py | 6 +++--- lib/spack/spack/util/gpg.py | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index b8d25bbca78232..c93d110a03228a 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -102,7 +102,7 @@ def default_env_path(): global _default_env_path if not _default_env_path: _default_env_path = paths.default_envs_path - paths.bypassed_old_envs_warning() + paths.warn_unused_old_envs() return _default_env_path diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 66048eb500690d..e70b51e4b3158e 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -148,9 +148,9 @@ def default_install_location(self): self._data_home_provenance, ) - def bypassed_old_installs_warning(self, _show=True): + def warn_unused_old_installs(self, _show=True): cfg_settings = ["config:install_tree:root"] - return self._bypass_warning( + return self._warn_on_unused_old_data( self.default_install_location, self.base.old_install_path, "installs", @@ -158,9 +158,9 @@ def bypassed_old_installs_warning(self, _show=True): _show=_show, ) - def bypassed_old_envs_warning(self, _show=True): + def warn_unused_old_envs(self, _show=True): cfg_settings = ["config:environments_root"] - return self._bypass_warning( + return self._warn_on_unused_old_data( self.default_envs_path, self.base.old_envs_path, "environments", @@ -168,13 +168,13 @@ def bypassed_old_envs_warning(self, _show=True): _show=_show, ) - def bypassed_old_gpg_warning(self, _show=True): + def warn_unused_old_gpg(self, _show=True): cfg_settings = ["SPACK_GNUPGHOME"] - return self._bypass_warning( + return self._warn_on_unused_old_data( self.gpg_path, self.base.old_gpg_path, "GPG keys", cfg_settings, _show=_show ) - def _bypass_warning(self, chosen_path, old_path, data_category, cfg_settings, _show=True): + def _warn_on_unused_old_data(self, chosen_path, old_path, data_category, cfg_settings, _show=True): generic_cfg = [ "config:locations:home", "config:locations:data", diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 5b96ad26d556cf..7a58c6ddf615cf 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -78,7 +78,7 @@ def parse_install_tree(config_dict: dict) -> Tuple[str, str, Dict[str, str]]: unpadded_root = install_tree.get("root", None) if not unpadded_root: unpadded_root = paths.default_install_location - paths.bypassed_old_installs_warning() + paths.warn_unused_old_installs() unpadded_root = spack.util.path.canonicalize_path(unpadded_root) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index d238b761381d52..b72e5c810bb014 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -84,7 +84,7 @@ def paths_base_nonempty_old_install(): ) (pathlib.Path(new_default_installs_dir) / "afile").touch() assert p1.default_install_location == new_default_installs_dir - warning_msg1 = p1.bypassed_old_installs_warning(_show=False) + warning_msg1 = p1.warn_unused_old_installs(_show=False) assert f"Bypassing data for installs existing in: {p1.base.old_install_path}" in warning_msg1 assert f"In favor of: {new_default_installs_dir}" in warning_msg1 @@ -98,7 +98,7 @@ def paths_base_nonempty_old_install(): p4 = SpackPaths(paths_base_nonempty_old_install()) xdg_installs_location = _ensure_dir(pathlib.Path(xdg_data_home) / "spack" / "installs") assert p4.default_install_location == str(xdg_installs_location) - warning_msg4 = p4.bypassed_old_installs_warning(_show=False) + warning_msg4 = p4.warn_unused_old_installs(_show=False) assert f"In favor of: {xdg_installs_location}" in warning_msg4 # (sanity) XDG_DATA_HOME still overrides when there is something in it @@ -117,7 +117,7 @@ def _unconditional_path_override_checks(tmp_path, base_paths_generator): assert p2.default_install_location == str( pathlib.Path(spack_home_cfg_prefix) / ".local" / "share" / "spack" / "installs" ) - assert not p2.bypassed_old_installs_warning(_show=False) + assert not p2.warn_unused_old_installs(_show=False) # "config:locations:data" overrides the above spack_data_prefix = _ensure_dir(tmp_path / "spack-data") diff --git a/lib/spack/spack/util/gpg.py b/lib/spack/spack/util/gpg.py index cf0425fd0e6af8..4db19cc0437e76 100644 --- a/lib/spack/spack/util/gpg.py +++ b/lib/spack/spack/util/gpg.py @@ -60,7 +60,7 @@ def init(gnupghome=None, force=False): GNUPGHOME = gnupghome or os.getenv("SPACK_GNUPGHOME") if not GNUPGHOME: GNUPGHOME = paths.gpg_path - paths.bypassed_old_gpg_warning() + paths.warn_unused_old_gpg() # Set the executable objects for "gpg" and "gpgconf" with spack.bootstrap.ensure_bootstrap_configuration(): From 13a0e0e75af16c4c7ee460a5071eb670127264c1 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 11:07:46 -0700 Subject: [PATCH 399/506] replace all read access of _data_home_provenance with getter to avoid ordering issues Signed-off-by: Peter Scheibel --- lib/spack/spack/paths.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index e70b51e4b3158e..a7ba6227839d98 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -98,7 +98,7 @@ def __init__(self, base): @property def state_home(self): if not self._state_home: - self._state_home, self._state_home_provenance = self.resolve_a_home( + self._state_home, _ = self.resolve_a_home( ["SPACK_STATE_HOME", "SPACK_USER_CACHE_PATH"], "state", SpackPaths.relative_state_home, @@ -109,7 +109,7 @@ def state_home(self): @property def cache_home(self): if not self._cache_home: - self._cache_home, self._cache_home_provenance = self.resolve_a_home( + self._cache_home, _ = self.resolve_a_home( "SPACK_CACHE_HOME", "cache", SpackPaths.relative_cache_home, "XDG_CACHE_HOME" ) return self._cache_home @@ -121,6 +121,14 @@ def data_home(self): "SPACK_DATA_HOME", "data", SpackPaths.relative_data_home, "XDG_DATA_HOME" ) return self._data_home + + @property + def data_home_provenance(self): + if not self._data_home_provenance: + self._data_home, self._data_home_provenance = self.resolve_a_home( + "SPACK_DATA_HOME", "data", SpackPaths.relative_data_home, "XDG_DATA_HOME" + ) + return self._data_home_provenance @property def spack_home(self): @@ -145,7 +153,7 @@ def default_install_location(self): self.base.old_install_path, os.path.join(self.data_home, "installs"), os.path.join(self.default_data_home, "installs"), - self._data_home_provenance, + self.data_home_provenance, ) def warn_unused_old_installs(self, _show=True): @@ -191,7 +199,7 @@ def _warn_on_unused_old_data(self, chosen_path, old_path, data_category, cfg_set if ( chosen_path != old_path and dir_is_occupied(old_path) - and not self._data_home_provenance.unilateral_override() + and not self.data_home_provenance.unilateral_override() ): if _show: tty.warn(msg) @@ -204,7 +212,7 @@ def default_envs_path(self): self.base.old_envs_path, os.path.join(self.data_home, "environments"), os.path.join(self.default_data_home, "environments"), - self._data_home_provenance, + self.data_home_provenance, ) @property @@ -250,7 +258,7 @@ def gpg_path(self): self.base.old_gpg_path, os.path.join(self.data_home, "gpg"), os.path.join(self.default_data_home, "gpg"), - self._data_home_provenance, + self.data_home_provenance, ) @property @@ -259,7 +267,7 @@ def gpg_keys_path(self): self.base.old_gpg_keys_path, os.path.join(self.data_home, "gpg-keys"), os.path.join(self.default_data_home, "gpg-keys"), - self._data_home_provenance, + self.data_home_provenance, ) def __getattr__(self, name): From ca10a888b2b23ec63bca4787864731cfda7c465b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 11:09:05 -0700 Subject: [PATCH 400/506] style edit Signed-off-by: Peter Scheibel --- lib/spack/spack/paths.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index a7ba6227839d98..109ae7b2b17404 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -121,7 +121,7 @@ def data_home(self): "SPACK_DATA_HOME", "data", SpackPaths.relative_data_home, "XDG_DATA_HOME" ) return self._data_home - + @property def data_home_provenance(self): if not self._data_home_provenance: @@ -182,7 +182,9 @@ def warn_unused_old_gpg(self, _show=True): self.gpg_path, self.base.old_gpg_path, "GPG keys", cfg_settings, _show=_show ) - def _warn_on_unused_old_data(self, chosen_path, old_path, data_category, cfg_settings, _show=True): + def _warn_on_unused_old_data( + self, chosen_path, old_path, data_category, cfg_settings, _show=True + ): generic_cfg = [ "config:locations:home", "config:locations:data", From 8f15ac0f79ca362f995cd4423fe79eda2a388da0 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 11:16:14 -0700 Subject: [PATCH 401/506] rm unused object init attrs Signed-off-by: Peter Scheibel --- lib/spack/spack/paths.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 109ae7b2b17404..49d3c837868e53 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -86,13 +86,8 @@ def __init__(self, base): self._data_home = None self._cache_home = None - self.default_state_home, self.default_data_home, self.default_cache_home = ( - os.path.join(os.path.expanduser("~"), x, "spack") - for x in [ - SpackPaths.relative_state_home, - SpackPaths.relative_data_home, - SpackPaths.relative_cache_home, - ] + self.default_data_home = os.path.join( + os.path.expanduser("~"), SpackPaths.relative_data_home, "spack" ) @property From 51d7400de4882a056bf276e638d7d0d4fd6d1eb0 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 11:34:05 -0700 Subject: [PATCH 402/506] relocate modules always and not conditionally (i.e. dont use modules in old location if they are detected) Signed-off-by: Peter Scheibel --- etc/spack/defaults/base/modules.yaml | 5 ++--- lib/spack/spack/config.py | 1 - lib/spack/spack/modules/common.py | 2 +- lib/spack/spack/paths.py | 22 ---------------------- lib/spack/spack/util/path.py | 1 - 5 files changed, 3 insertions(+), 28 deletions(-) diff --git a/etc/spack/defaults/base/modules.yaml b/etc/spack/defaults/base/modules.yaml index 5834f31c98841f..c43dcee865abde 100644 --- a/etc/spack/defaults/base/modules.yaml +++ b/etc/spack/defaults/base/modules.yaml @@ -32,10 +32,9 @@ modules: # These are configurations for the module set named "default" default: # Where to install modules - # If set to a non-default value, this setting has precedence. roots: - tcl: $modules_base/modules - lmod: $modules_base/lmod + tcl: $data_home/modules + lmod: $data_home/lmod # What type of modules to use ("tcl" and/or "lmod") enable: [] diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 96e5aa83c68b2f..3e77be4b516cd6 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -144,7 +144,6 @@ "cache_home", "data_home", "spack_home", - "modules_base", "default_envs_root", "default_install_root", "user_cache_path", diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py index c4ad62dd1a432f..7bbd4c96b452fc 100644 --- a/lib/spack/spack/modules/common.py +++ b/lib/spack/spack/modules/common.py @@ -212,7 +212,7 @@ def root_path(name, module_set_name): # Root folders where the various module files should be written roots = spack.config.get(f"modules:{module_set_name}:roots", {}) - path = roots.get(name, os.path.join(paths.modules_base, name)) + path = roots.get(name, os.path.join(paths.data_home, name)) return spack.util.path.canonicalize_path(path) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 49d3c837868e53..667987a8aece4e 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -376,28 +376,6 @@ def _decide_old_or_new_location( else: return new_location - @property - def modules_base(self): - # This is similar to logic _decide_old_or_new_location, but this - # moves the modules base if any component (typically one of lmod or - # tcl) has been relocated, so is examining one-layer deeper - for module_dir in ["lmod", "modules"]: - if dir_is_occupied(os.path.join(self.data_home, module_dir)): - return self.data_home - - new_default_is_occupied = False - for module_dir in ["lmod", "modules"]: - if dir_is_occupied(os.path.join(self.data_home, module_dir)): - new_default_is_occupied = True - break - if new_default_is_occupied: - return self.data_home - - for module_dir in ["lmod", "modules"]: - if dir_is_occupied(os.path.join(self.base.share_path, module_dir)): - return self.base.share_path - return self.data_home - locations = SpackPaths(paths_base.locations) diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index c9f1856e9cc3f9..78f9d18b3d9f4c 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -81,7 +81,6 @@ def replacements(): "user_cache_path": lambda: paths.user_cache_path, "default_install_root": lambda: paths.default_install_location, "default_envs_root": lambda: paths.default_envs_path, - "modules_base": lambda: paths.modules_base, "data_home": lambda: paths.data_home, "cache_home": lambda: paths.cache_home, "state_home": lambda: paths.state_home, From ca953bdc48f80ced0f8d12e240509d7ab020fa7c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 11:34:49 -0700 Subject: [PATCH 403/506] update comment based on updated static path Signed-off-by: Peter Scheibel --- etc/spack/defaults/base/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index 76682cde186abc..a00030948bd503 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -83,7 +83,7 @@ config: ## Directory where spack managed environments are created and stored # If set to a non-default value, this setting has precedence. # This default value tells spack to: - # * Prefer $data_home/envs + # * Prefer $data_home/environments # * The old location for envs is $spack/var/spack/environments: Spack will # use this old location if the new location is empty and the old one isn't environments_root: $default_envs_root From eb7e756c4aedc534d5529d3af6c3b6d19c7cfb60 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 11:43:30 -0700 Subject: [PATCH 404/506] Revert "isolate cmd (and test)" This reverts commit f7be065ef1c1eba9f98f769549bb69d078c3184e. --- lib/spack/spack/cmd/isolate.py | 161 --------------------------- lib/spack/spack/test/cmd/isolate.py | 164 ---------------------------- share/spack/spack-completion.bash | 11 +- share/spack/spack-completion.fish | 13 --- 4 files changed, 1 insertion(+), 348 deletions(-) delete mode 100644 lib/spack/spack/cmd/isolate.py delete mode 100644 lib/spack/spack/test/cmd/isolate.py diff --git a/lib/spack/spack/cmd/isolate.py b/lib/spack/spack/cmd/isolate.py deleted file mode 100644 index fcdf290ad36d98..00000000000000 --- a/lib/spack/spack/cmd/isolate.py +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright Spack Project Developers. See COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import argparse -from pathlib import Path - -import spack -import spack.config -import spack.llnl.util.tty as tty -import spack.paths_base -import spack.util.path -import spack.util.spack_yaml as syaml -from spack.cmd.common import arguments - -description = "force spack to only use/write data from/to an isolated prefix" -section = "config" -level = "long" - - -def setup_parser(subparser: argparse.ArgumentParser) -> None: - subparser.add_argument( - "--force-scopes", action="store_true", help="remove all config scopes in ~" - ) - subparser.add_argument( - "--force-home", action="store_true", help="override home even if user setting is detected" - ) - subparser.add_argument("--force-all", action="store_true", help="set all 'force' options") - # TODO: add an --undo option: delete config:locations:home and restore - # default config scopes in the include: section of the spack scope - arguments.add_common_arguments(subparser, ["specs"]) - - -class Force: - - SCOPES = 1 - HOME = 2 - - options = {SCOPES: "--force-scopes", HOME: "--force-home"} - - def __init__(self, args): - self.all = args.force_all - self.active = {Force.SCOPES: args.force_scopes, Force.HOME: args.force_home} - - def go_ahead(self, category, context): - do_it = self.all or self.active[category] - if do_it: - tty.debug(f"{context}\nOverriding because{Force.options[category]} is set") - else: - tty.warn(f"{context}\nYou can override with {Force.options[category]}") - return do_it - - -def isolate(parser, args): - # TODO: warn if installed specs are detected? If we move config:locations:home - # then these will be "lost" - - force = Force(args) - - spack_root = Path(spack.paths_base.locations.prefix) - - def is_in_spack_prefix(path): - resolved = Path(spack.util.path.canonicalize_path(path)).resolve() - return path == spack_root or spack_root in resolved.parents - - def same_path(x, y): - return Path(x).resolve(strict=False) == Path(y).resolve(strict=False) - - current_install_root = spack.config.get("config:install_tree:root") - current_env_root = spack.config.get("config:environments_root") - - if current_install_root and current_install_root != "$default_install_root": - tty.warn( - f"config:install_tree:root is set to {current_install_root}," - " the install tree will not be relocated because this setting" - " has precedence." - ) - - if current_env_root and current_env_root != "$default_envs_root": - tty.warn( - f"config:environments_root is set to {current_env_root}, " - " environments will not be relocated because this setting" - " has precedence." - ) - - current_home = spack.config.get("config:locations:home") - change_home = True - if current_home: - if is_in_spack_prefix(current_home): - change_home = False - tty.debug(f"config:locations:home is inside $spack: {current_home}\nkeeping it") - else: - msg = f"config:locations:home is outside of $spack: {current_home}" - if not force.go_ahead(Force.HOME, msg): - # By default for a new clone of Spack, config:locations:home is not - # set. So if we're here, it implies we will be changing something - # that was set by the user. - change_home = False - - # else: home is unset (typical) - we will set it to relocate everything - # except for config (and then redirect config) - - if change_home: - config_spack = spack.config.get("config", scope="spack") - locations = config_spack.setdefault("locations", {}) - locations["home"] = "$spack/all-data" - - include_spack = spack.config.get("include", scope="spack") or [] - - # Take included scopes defined in the "spack" scope and remove the - # user/system scopes. Look for other scopes defined here that exist - # outside the spack prefix; --force will remove those scopes but - # otherwise this command will just print a warning - visited = set() - new_include_spack = syaml.syaml_list() - for scope_def in include_spack: - scope_name = scope_def.get("name", None) - path = scope_def.get("path") - if scope_name: - scope_id = f"{scope_name} - {path}" - else: - scope_id = f"(unnamed) - {path}" - - if scope_name == "user": - path = spack.util.path.canonicalize_path(path) - if same_path(path, spack.util.path.canonicalize_path("~/.config/spack")): - tty.debug("Removing default user scope") - elif not force.go_ahead(Force.SCOPES, "user scope is not in default path"): - new_include_spack.append(scope_def) - visited.add("user") - elif scope_name == "system": - if same_path(path, "/etc/spack"): - tty.debug("Removing default system scope") - elif not force.go_ahead(Force.SCOPES, "system scope is not in default path"): - new_include_spack.append(scope_def) - visited.add("system") - else: - if not is_in_spack_prefix(path): - if not force.go_ahead( - Force.SCOPES, f"User-defined scope is not in spack prefix: {scope_id}" - ): - new_include_spack.append(scope_def) - else: - new_include_spack.append(scope_def) - - for expect_remove in ["user", "system"]: - if expect_remove not in visited: - tty.warn( - f"Expected to find (and remove): {expect_remove}" - "\nbut it wasn't present (suggests prior run of `spack isolate`, or)" - " manual editing)" - ) - - if change_home: - spack.config.set("config", config_spack, scope="spack") - # TODO: the "spack" config scope is version-controlled in the - # spack repo, so this change can be dropped or committed by - # accident (e.g. if the user runs `git checkout` or - # `git commit -a` without being aware of the effects of this - # command. - spack.config.set("include", new_include_spack, scope="spack") diff --git a/lib/spack/spack/test/cmd/isolate.py b/lib/spack/spack/test/cmd/isolate.py deleted file mode 100644 index f9f416e0e8448d..00000000000000 --- a/lib/spack/spack/test/cmd/isolate.py +++ /dev/null @@ -1,164 +0,0 @@ -# Copyright Spack Project Developers. See COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import shutil -from pathlib import Path - -import pytest - -import spack.config -import spack.paths_base -import spack.util.spack_yaml as syaml -from spack.main import SpackCommand -from spack.paths import SpackPaths -from spack.paths import locations as paths -from spack.paths_base import SpackPathsBase - -isolate = SpackCommand("isolate") - - -@pytest.fixture() -def mock_spack_scope(tmp_path_factory, mutable_config): - higher_scope_dir = tmp_path_factory.mktemp("higher") - spack_scope_dir = tmp_path_factory.mktemp("test-spack-scope") - lower_scope_dir = tmp_path_factory.mktemp("higher") - - # Copy the real spack scope includes into a test location, so that - # we can check that this command modifies it. - src_file = Path(paths.prefix) / "etc" / "spack" / "include.yaml" - cpy_dst = str(Path(spack_scope_dir) / "include.yaml") - shutil.copyfile(src_file, cpy_dst) - - scopes = [ - spack.config.DirectoryConfigScope("lower", str(lower_scope_dir)), - spack.config.DirectoryConfigScope("spack", str(spack_scope_dir)), - spack.config.DirectoryConfigScope("higher", str(higher_scope_dir)), - ] - - with spack.config.use_configuration(*scopes) as config: - yield config - - -def includes_as_a_dict(): - includes = spack.config.get("include") - return dict((scope["name"], scope) for scope in includes) - - -def test_isolate_removes_scopes(mock_spack_scope): - # In default case, spack has two scopes that point to locations - # outside of spack. `spack isolate` should remove both if they - # haven't been modified. - - assert not spack.config.get("config:locations:home") - initial_cfg = includes_as_a_dict() - assert "user" in initial_cfg - assert "system" in initial_cfg - - spack.config.get("include") - output = isolate() - - # Warnings would be printed if user modifications are detected. - # In this test we haven't made any, so there shouldn't be any - # warnings. - assert "Warning:" not in output - - after_cfg = includes_as_a_dict() - assert "user" not in after_cfg - assert "system" not in after_cfg - - -def test_isolate_retains_scope_when_not_default(mock_spack_scope): - cfg_as_dict = includes_as_a_dict() - non_default_path = "/something/else/not/the/default" - cfg_as_dict["user"]["path"] = non_default_path - - spack.config.set("include", syaml.syaml_list(cfg_as_dict.values()), scope="spack") - isolate() - - after_cfg = includes_as_a_dict() - assert "system" not in after_cfg - assert after_cfg["user"]["path"] == non_default_path - - -@pytest.fixture -def redirect_base_paths(tmp_path_factory, monkeypatch): - base_prefix = str(tmp_path_factory.mktemp("base-test")) - pb = SpackPathsBase(base_prefix) - # spack.util.path.canonicalize_path uses paths_base.locations - # so this should generally be monkeypatched when testing anything - # that modifies path vars - monkeypatch.setattr(spack.paths_base, "locations", pb) - yield pb - - -def test_isolate_redirect_installs( - mock_spack_scope, set_home, tmp_path_factory, redirect_base_paths -): - # in the default case where a user clones a new spack instance, it will - # write into ~. Check that `spack isolate` redirects installs (for example) - # into $spack - base = redirect_base_paths - home_prefix = str(tmp_path_factory.mktemp("home-test")) - set_home(home_prefix) - - p1 = SpackPaths(base) - assert p1.default_install_location == str( - Path(home_prefix) / ".local" / "share" / "spack" / "installs" - ) - - isolate() - - p2 = SpackPaths(base) - assert p2.default_install_location == str( - Path(p2.base.prefix) / "all-data" / ".local" / "share" / "spack" / "installs" - ) - - -def test_isolate_cfg_points_outside_spack( - mock_spack_scope, tmp_path_factory, mutable_config, redirect_base_paths -): - # check that `spack isolate` will rewrite config:locations:home - # when it points outside of $spack - base = redirect_base_paths - dir_outside_spack = str(tmp_path_factory.mktemp("outside-dir")) - - spack.config.set("config:locations", {"home": dir_outside_spack}, scope="spack") - - initial_expected = str(Path(dir_outside_spack) / ".local" / "share" / "spack" / "installs") - p1 = SpackPaths(base) - assert p1.default_install_location == initial_expected - - # First try to isolate without --force-... - nothing should happen - cmd_output = isolate() - assert "config:locations:home is outside of $spack" in cmd_output - assert "You can override with --force-home" in cmd_output - p2 = SpackPaths(base) - assert p2.default_install_location == initial_expected - - isolate("--force-home") - p3 = SpackPaths(base) - assert p3.default_install_location == str( - Path(base.prefix) / "all-data" / ".local" / "share" / "spack" / "installs" - ) - - -def test_isolate_unaffected_by_higher_scope( - mock_spack_scope, tmp_path_factory, mutable_config, redirect_base_paths -): - # set config:locations:home in a higher-precedence scope: make sure - # that `spack isolate` does not interfere - base = redirect_base_paths - dir_outside_spack = str(tmp_path_factory.mktemp("outside-dir")) - - spack.config.set("config", {"locations": {"home": dir_outside_spack}}, scope="higher") - - p1 = SpackPaths(base) - expected_path = str(Path(dir_outside_spack) / ".local" / "share" / "spack" / "installs") - assert p1.default_install_location == expected_path - - isolate() - - p2 = SpackPaths(base) - # `spack isolate` does not modify scopes other than the 'spack' scope - assert p2.default_install_location == expected_path diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index c0306edd88e94b..39a7c8c42d1c65 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -396,7 +396,7 @@ _spack() { then SPACK_COMPREPLY="--color -v --verbose -k --insecure -b --bootstrap -V --version -h --help -H --all-help -c --config -C --config-scope -e --env -D --env-dir -E --no-env --use-env-repo -d --debug -t --backtrace --pdb --timestamp -m --mock --print-shell-vars --stacktrace --warn-writes-into-spack -l --enable-locks -L --disable-locks -p --profile --profile-file --sorted-profile --lines" else - SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install isolate license list load location log-parse logs maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" + SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse logs maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" fi } @@ -1368,15 +1368,6 @@ _spack_install() { fi } -_spack_isolate() { - if $list_options - then - SPACK_COMPREPLY="-h --help --force-scopes --force-home --force-all" - else - _all_packages - fi -} - _spack_license() { if $list_options then diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 43d26c55de814e..2a192a0cdb0b18 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -389,7 +389,6 @@ complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a graph -d 'generat complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a help -d 'get help on spack and its commands' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a info -d 'get detailed information on a particular package' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a install -d 'build and install packages' -complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a isolate -d 'force spack to only use/write data from/to an isolated prefix' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a license -d 'list and check license headers on files in spack' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a list -d 'list and search available packages' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a load -d 'add package to the user environment' @@ -2202,18 +2201,6 @@ complete -c spack -n '__fish_spack_using_command install' -l fresh-roots -l reus complete -c spack -n '__fish_spack_using_command install' -l deprecated -f -a config_deprecated complete -c spack -n '__fish_spack_using_command install' -l deprecated -d 'allow concretizer to select deprecated versions' -# spack isolate -set -g __fish_spack_optspecs_spack_isolate h/help force-scopes force-home force-all -complete -c spack -n '__fish_spack_using_command_pos_remainder 0 isolate' -f -k -a '(__fish_spack_specs)' -complete -c spack -n '__fish_spack_using_command isolate' -s h -l help -f -a help -complete -c spack -n '__fish_spack_using_command isolate' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command isolate' -l force-scopes -f -a force_scopes -complete -c spack -n '__fish_spack_using_command isolate' -l force-scopes -d 'remove all config scopes in ~' -complete -c spack -n '__fish_spack_using_command isolate' -l force-home -f -a force_home -complete -c spack -n '__fish_spack_using_command isolate' -l force-home -d 'override home even if user setting is detected' -complete -c spack -n '__fish_spack_using_command isolate' -l force-all -f -a force_all -complete -c spack -n '__fish_spack_using_command isolate' -l force-all -d 'set all '"'"'force'"'"' options' - # spack license set -g __fish_spack_optspecs_spack_license h/help root= complete -c spack -n '__fish_spack_using_command_pos 0 license' -f -a list-files -d 'list files in spack that should have license headers' From 11e3c85d8b10878c6236fccf655cf00e4ea4c4d4 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 11:50:41 -0700 Subject: [PATCH 405/506] add backwards compat scope which reads config from old user scope dir Signed-off-by: Peter Scheibel --- etc/spack/include.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/etc/spack/include.yaml b/etc/spack/include.yaml index b8ab4ebfeb3b8b..0ddb74c81c1865 100644 --- a/etc/spack/include.yaml +++ b/etc/spack/include.yaml @@ -7,6 +7,14 @@ include: prefer_modify: true when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' + # This is where the user scope used to be located before 1.2 + # If any config was stored here, we want to be able to read it + # (new config should not be written here though) + - name: "backwards-compat" + path: "~/.spack/" + optional: true + when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' + # site configuration scope - name: "site" path: "$spack/etc/spack/site" From b09f1c88026019bb25192bebb734115a321c09e4 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 12:55:02 -0700 Subject: [PATCH 406/506] forgot to add new scope to command completion Signed-off-by: Peter Scheibel --- share/spack/spack-completion.fish | 60 +++++++++++++++---------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 2a192a0cdb0b18..3320060063704c 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -605,7 +605,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_enable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap enable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap disable @@ -613,7 +613,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_disable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap disable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap reset @@ -628,14 +628,14 @@ set -g __fish_spack_optspecs_spack_bootstrap_root h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap root' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap list set -g __fish_spack_optspecs_spack_bootstrap_list h/help scope= complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap add @@ -644,7 +644,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap add' -f -a '(__ complete -c spack -n '__fish_spack_using_command_pos 1 bootstrap add' -f -a '(__fish_spack_environments)' complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -f -a trust complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -d 'enable the source immediately upon addition' @@ -822,7 +822,7 @@ complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirro complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -d 'override any configured mirrors with this mirror URL' complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -f -a output_file complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -d 'file where rebuild info should be written' -complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -d 'configuration scope containing mirrors to check' # spack buildcache download @@ -1130,7 +1130,7 @@ set -g __fish_spack_optspecs_spack_compiler_find h/help scope= j/jobs= complete -c spack -n '__fish_spack_using_command compiler find' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler find' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1140,7 +1140,7 @@ set -g __fish_spack_optspecs_spack_compiler_add h/help scope= j/jobs= complete -c spack -n '__fish_spack_using_command compiler add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1152,7 +1152,7 @@ complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help - complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -d 'configuration scope to modify' # spack compiler rm @@ -1162,14 +1162,14 @@ complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -d 'configuration scope to modify' # spack compiler list set -g __fish_spack_optspecs_spack_compiler_list h/help scope= remote complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler list' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler list' -l remote -d 'list also compilers from registered buildcaches' @@ -1178,7 +1178,7 @@ complete -c spack -n '__fish_spack_using_command compiler list' -l remote -d 'li set -g __fish_spack_optspecs_spack_compiler_ls h/help scope= remote complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler ls' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler ls' -l remote -d 'list also compilers from registered buildcaches' @@ -1188,7 +1188,7 @@ set -g __fish_spack_optspecs_spack_compiler_info h/help scope= remote complete -c spack -n '__fish_spack_using_command_pos 0 compiler info' -f -a '(__fish_spack_installed_compilers)' complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler info' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler info' -l remote -d 'list also compilers from registered buildcaches' @@ -1197,7 +1197,7 @@ complete -c spack -n '__fish_spack_using_command compiler info' -l remote -d 'li set -g __fish_spack_optspecs_spack_compilers h/help scope= remote complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command compilers' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compilers' -l remote -d 'list also compilers from registered buildcaches' @@ -1264,7 +1264,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a update -d ' complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a revert -d 'revert configuration files to their state before update' complete -c spack -n '__fish_spack_using_command config' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify' # spack config get @@ -1824,7 +1824,7 @@ complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -f complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -d 'packages to exclude from search' complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -f -a path complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -d 'one or more alternative search paths for finding externals' -complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command external find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command external find' -l all -f -a all complete -c spack -n '__fish_spack_using_command external find' -l all -d 'search for all packages that Spack knows about' @@ -2416,7 +2416,7 @@ set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsig complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use ``--type binary --type source`` (default)' @@ -2452,7 +2452,7 @@ set -g __fish_spack_optspecs_spack_mirror_remove h/help scope= all-scopes complete -c spack -n '__fish_spack_using_command_pos 0 mirror remove' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror remove' -l all-scopes -f -a all_scopes complete -c spack -n '__fish_spack_using_command mirror remove' -l all-scopes -d 'remove from all config scopes (default: highest scope with matching mirror)' @@ -2462,7 +2462,7 @@ set -g __fish_spack_optspecs_spack_mirror_rm h/help scope= all-scopes complete -c spack -n '__fish_spack_using_command_pos 0 mirror rm' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror rm' -l all-scopes -f -a all_scopes complete -c spack -n '__fish_spack_using_command mirror rm' -l all-scopes -d 'remove from all config scopes (default: highest scope with matching mirror)' @@ -2476,7 +2476,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -f -a p complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -d 'set only the URL used for uploading' complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -f -a fetch complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -d 'set only the URL used for downloading' -complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2518,7 +2518,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a s complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache' complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed complete -c spack -n '__fish_spack_using_command mirror set' -l signed -d 'require signing and signature verification when pushing and installing from this build cache' -complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2543,14 +2543,14 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password-var set -g __fish_spack_optspecs_spack_mirror_list h/help scope= complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'configuration scope to read from' # spack mirror ls set -g __fish_spack_optspecs_spack_mirror_ls h/help scope= complete -c spack -n '__fish_spack_using_command mirror ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -d 'configuration scope to read from' # spack module @@ -2863,7 +2863,7 @@ complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirecto set -g __fish_spack_optspecs_spack_repo_list h/help scope= names namespaces json complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command repo list' -l names -f -a names complete -c spack -n '__fish_spack_using_command repo list' -l names -d 'show configuration names only' @@ -2876,7 +2876,7 @@ complete -c spack -n '__fish_spack_using_command repo list' -l json -d 'output r set -g __fish_spack_optspecs_spack_repo_ls h/help scope= names namespaces json complete -c spack -n '__fish_spack_using_command repo ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command repo ls' -l names -f -a names complete -c spack -n '__fish_spack_using_command repo ls' -l names -d 'show configuration names only' @@ -2894,7 +2894,7 @@ complete -c spack -n '__fish_spack_using_command repo add' -l name -r -f -a name complete -c spack -n '__fish_spack_using_command repo add' -l name -r -d 'config name for the package repository, defaults to the namespace of the repository' complete -c spack -n '__fish_spack_using_command repo add' -l path -r -f -a path complete -c spack -n '__fish_spack_using_command repo add' -l path -r -d 'relative path to the Spack package repository inside a git repository. Can be repeated to add multiple package repositories in case of a monorepo' -complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -d 'configuration scope to modify' # spack repo set @@ -2906,7 +2906,7 @@ complete -c spack -n '__fish_spack_using_command repo set' -l destination -r -f complete -c spack -n '__fish_spack_using_command repo set' -l destination -r -d 'destination to clone git repository into' complete -c spack -n '__fish_spack_using_command repo set' -l path -r -f -a path complete -c spack -n '__fish_spack_using_command repo set' -l path -r -d 'relative path to the Spack package repository inside a git repository. Can be repeated to add multiple package repositories in case of a monorepo' -complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -d 'configuration scope to modify' # spack repo remove @@ -2914,7 +2914,7 @@ set -g __fish_spack_optspecs_spack_repo_remove h/help scope= all-scopes complete -c spack -n '__fish_spack_using_command_pos 0 repo remove' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command repo remove' -l all-scopes -f -a all_scopes complete -c spack -n '__fish_spack_using_command repo remove' -l all-scopes -d 'remove from all config scopes (default: highest scope with matching repo)' @@ -2924,7 +2924,7 @@ set -g __fish_spack_optspecs_spack_repo_rm h/help scope= all-scopes complete -c spack -n '__fish_spack_using_command_pos 0 repo rm' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command repo rm' -l all-scopes -f -a all_scopes complete -c spack -n '__fish_spack_using_command repo rm' -l all-scopes -d 'remove from all config scopes (default: highest scope with matching repo)' @@ -2946,7 +2946,7 @@ complete -c spack -n '__fish_spack_using_command repo update' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command repo update' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command repo update' -l remote -s r -r -f -a remote complete -c spack -n '__fish_spack_using_command repo update' -l remote -s r -r -d 'name of remote to check for branches, tags, or commits' -complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command repo update' -l branch -s b -r -f -a branch complete -c spack -n '__fish_spack_using_command repo update' -l branch -s b -r -d 'name of a branch to change to' From 110c01d61ff50ec12b06f9bda9beb20b2b4cd24a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 13:31:53 -0700 Subject: [PATCH 407/506] move all docs on controlling data location into dedicated section; add table Signed-off-by: Peter Scheibel --- lib/spack/docs/configuration_basics.rst | 55 -------------- lib/spack/docs/where_spack_writes_data.rst | 86 +++++++++++++++++++++- 2 files changed, 83 insertions(+), 58 deletions(-) diff --git a/lib/spack/docs/configuration_basics.rst b/lib/spack/docs/configuration_basics.rst index 6b3fed06d090ac..208ecc3672ec5f 100644 --- a/lib/spack/docs/configuration_basics.rst +++ b/lib/spack/docs/configuration_basics.rst @@ -535,54 +535,6 @@ Note that, as with shell variables, you can write these as ``$varname`` or with Their names are also case insensitive, meaning that ``$SPACK`` works just as well as ``$spack``. These special variables are substituted first, so any environment variables with the same name will not be used. -.. _config-file-data-variables: - -Spack-specific variables controlling data location -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Files generated and used by spack are organized roughly into four categories: - -* Persistent, large quantities of data (e.g. installs and environments) -* Temporary (or assumed temporary) large quantities of data (e.g. stages for installs) -* Persistent caches/indices used by spack to speed up its commands (small quantities of data) -* The configuration files themselves - -The corresponding variables that describe where this data is placed are: - -* ``$data_home`` -* ``$cache_home`` -* ``$state_home`` (also known as ``$user_cache_path``) -* Config file locations are an exception: they can only be controlled with :ref:`environment variables ` or with :ref:`include.yaml ` - -You can refer to these variables when configuring locations for stages, misc cache, etc. - -Each of these variables are the *default* (fallback) for data in their category: more-specific data in that category may have config that overrides these defaults. -For example while build stages would reasonably be placed in ``$cache_home``, Spack's default configuration sets ``config:build_stage`` to the user's tempdir. -Any configuration controlling location that is more-specific than the above variables will always take precedence (e.g. ``config:install_tree:root``). - -Each of these variables can be set with config or with environment variables. -For example ``$data_home`` evaluates to one of the following (highest-priority first): - -#. ``SPACK_DATA_HOME`` env var if that is set -#. Under ``SPACK_HOME`` env var; for ``$data_home``, it is ``$SPACK_HOME/.local/share/spack`` -#. ``config:locations:data`` -#. Under ``config:locations:home``; for ``$data_home`` it is ``$spack_home/.local/share/spack`` -#. ``XDG_DATA_HOME/spack`` if XDG_DATA_HOME is set -#. Under the default for ``XDG_DATA_HOME``: ``~/.local/share/spack`` - -``config:locations:home`` / ``SPACK_HOME`` can be used to control all 3 of ``data_home``, ``cache_home``, and ``state_home``. - -Of particular interest is where the environments and installs are placed by Spack, because these can take up a lot of space. -These are controlled by ``$data_home``. -Older installs of spack placed these within ``$spack``, and fallback scheme in these cases is augmented to prefer these old locations if no data is detected in the corresponding new locations: - -* ``$default_install_root``: the location where installs go by default. - Overridden by ``config:install_tree:root``. - Prefers ``$data_home/installs``, but if there are no installs there and there are installs in the old location ``$spack/opt/spack``, then the old location will be used. -* ``$default_envs_root``: the location where environments are managed by default. - Overridden by ``config:environments_root``. - Prefers ``$data_home/envs`` but if there are no envs there and there are envs in the old location ``$spack/var/spack/environments``, then the old location will be used. - Environment variables ^^^^^^^^^^^^^^^^^^^^^ @@ -718,10 +670,3 @@ Spack provides three environment variables that allow you to override or opt out * ``SPACK_SYSTEM_CONFIG_PATH``: Override the path to use for the ``system`` scope (``/etc/spack`` by default). * ``SPACK_DISABLE_LOCAL_CONFIG``: Set this environment variable to completely disable **all** configurations from the system and user directories. Spack will then only consider its own defaults and ``site`` configuration locations. - -With these settings, if you want to isolate Spack in a CI environment, you can do this: - -.. code-block:: console - - $ export SPACK_DISABLE_LOCAL_CONFIG=true - $ export SPACK_USER_CACHE_PATH=/tmp/spack diff --git a/lib/spack/docs/where_spack_writes_data.rst b/lib/spack/docs/where_spack_writes_data.rst index 982002ef8e3c53..f019b92a3499da 100644 --- a/lib/spack/docs/where_spack_writes_data.rst +++ b/lib/spack/docs/where_spack_writes_data.rst @@ -5,7 +5,7 @@ .. meta:: :description lang=en: - Learn how to control where Spack generates files. + Learn how to control where Spack generates files and reads files, and how to effectively isolate a Spack installation. .. _where_spack_writes_data: @@ -28,8 +28,88 @@ You can control this in the following ways: In the absence of any Spack-specific settings, Spack will respect [XDG](https://specifications.freedesktop.org/basedir/latest/) environment variables controlling the home directory for specific types of data. +.. _config-file-data-variables: + +Spack-specific variables controlling data location +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Files generated and used by spack are organized roughly into four categories: + +* Persistent, large quantities of data (e.g. installs and environments) +* Temporary (or assumed temporary) large quantities of data (e.g. stages for installs) +* Persistent caches/indices used by spack to speed up its commands (small quantities of data) +* The configuration files themselves + +The corresponding variables that describe where this data is placed are: + +* ``$data_home`` +* ``$cache_home`` +* ``$state_home`` (also known as ``$user_cache_path``) +* Config file locations are an exception: they can only be controlled with :ref:`environment variables ` or with :ref:`include.yaml ` + +You can refer to these variables when configuring locations for stages, misc cache, etc. + +The following table may help visualize where spack puts all the files it may generate: + ++-----------------+-----------+------------+------------+--------------------+ +| | data_home | state_home | cache_home | somewhere_else | ++=================+===========+============+============+====================+ +| installs | x | | | | ++-----------------+-----------+------------+------------+--------------------+ +| build stages | | | | x [#wheretable-1]_ | ++-----------------+-----------+------------+------------+--------------------+ +| download cache | x | | | | ++-----------------+-----------+------------+------------+--------------------+ +| gpg keys | x | | | | ++-----------------+-----------+------------+------------+--------------------+ +| modules | x | | | | ++-----------------+-----------+------------+------------+--------------------+ +| environments | x | | | | ++-----------------+-----------+------------+------------+--------------------+ +| misc cache | | x | | | ++-----------------+-----------+------------+------------+--------------------+ +| test stages | | x | | | ++-----------------+-----------+------------+------------+--------------------+ +| licenses | x | | | | ++-----------------+-----------+------------+------------+--------------------+ +| config files | | | | x [#wheretable-2]_ | ++-----------------+-----------+------------+------------+--------------------+ + +.. [#wheretable-1] cache_home is used as a backup, but Spack prefers to write into the users temp dir if it's available +.. [#wheretable-2] as discussed elsewhere in this section, config is controlled with :ref:`environment variables ` or with :ref:`include.yaml ` + +Each of these variables are the *default* (fallback) for data in their category: more-specific data in that category may have config that overrides these defaults. +For example while build stages would reasonably be placed in ``$cache_home``, Spack's default configuration sets ``config:build_stage`` to the user's tempdir. +Any configuration controlling location that is more-specific than the above variables will always take precedence (e.g. ``config:install_tree:root``). + +Each of these variables can be set with config or with environment variables. +For example ``$data_home`` evaluates to one of the following (highest-priority first): + +#. ``SPACK_DATA_HOME`` env var if that is set +#. Under ``SPACK_HOME`` env var; for ``$data_home``, it is ``$SPACK_HOME/.local/share/spack`` +#. ``config:locations:data`` +#. Under ``config:locations:home``; for ``$data_home`` it is ``$spack_home/.local/share/spack`` +#. ``XDG_DATA_HOME/spack`` if XDG_DATA_HOME is set +#. Under the default for ``XDG_DATA_HOME``: ``~/.local/share/spack`` + +``config:locations:home`` / ``SPACK_HOME`` can be used to control all 3 of ``data_home``, ``cache_home``, and ``state_home``. + +Of particular interest is where the environments and installs are placed by Spack, because these can take up a lot of space. +These are controlled by ``$data_home``. +Older installs of spack placed these within ``$spack``, and fallback scheme in these cases is augmented to prefer these old locations if no data is detected in the corresponding new locations: + +* ``$default_install_root``: the location where installs go by default. + Overridden by ``config:install_tree:root``. + Prefers ``$data_home/installs``, but if there are no installs there and there are installs in the old location ``$spack/opt/spack``, then the old location will be used. +* ``$default_envs_root``: the location where environments are managed by default. + Overridden by ``config:environments_root``. + Prefers ``$data_home/envs`` but if there are no envs there and there are envs in the old location ``$spack/var/spack/environments``, then the old location will be used. + +References +^^^^^^^^^^ + For more on this, see: -* :ref:`Variables controlling data location ` * :ref:`include.yaml ` -* :ref:`config.yaml ` \ No newline at end of file +* :ref:`config.yaml ` +* :ref:`Environment variables controlling config scopes ` \ No newline at end of file From 7ef235e4775da9523bc3ff3673f4896377b39bdc Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 7 Apr 2026 14:06:01 -0700 Subject: [PATCH 408/506] add stub initial description of location vars with pointer to longer section Signed-off-by: Peter Scheibel --- lib/spack/docs/configuration_basics.rst | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/spack/docs/configuration_basics.rst b/lib/spack/docs/configuration_basics.rst index 208ecc3672ec5f..9ac64b962ef730 100644 --- a/lib/spack/docs/configuration_basics.rst +++ b/lib/spack/docs/configuration_basics.rst @@ -45,7 +45,7 @@ Here is an example ``config.yaml`` file: config: install_tree: - root: $data_home/installs + root: $default_install_root build_stage: - $tempdir/$user/spack-stage - $cache_home/stage @@ -319,7 +319,7 @@ If your configurations look like this: config: install_tree: - root: $data_home/installs + root: $default_install_root build_stage: - $tempdir/$user/spack-stage - $cache_home/stage @@ -535,6 +535,17 @@ Note that, as with shell variables, you can write these as ``$varname`` or with Their names are also case insensitive, meaning that ``$SPACK`` works just as well as ``$spack``. These special variables are substituted first, so any environment variables with the same name will not be used. +The following are additional special variables that describe locations where spack writes data: + +* ``$data_home`` +* ``$state_home`` +* ``$cache_home`` +* ``$spack_home`` +* ``$default_install_root`` +* ``$default_envs_root`` + +These are described in more detail in :ref:`config-file-data-variables`. + Environment variables ^^^^^^^^^^^^^^^^^^^^^ From 61b627ce3e647c0e72e29fd671040a8d1eada724 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 13 Apr 2026 10:06:40 -0700 Subject: [PATCH 409/506] describe placement of data/state/cache relative to spack_home Signed-off-by: Peter Scheibel --- lib/spack/docs/where_spack_writes_data.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/spack/docs/where_spack_writes_data.rst b/lib/spack/docs/where_spack_writes_data.rst index f019b92a3499da..b034f7d1db6af4 100644 --- a/lib/spack/docs/where_spack_writes_data.rst +++ b/lib/spack/docs/where_spack_writes_data.rst @@ -93,6 +93,14 @@ For example ``$data_home`` evaluates to one of the following (highest-priority f #. Under the default for ``XDG_DATA_HOME``: ``~/.local/share/spack`` ``config:locations:home`` / ``SPACK_HOME`` can be used to control all 3 of ``data_home``, ``cache_home``, and ``state_home``. +They are placed relative to this directory (``$spack_home``): + +* ``data_home`` is placed in ``$spack_home/.local/share/spack`` (as described above). +* ``state_home`` is placed in ``$spack_home/.local/state/spack``. +* ``cache_home`` is placed in ``$spack_home/.cache/spack``. + +Location of installs and environments +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Of particular interest is where the environments and installs are placed by Spack, because these can take up a lot of space. These are controlled by ``$data_home``. From fb3c437c6d23c867b8ccdc343c2eabca3efc617b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 13 Apr 2026 10:48:14 -0700 Subject: [PATCH 410/506] update docs based on review Signed-off-by: Peter Scheibel --- lib/spack/docs/where_spack_writes_data.rst | 53 +++++++++++----------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/lib/spack/docs/where_spack_writes_data.rst b/lib/spack/docs/where_spack_writes_data.rst index b034f7d1db6af4..f067edc6bbc113 100644 --- a/lib/spack/docs/where_spack_writes_data.rst +++ b/lib/spack/docs/where_spack_writes_data.rst @@ -51,32 +51,33 @@ You can refer to these variables when configuring locations for stages, misc cac The following table may help visualize where spack puts all the files it may generate: -+-----------------+-----------+------------+------------+--------------------+ -| | data_home | state_home | cache_home | somewhere_else | -+=================+===========+============+============+====================+ -| installs | x | | | | -+-----------------+-----------+------------+------------+--------------------+ -| build stages | | | | x [#wheretable-1]_ | -+-----------------+-----------+------------+------------+--------------------+ -| download cache | x | | | | -+-----------------+-----------+------------+------------+--------------------+ -| gpg keys | x | | | | -+-----------------+-----------+------------+------------+--------------------+ -| modules | x | | | | -+-----------------+-----------+------------+------------+--------------------+ -| environments | x | | | | -+-----------------+-----------+------------+------------+--------------------+ -| misc cache | | x | | | -+-----------------+-----------+------------+------------+--------------------+ -| test stages | | x | | | -+-----------------+-----------+------------+------------+--------------------+ -| licenses | x | | | | -+-----------------+-----------+------------+------------+--------------------+ -| config files | | | | x [#wheretable-2]_ | -+-----------------+-----------+------------+------------+--------------------+ - -.. [#wheretable-1] cache_home is used as a backup, but Spack prefers to write into the users temp dir if it's available -.. [#wheretable-2] as discussed elsewhere in this section, config is controlled with :ref:`environment variables ` or with :ref:`include.yaml ` ++----------------+-----------+--------------------+------------+--------------------+ +| | data_home | state_home | cache_home | somewhere_else | ++================+===========+====================+============+====================+ +| installs | x | | | | ++----------------+-----------+--------------------+------------+--------------------+ +| build stages | | | | x [#wheretable-1]_ | ++----------------+-----------+--------------------+------------+--------------------+ +| download cache | x | | | | ++----------------+-----------+--------------------+------------+--------------------+ +| gpg keys | x | | | | ++----------------+-----------+--------------------+------------+--------------------+ +| modules | x | | | | ++----------------+-----------+--------------------+------------+--------------------+ +| environments | x | | | | ++----------------+-----------+--------------------+------------+--------------------+ +| misc cache | | x [#wheretable-2]_ | | | ++----------------+-----------+--------------------+------------+--------------------+ +| test stages | | x | | | ++----------------+-----------+--------------------+------------+--------------------+ +| licenses | x | | | | ++----------------+-----------+--------------------+------------+--------------------+ +| config files | | | | x [#wheretable-3]_ | ++----------------+-----------+--------------------+------------+--------------------+ + +.. [#wheretable-1] ``cache_home`` is used as a backup, but Spack prefers to write into the user's temp dir if it's available +.. [#wheretable-2] ``cache_home`` is modeled after `$XDG_CACHE_HOME `_. Spack assumes that ``$XDG_CACHE_HOME`` can be removed on user log-out. Spack caches are intended to be longer-lived, so they live in ``state_home`` instead. +.. [#wheretable-3] as discussed elsewhere in this section, user-scope config is controlled with :ref:`environment variables ` or with :ref:`include.yaml ` to avoid recursion issues with configurable locations. For the locations of other config scopes and how to write to them instead, see :ref:`configuration scopes `. Each of these variables are the *default* (fallback) for data in their category: more-specific data in that category may have config that overrides these defaults. For example while build stages would reasonably be placed in ``$cache_home``, Spack's default configuration sets ``config:build_stage`` to the user's tempdir. From b7d958652195f959a6954c5942595177b7a35772 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 13 Apr 2026 13:51:45 -0700 Subject: [PATCH 411/506] add warning about using deprecated user scope Signed-off-by: Peter Scheibel --- lib/spack/spack/main.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 107e0fd2baac57..6e6921ee45031e 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -1047,6 +1047,8 @@ def add_environment_scope(): ) setup_main_options(args) + check_for_deprecated_user_scope() + # ------------------------------------------------------------------------ # Things that require configuration should go below here # ------------------------------------------------------------------------ @@ -1086,6 +1088,27 @@ def add_environment_scope(): return finish_parse_and_run(parser, cmd_name, args, env_format_error) +def check_for_deprecated_user_scope(): + import spack.util.path + + cfg = spack.config.CONFIG + + def is_default(path): + resolved = spack.util.path.canonicalize_path(path) + test_for = spack.util.path.canonicalize_path("~/.spack/") + return resolved == test_for + + for scope in cfg.scopes.values(): + if scope.name == "backwards-compat" and hasattr(scope, "path") and is_default(scope.path): + sentinel = os.path.join(scope.path, ".suppress-deprecation-warning") + if os.path.exists(scope.path) and not os.path.exists(sentinel): + tty.warn( + f"Spack detected (and is using) legacy user scope at {scope.path}" + f"\n\tRun `touch {sentinel}`" + "\n\tor remove this scope to avoid this warning" + ) + + def finish_parse_and_run(parser, cmd_name, main_args, env_format_error): """Finish parsing after we know the command to run.""" # add the found command to the parser and re-run then re-parse From f561865be0b5ab45eb1cef836016dd782af71eb5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 13 Apr 2026 14:05:12 -0700 Subject: [PATCH 412/506] doc style issues Signed-off-by: Peter Scheibel --- lib/spack/docs/where_spack_writes_data.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/spack/docs/where_spack_writes_data.rst b/lib/spack/docs/where_spack_writes_data.rst index f067edc6bbc113..b4127f57ef5366 100644 --- a/lib/spack/docs/where_spack_writes_data.rst +++ b/lib/spack/docs/where_spack_writes_data.rst @@ -76,8 +76,11 @@ The following table may help visualize where spack puts all the files it may gen +----------------+-----------+--------------------+------------+--------------------+ .. [#wheretable-1] ``cache_home`` is used as a backup, but Spack prefers to write into the user's temp dir if it's available -.. [#wheretable-2] ``cache_home`` is modeled after `$XDG_CACHE_HOME `_. Spack assumes that ``$XDG_CACHE_HOME`` can be removed on user log-out. Spack caches are intended to be longer-lived, so they live in ``state_home`` instead. -.. [#wheretable-3] as discussed elsewhere in this section, user-scope config is controlled with :ref:`environment variables ` or with :ref:`include.yaml ` to avoid recursion issues with configurable locations. For the locations of other config scopes and how to write to them instead, see :ref:`configuration scopes `. +.. [#wheretable-2] ``cache_home`` is modeled after `$XDG_CACHE_HOME `_. + Spack assumes that ``$XDG_CACHE_HOME`` can be removed on user log-out. + Spack caches are intended to be longer-lived, so they live in ``state_home`` instead. +.. [#wheretable-3] as discussed elsewhere in this section, user-scope config is controlled with :ref:`environment variables ` or with :ref:`include.yaml ` to avoid recursion issues with configurable locations. + For the locations of other config scopes and how to write to them instead, see :ref:`configuration scopes `. Each of these variables are the *default* (fallback) for data in their category: more-specific data in that category may have config that overrides these defaults. For example while build stages would reasonably be placed in ``$cache_home``, Spack's default configuration sets ``config:build_stage`` to the user's tempdir. @@ -121,4 +124,4 @@ For more on this, see: * :ref:`include.yaml ` * :ref:`config.yaml ` -* :ref:`Environment variables controlling config scopes ` \ No newline at end of file +* :ref:`Environment variables controlling config scopes ` From c093bd05162290d73f969ca4ac41bbceafbb91fa Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 14 Apr 2026 11:33:33 -0700 Subject: [PATCH 413/506] replace (sentinel approach with backwards compat scope) with (relocated user scope plus warning) Signed-off-by: Peter Scheibel --- etc/spack/include.yaml | 8 -------- lib/spack/spack/main.py | 24 ++---------------------- lib/spack/spack/paths.py | 24 ++++++++++++++++++++++++ 3 files changed, 26 insertions(+), 30 deletions(-) diff --git a/etc/spack/include.yaml b/etc/spack/include.yaml index 0ddb74c81c1865..b8ab4ebfeb3b8b 100644 --- a/etc/spack/include.yaml +++ b/etc/spack/include.yaml @@ -7,14 +7,6 @@ include: prefer_modify: true when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' - # This is where the user scope used to be located before 1.2 - # If any config was stored here, we want to be able to read it - # (new config should not be written here though) - - name: "backwards-compat" - path: "~/.spack/" - optional: true - when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' - # site configuration scope - name: "site" path: "$spack/etc/spack/site" diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index d49353fdbd006b..a7fca72426df21 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -39,6 +39,7 @@ import spack.llnl.util.tty.colify import spack.llnl.util.tty.color as color import spack.paths_base +import spack.paths import spack.platforms import spack.solver.asp import spack.spec @@ -1035,7 +1036,7 @@ def add_environment_scope(): ) setup_main_options(args) - check_for_deprecated_user_scope() + spack.paths.locations._warn_unused_old_config() # ------------------------------------------------------------------------ # Things that require configuration should go below here @@ -1076,27 +1077,6 @@ def add_environment_scope(): return finish_parse_and_run(parser, cmd_name, args, env_format_error) -def check_for_deprecated_user_scope(): - import spack.util.path - - cfg = spack.config.CONFIG - - def is_default(path): - resolved = spack.util.path.canonicalize_path(path) - test_for = spack.util.path.canonicalize_path("~/.spack/") - return resolved == test_for - - for scope in cfg.scopes.values(): - if scope.name == "backwards-compat" and hasattr(scope, "path") and is_default(scope.path): - sentinel = os.path.join(scope.path, ".suppress-deprecation-warning") - if os.path.exists(scope.path) and not os.path.exists(sentinel): - tty.warn( - f"Spack detected (and is using) legacy user scope at {scope.path}" - f"\n\tRun `touch {sentinel}`" - "\n\tor remove this scope to avoid this warning" - ) - - def finish_parse_and_run(parser, cmd_name, main_args, env_format_error): """Finish parsing after we know the command to run.""" # add the found command to the parser and re-run then re-parse diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 667987a8aece4e..e5f00428643382 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -203,6 +203,30 @@ def _warn_on_unused_old_data( return msg return "" + def _warn_unused_old_config(self): + user_scope = [s for s in config.CONFIG.active_scopes if s.name == "user"] + + if not user_scope: + return + + # Note: does not require spack.util.path.canonicalize_path because the + # default user scope path only uses "~" + default_path = pathlib.Path("~/.config/spack").expanduser().resolve() + scope_path = pathlib.Path(getattr(user_scope[0], "path", "")).expanduser().resolve() + if scope_path != default_path: + # User overrode the scope path, they meant it + return + + old_path = pathlib.Path("~/.spack").expanduser() + if dir_is_occupied(old_path): + msg = ( + f"Bypassing config in '{old_path}' in favor of '{default_path}'.\n" + f" Detected config in old location: '{default_path}'.\n" + f" Set `SPACK_USER_CONFIG_PATH` or modify your include.yaml to use '{old_path}'\n" + f" or move files from '{old_path}' to '{default_path}' to suppress this warning." + ) + tty.warn(msg) + @property def default_envs_path(self): return self._decide_old_or_new_location( From b4dc303a1e15bf2c842cdd16c32ba06fb7b6d765 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 14 Apr 2026 11:38:12 -0700 Subject: [PATCH 414/506] style edits Signed-off-by: Peter Scheibel --- lib/spack/spack/main.py | 2 +- lib/spack/spack/paths.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index a7fca72426df21..dd4d3b560e70bb 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -38,8 +38,8 @@ import spack.llnl.util.tty as tty import spack.llnl.util.tty.colify import spack.llnl.util.tty.color as color -import spack.paths_base import spack.paths +import spack.paths_base import spack.platforms import spack.solver.asp import spack.spec diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index e5f00428643382..05d09ea3fed3e3 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -222,8 +222,10 @@ def _warn_unused_old_config(self): msg = ( f"Bypassing config in '{old_path}' in favor of '{default_path}'.\n" f" Detected config in old location: '{default_path}'.\n" - f" Set `SPACK_USER_CONFIG_PATH` or modify your include.yaml to use '{old_path}'\n" - f" or move files from '{old_path}' to '{default_path}' to suppress this warning." + " Set `SPACK_USER_CONFIG_PATH` or modify your include.yaml" + f" to use '{old_path}'\n" + f" or move files from '{old_path}' to '{default_path}'" + " to suppress this warning." ) tty.warn(msg) From 537c3ec80441a30c6bb1f77cd6ef0af9308cea4c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 14 Apr 2026 11:51:41 -0700 Subject: [PATCH 415/506] command autocompletion update (no more backwards-compat scope) Signed-off-by: Peter Scheibel --- share/spack/spack-completion.fish | 60 +++++++++++++++---------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 141ca01bf89d5b..bf2da0511b9e09 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -605,7 +605,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_enable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap enable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap disable @@ -613,7 +613,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_disable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap disable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap reset @@ -628,14 +628,14 @@ set -g __fish_spack_optspecs_spack_bootstrap_root h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap root' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap list set -g __fish_spack_optspecs_spack_bootstrap_list h/help scope= complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap add @@ -644,7 +644,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap add' -f -a '(__ complete -c spack -n '__fish_spack_using_command_pos 1 bootstrap add' -f -a '(__fish_spack_environments)' complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -f -a trust complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -d 'enable the source immediately upon addition' @@ -826,7 +826,7 @@ complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirro complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -d 'override any configured mirrors with this mirror URL' complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -f -a output_file complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -d 'file where rebuild info should be written' -complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -d 'configuration scope containing mirrors to check' # spack buildcache download @@ -1134,7 +1134,7 @@ set -g __fish_spack_optspecs_spack_compiler_find h/help scope= j/jobs= complete -c spack -n '__fish_spack_using_command compiler find' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler find' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1144,7 +1144,7 @@ set -g __fish_spack_optspecs_spack_compiler_add h/help scope= j/jobs= complete -c spack -n '__fish_spack_using_command compiler add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1156,7 +1156,7 @@ complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help - complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -d 'configuration scope to modify' # spack compiler rm @@ -1166,14 +1166,14 @@ complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -d 'configuration scope to modify' # spack compiler list set -g __fish_spack_optspecs_spack_compiler_list h/help scope= remote complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler list' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler list' -l remote -d 'list also compilers from registered buildcaches' @@ -1182,7 +1182,7 @@ complete -c spack -n '__fish_spack_using_command compiler list' -l remote -d 'li set -g __fish_spack_optspecs_spack_compiler_ls h/help scope= remote complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler ls' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler ls' -l remote -d 'list also compilers from registered buildcaches' @@ -1192,7 +1192,7 @@ set -g __fish_spack_optspecs_spack_compiler_info h/help scope= remote complete -c spack -n '__fish_spack_using_command_pos 0 compiler info' -f -a '(__fish_spack_installed_compilers)' complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler info' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler info' -l remote -d 'list also compilers from registered buildcaches' @@ -1201,7 +1201,7 @@ complete -c spack -n '__fish_spack_using_command compiler info' -l remote -d 'li set -g __fish_spack_optspecs_spack_compilers h/help scope= remote complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command compilers' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compilers' -l remote -d 'list also compilers from registered buildcaches' @@ -1268,7 +1268,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a update -d ' complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a revert -d 'revert configuration files to their state before update' complete -c spack -n '__fish_spack_using_command config' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify' # spack config get @@ -1828,7 +1828,7 @@ complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -f complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -d 'packages to exclude from search' complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -f -a path complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -d 'one or more alternative search paths for finding externals' -complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command external find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command external find' -l all -f -a all complete -c spack -n '__fish_spack_using_command external find' -l all -d 'search for all packages that Spack knows about' @@ -2420,7 +2420,7 @@ set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsig complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use ``--type binary --type source`` (default)' @@ -2456,7 +2456,7 @@ set -g __fish_spack_optspecs_spack_mirror_remove h/help scope= all-scopes complete -c spack -n '__fish_spack_using_command_pos 0 mirror remove' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror remove' -l all-scopes -f -a all_scopes complete -c spack -n '__fish_spack_using_command mirror remove' -l all-scopes -d 'remove from all config scopes (default: highest scope with matching mirror)' @@ -2466,7 +2466,7 @@ set -g __fish_spack_optspecs_spack_mirror_rm h/help scope= all-scopes complete -c spack -n '__fish_spack_using_command_pos 0 mirror rm' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror rm' -l all-scopes -f -a all_scopes complete -c spack -n '__fish_spack_using_command mirror rm' -l all-scopes -d 'remove from all config scopes (default: highest scope with matching mirror)' @@ -2480,7 +2480,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -f -a p complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -d 'set only the URL used for uploading' complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -f -a fetch complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -d 'set only the URL used for downloading' -complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2522,7 +2522,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a s complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache' complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed complete -c spack -n '__fish_spack_using_command mirror set' -l signed -d 'require signing and signature verification when pushing and installing from this build cache' -complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2547,14 +2547,14 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password-var set -g __fish_spack_optspecs_spack_mirror_list h/help scope= complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'configuration scope to read from' # spack mirror ls set -g __fish_spack_optspecs_spack_mirror_ls h/help scope= complete -c spack -n '__fish_spack_using_command mirror ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -d 'configuration scope to read from' # spack module @@ -2867,7 +2867,7 @@ complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirecto set -g __fish_spack_optspecs_spack_repo_list h/help scope= names namespaces json complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command repo list' -l names -f -a names complete -c spack -n '__fish_spack_using_command repo list' -l names -d 'show configuration names only' @@ -2880,7 +2880,7 @@ complete -c spack -n '__fish_spack_using_command repo list' -l json -d 'output r set -g __fish_spack_optspecs_spack_repo_ls h/help scope= names namespaces json complete -c spack -n '__fish_spack_using_command repo ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command repo ls' -l names -f -a names complete -c spack -n '__fish_spack_using_command repo ls' -l names -d 'show configuration names only' @@ -2898,7 +2898,7 @@ complete -c spack -n '__fish_spack_using_command repo add' -l name -r -f -a name complete -c spack -n '__fish_spack_using_command repo add' -l name -r -d 'config name for the package repository, defaults to the namespace of the repository' complete -c spack -n '__fish_spack_using_command repo add' -l path -r -f -a path complete -c spack -n '__fish_spack_using_command repo add' -l path -r -d 'relative path to the Spack package repository inside a git repository. Can be repeated to add multiple package repositories in case of a monorepo' -complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -d 'configuration scope to modify' # spack repo set @@ -2910,7 +2910,7 @@ complete -c spack -n '__fish_spack_using_command repo set' -l destination -r -f complete -c spack -n '__fish_spack_using_command repo set' -l destination -r -d 'destination to clone git repository into' complete -c spack -n '__fish_spack_using_command repo set' -l path -r -f -a path complete -c spack -n '__fish_spack_using_command repo set' -l path -r -d 'relative path to the Spack package repository inside a git repository. Can be repeated to add multiple package repositories in case of a monorepo' -complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -d 'configuration scope to modify' # spack repo remove @@ -2918,7 +2918,7 @@ set -g __fish_spack_optspecs_spack_repo_remove h/help scope= all-scopes complete -c spack -n '__fish_spack_using_command_pos 0 repo remove' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command repo remove' -l all-scopes -f -a all_scopes complete -c spack -n '__fish_spack_using_command repo remove' -l all-scopes -d 'remove from all config scopes (default: highest scope with matching repo)' @@ -2928,7 +2928,7 @@ set -g __fish_spack_optspecs_spack_repo_rm h/help scope= all-scopes complete -c spack -n '__fish_spack_using_command_pos 0 repo rm' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command repo rm' -l all-scopes -f -a all_scopes complete -c spack -n '__fish_spack_using_command repo rm' -l all-scopes -d 'remove from all config scopes (default: highest scope with matching repo)' @@ -2950,7 +2950,7 @@ complete -c spack -n '__fish_spack_using_command repo update' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command repo update' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command repo update' -l remote -s r -r -f -a remote complete -c spack -n '__fish_spack_using_command repo update' -l remote -s r -r -d 'name of remote to check for branches, tags, or commits' -complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -f -a '_builtin defaults:base defaults system site backwards-compat user spack command_line' +complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command repo update' -l branch -s b -r -f -a branch complete -c spack -n '__fish_spack_using_command repo update' -l branch -s b -r -d 'name of a branch to change to' From aade2ff3857ed50412b57a49fd35c94ca9d84130 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 14 Apr 2026 13:21:24 -0700 Subject: [PATCH 416/506] add test for warning when bypassing old config Signed-off-by: Peter Scheibel --- lib/spack/spack/paths.py | 7 +++++-- lib/spack/spack/test/config.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 05d09ea3fed3e3..9e1753c9471040 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -203,7 +203,7 @@ def _warn_on_unused_old_data( return msg return "" - def _warn_unused_old_config(self): + def _warn_unused_old_config(self, _show=True): user_scope = [s for s in config.CONFIG.active_scopes if s.name == "user"] if not user_scope: @@ -227,7 +227,10 @@ def _warn_unused_old_config(self): f" or move files from '{old_path}' to '{default_path}'" " to suppress this warning." ) - tty.warn(msg) + if _show: + tty.warn(msg) + else: + return msg @property def default_envs_path(self): diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 2a9b2e7d1604ef..44f7c2f4014db3 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -19,6 +19,8 @@ import spack.environment as ev import spack.error import spack.llnl.util.filesystem as fs +import spack.paths +import spack.paths_base import spack.platforms import spack.schema.compilers import spack.schema.config @@ -2083,3 +2085,20 @@ def test_config_invalid_scope(mock_low_high_config): err = "Must be one of \\['low', 'high'\\]" # noqa: W605 with pytest.raises(ValueError, match=err): spack.config.CONFIG.get_config_filename("noscope", "nosection") + + +def test_unused_old_cfg_warning(set_home, tmp_path_factory): + base_prefix = tmp_path_factory.mktemp("home-prefix") + home_prefix = tmp_path_factory.mktemp("home-prefix") + set_home(str(home_prefix)) + + (home_prefix / ".spack").mkdir() + (home_prefix / ".spack" / "some-file").touch() + + default_usr_scope_dir = home_prefix / ".config" / "spack" + test_cfg = [spack.config.DirectoryConfigScope("user", str(default_usr_scope_dir))] + + with spack.config.use_configuration(*test_cfg) as config: + p1 = spack.paths.SpackPaths(spack.paths_base.SpackPathsBase(str(base_prefix))) + warning = p1._warn_unused_old_config(_show=False) + assert warning and "Detected config in old location" in warning From 561f6321c02fb643c986395968516f3df28c1f2e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 14 Apr 2026 13:36:12 -0700 Subject: [PATCH 417/506] more cases tested Signed-off-by: Peter Scheibel --- lib/spack/spack/test/config.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 44f7c2f4014db3..21fac068327239 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -2092,13 +2092,27 @@ def test_unused_old_cfg_warning(set_home, tmp_path_factory): home_prefix = tmp_path_factory.mktemp("home-prefix") set_home(str(home_prefix)) - (home_prefix / ".spack").mkdir() - (home_prefix / ".spack" / "some-file").touch() - default_usr_scope_dir = home_prefix / ".config" / "spack" - test_cfg = [spack.config.DirectoryConfigScope("user", str(default_usr_scope_dir))] + test_cfg1 = [spack.config.DirectoryConfigScope("user", str(default_usr_scope_dir))] - with spack.config.use_configuration(*test_cfg) as config: + with spack.config.use_configuration(*test_cfg1): + # Start with the default user scope pointing to the 1.2 location, + # in ~/.config/spack. ~/.spack is empty, so no warning should be + # generated p1 = spack.paths.SpackPaths(spack.paths_base.SpackPathsBase(str(base_prefix))) + assert not p1._warn_unused_old_config(_show=False) + + # Now make it so ~/.spack is not empty: a warning should be + # generated + (home_prefix / ".spack").mkdir() + (home_prefix / ".spack" / "some-file").touch() warning = p1._warn_unused_old_config(_show=False) assert warning and "Detected config in old location" in warning + + # Simulate a scenario where the user edits their include.yaml to point the user + # scope at the old location (no warning should be generated) + custom_usr_scope_dir = home_prefix / ".scope" + test_cfg2 = [spack.config.DirectoryConfigScope("user", str(custom_usr_scope_dir))] + with spack.config.use_configuration(*test_cfg2): + p3 = spack.paths.SpackPaths(spack.paths_base.SpackPathsBase(str(base_prefix))) + assert not p3._warn_unused_old_config(_show=False) From 8b6e5c45e8f5998cc27026a636bfe38b435c17d2 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 14 Apr 2026 13:48:04 -0700 Subject: [PATCH 418/506] another case Signed-off-by: Peter Scheibel --- lib/spack/spack/test/config.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 21fac068327239..340d35205fab4d 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -2094,7 +2094,6 @@ def test_unused_old_cfg_warning(set_home, tmp_path_factory): default_usr_scope_dir = home_prefix / ".config" / "spack" test_cfg1 = [spack.config.DirectoryConfigScope("user", str(default_usr_scope_dir))] - with spack.config.use_configuration(*test_cfg1): # Start with the default user scope pointing to the 1.2 location, # in ~/.config/spack. ~/.spack is empty, so no warning should be @@ -2109,10 +2108,14 @@ def test_unused_old_cfg_warning(set_home, tmp_path_factory): warning = p1._warn_unused_old_config(_show=False) assert warning and "Detected config in old location" in warning - # Simulate a scenario where the user edits their include.yaml to point the user - # scope at the old location (no warning should be generated) - custom_usr_scope_dir = home_prefix / ".scope" - test_cfg2 = [spack.config.DirectoryConfigScope("user", str(custom_usr_scope_dir))] + # If the scope isn't called "user" -> no problem + test_cfg2 = [spack.config.DirectoryConfigScope("customuser", str(default_usr_scope_dir))] with spack.config.use_configuration(*test_cfg2): - p3 = spack.paths.SpackPaths(spack.paths_base.SpackPathsBase(str(base_prefix))) - assert not p3._warn_unused_old_config(_show=False) + assert not p1._warn_unused_old_config(_show=False) + + # If the scope is called "user" but was edited to point at the old location + # -> no problem + custom_usr_scope_dir = home_prefix / ".scope" + test_cfg3 = [spack.config.DirectoryConfigScope("user", str(custom_usr_scope_dir))] + with spack.config.use_configuration(*test_cfg3): + assert not p1._warn_unused_old_config(_show=False) From 684e69df8ee1eca844a759f830e3329de4925818 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 16 Apr 2026 12:55:32 -0700 Subject: [PATCH 419/506] new logic: if any old dir is used, we keep the old layout entirely (including gpg keys, install paths); new checkouts will write into new (shared) paths; tests are partially updated Signed-off-by: Peter Scheibel --- lib/spack/spack/paths.py | 34 ++++++++++++++++------- lib/spack/spack/test/paths.py | 51 +++++++++++++++-------------------- 2 files changed, 47 insertions(+), 38 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 9e1753c9471040..b3450f97df5653 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -23,7 +23,12 @@ def dir_is_occupied(x, except_for=None): x = pathlib.Path(x) except_for = except_for or set() - return x.is_dir() and bool(set(x.iterdir()) - except_for) + if not x.is_dir(): + return False + for path in x.iterdir(): + if path.parts[-1] not in except_for: + return True + return False class Provenance(Enum): @@ -90,6 +95,8 @@ def __init__(self, base): os.path.expanduser("~"), SpackPaths.relative_data_home, "spack" ) + self.old_layout = detect_old_spack_layout(base) + @property def state_home(self): if not self._state_home: @@ -393,19 +400,28 @@ def env_check(env_vars, provenance, rel=None): def _decide_old_or_new_location( self, old_location, new_location, default_new_location, provenance ): - if dir_is_occupied(new_location) or provenance.unilateral_override(): - return new_location - elif dir_is_occupied(default_new_location): - # This can occur e.g. if someone clones a new instance of spack, - # which would write into the default new location, and then later - # they set XDG_DATA_HOME - return new_location - elif dir_is_occupied(old_location): + if self.old_layout: return old_location else: return new_location +def detect_old_spack_layout(paths: paths_base.SpackPathsBase): + checks = [ + # It's important if this directory is occupied but we have a separate + # check for that, so exclude that directory here + (paths.old_install_path, ["gpg"]), + (paths.old_envs_path, []), + (paths.old_fetch_cache_path, []), + (paths.old_gpg_path, []), + (paths.old_gpg_keys_path, ["README.md"]), + ] + for x, y in checks: + if dir_is_occupied(x, except_for=set(y)): + return True + return False + + locations = SpackPaths(paths_base.locations) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index b72e5c810bb014..45f2599940efe1 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -72,38 +72,34 @@ def paths_base_nonempty_old_install(): set_home(home_prefix) - # The new default installs dir is ignored if it is empty and - # the old install location has anything in it + # The new default installs dir is ignored if the old install + # location has anything in it p1 = SpackPaths(paths_base_nonempty_old_install()) assert p1.default_install_location == nonempty_dir - # If there are spack installs in the new installs dir, it is - # preferred over the old dir + # This is continues to hold even if the new default install + # directory has installs in it new_default_installs_dir = _ensure_dir( pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs" ) (pathlib.Path(new_default_installs_dir) / "afile").touch() - assert p1.default_install_location == new_default_installs_dir - warning_msg1 = p1.warn_unused_old_installs(_show=False) - assert f"Bypassing data for installs existing in: {p1.base.old_install_path}" in warning_msg1 - assert f"In favor of: {new_default_installs_dir}" in warning_msg1 + assert p1.default_install_location == nonempty_dir + + # TODO: no warning here spack.config.set("config:locations", {}) - # XDG_DATA_HOME overrides the old install location *even if it is empty - # and the old install location is not*, if there are installs in the new - # default location + # XDG_DATA_HOME does not override the old install location if it + # if there are installs in the old location xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") os.environ["XDG_DATA_HOME"] = xdg_data_home p4 = SpackPaths(paths_base_nonempty_old_install()) xdg_installs_location = _ensure_dir(pathlib.Path(xdg_data_home) / "spack" / "installs") - assert p4.default_install_location == str(xdg_installs_location) - warning_msg4 = p4.warn_unused_old_installs(_show=False) - assert f"In favor of: {xdg_installs_location}" in warning_msg4 + assert p4.default_install_location == nonempty_dir - # (sanity) XDG_DATA_HOME still overrides when there is something in it + # ... XDG_DATA_HOME still does not override the old install location (pathlib.Path(xdg_installs_location) / "afile").touch() - assert p4.default_install_location == str(xdg_installs_location) + assert p4.default_install_location == nonempty_dir _unconditional_path_override_checks(tmp_path, paths_base_nonempty_old_install) @@ -196,6 +192,7 @@ def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmp_path, set_home): new_default_gpg_base = pathlib.Path(base_prefix) / ".local" / "share" / "spack" + # Nothing in any of the old locations: we should use the new one p1 = SpackPaths(SpackPathsBase(base_prefix)) assert p1.gpg_path == str(new_default_gpg_base / "gpg") assert p1.gpg_keys_path == str(new_default_gpg_base / "gpg-keys") @@ -203,7 +200,7 @@ def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmp_path, set_home): old_gpg_dir = pathlib.Path(base_prefix) / "opt" / "spack" / "gpg" (old_gpg_dir).mkdir(parents=True) p1 = SpackPaths(SpackPathsBase(base_prefix)) - # Old dir exists, but is empty, so it should not be used + # Old dir exists, but is empty, so it should still not be used assert p1.gpg_path == str(new_default_gpg_base / "gpg") # Put something in the old dir: it should now redirect @@ -211,21 +208,17 @@ def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmp_path, set_home): p1 = SpackPaths(SpackPathsBase(base_prefix)) assert p1.gpg_path == str(old_gpg_dir) - # But the keys are handled separately and should use the new path - new_gpg_keys_dir = pathlib.Path(new_default_gpg_base / "gpg-keys") - assert p1.gpg_keys_path == str(new_gpg_keys_dir) - - # Check that the keys will also redirect + # Old keys path is used if old gpg path is used: all data is + # relocated together old_gpg_keys_dir = pathlib.Path(base_prefix) / "var" / "spack" / "gpg" - old_gpg_keys_dir.mkdir(parents=True) - (old_gpg_keys_dir / "something").touch() - p1 = SpackPaths(SpackPathsBase(base_prefix)) assert p1.gpg_keys_path == str(old_gpg_keys_dir) - # When something is in both the new and old locations, prefer the new - new_gpg_keys_dir.mkdir(parents=True) - (new_gpg_keys_dir / "something").touch() - assert p1.gpg_keys_path == str(new_gpg_keys_dir) + # When something is in both the new and old locations, prefer the old + new_gpg_dir = new_default_gpg_base / "gpg" + new_gpg_dir.mkdir(parents=True) + (new_gpg_dir / "something").touch() + p1 = SpackPaths(SpackPathsBase(base_prefix)) + assert p1.gpg_keys_path == str(old_gpg_keys_dir) # And the gpg dir itself remains the old dir: reaffirm that assert p1.gpg_path == str(old_gpg_dir) From ba87fd6299151b5229d71e3b469f5f79f9423f35 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 16 Apr 2026 13:07:58 -0700 Subject: [PATCH 420/506] remove all warnings about old unused paths (because if they exist, they are now getting used) Signed-off-by: Peter Scheibel --- lib/spack/spack/environment/environment.py | 1 - lib/spack/spack/paths.py | 52 ---------------------- lib/spack/spack/store.py | 1 - lib/spack/spack/test/paths.py | 1 - lib/spack/spack/util/gpg.py | 1 - 5 files changed, 56 deletions(-) diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index e4635a879c060f..97a81e62b49959 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -102,7 +102,6 @@ def default_env_path(): global _default_env_path if not _default_env_path: _default_env_path = paths.default_envs_path - paths.warn_unused_old_envs() return _default_env_path diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index b3450f97df5653..5205690810796c 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -158,58 +158,6 @@ def default_install_location(self): self.data_home_provenance, ) - def warn_unused_old_installs(self, _show=True): - cfg_settings = ["config:install_tree:root"] - return self._warn_on_unused_old_data( - self.default_install_location, - self.base.old_install_path, - "installs", - cfg_settings, - _show=_show, - ) - - def warn_unused_old_envs(self, _show=True): - cfg_settings = ["config:environments_root"] - return self._warn_on_unused_old_data( - self.default_envs_path, - self.base.old_envs_path, - "environments", - cfg_settings, - _show=_show, - ) - - def warn_unused_old_gpg(self, _show=True): - cfg_settings = ["SPACK_GNUPGHOME"] - return self._warn_on_unused_old_data( - self.gpg_path, self.base.old_gpg_path, "GPG keys", cfg_settings, _show=_show - ) - - def _warn_on_unused_old_data( - self, chosen_path, old_path, data_category, cfg_settings, _show=True - ): - generic_cfg = [ - "config:locations:home", - "config:locations:data", - "SPACK_DATA_HOME", - "SPACK_HOME", - ] - msg = ( - f"Bypassing data for {data_category} existing in: {old_path}" - f"\nIn favor of: {chosen_path}" - "\nYou can explicitly designate a location (and suppress this warning)" - f" by setting: {', '.join(cfg_settings)}" - f"\nOr use a catch-all setting: {', '.join(generic_cfg)}" - ) - if ( - chosen_path != old_path - and dir_is_occupied(old_path) - and not self.data_home_provenance.unilateral_override() - ): - if _show: - tty.warn(msg) - return msg - return "" - def _warn_unused_old_config(self, _show=True): user_scope = [s for s in config.CONFIG.active_scopes if s.name == "user"] diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 7095d1f8b4c426..4e566a9e345af4 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -78,7 +78,6 @@ def parse_install_tree(config_dict: dict) -> Tuple[str, str, Dict[str, str]]: unpadded_root = install_tree.get("root", None) if not unpadded_root: unpadded_root = paths.default_install_location - paths.warn_unused_old_installs() unpadded_root = spack.util.path.canonicalize_path(unpadded_root) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 45f2599940efe1..48997b9c1087c8 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -113,7 +113,6 @@ def _unconditional_path_override_checks(tmp_path, base_paths_generator): assert p2.default_install_location == str( pathlib.Path(spack_home_cfg_prefix) / ".local" / "share" / "spack" / "installs" ) - assert not p2.warn_unused_old_installs(_show=False) # "config:locations:data" overrides the above spack_data_prefix = _ensure_dir(tmp_path / "spack-data") diff --git a/lib/spack/spack/util/gpg.py b/lib/spack/spack/util/gpg.py index 4db19cc0437e76..f94ca296aa09fa 100644 --- a/lib/spack/spack/util/gpg.py +++ b/lib/spack/spack/util/gpg.py @@ -60,7 +60,6 @@ def init(gnupghome=None, force=False): GNUPGHOME = gnupghome or os.getenv("SPACK_GNUPGHOME") if not GNUPGHOME: GNUPGHOME = paths.gpg_path - paths.warn_unused_old_gpg() # Set the executable objects for "gpg" and "gpgconf" with spack.bootstrap.ensure_bootstrap_configuration(): From 627d015f987b57b00fd068ddf4492cb347e64f70 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 16 Apr 2026 13:25:31 -0700 Subject: [PATCH 421/506] tests are now updated with new logic (which favors old paths more) Signed-off-by: Peter Scheibel --- lib/spack/spack/test/paths.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 48997b9c1087c8..22d3b6c1ba60d8 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -53,7 +53,7 @@ def paths_base_empty_old_install(): p4 = SpackPaths(paths_base_empty_old_install()) assert p4.default_install_location == str(pathlib.Path(xdg_data_home) / "spack" / "installs") - _unconditional_path_override_checks(tmp_path, paths_base_empty_old_install) + _unconditional_path_override_checks(tmp_path, paths_base_empty_old_install, False) def test_install_location_old_installs_exist(working_env, tmp_path, mutable_config, set_home): @@ -85,8 +85,6 @@ def paths_base_nonempty_old_install(): (pathlib.Path(new_default_installs_dir) / "afile").touch() assert p1.default_install_location == nonempty_dir - # TODO: no warning here - spack.config.set("config:locations", {}) # XDG_DATA_HOME does not override the old install location if it @@ -101,45 +99,47 @@ def paths_base_nonempty_old_install(): (pathlib.Path(xdg_installs_location) / "afile").touch() assert p4.default_install_location == nonempty_dir - _unconditional_path_override_checks(tmp_path, paths_base_nonempty_old_install) + _unconditional_path_override_checks(tmp_path, paths_base_nonempty_old_install, True) + +def _unconditional_path_override_checks(tmp_path, base_paths_generator, force_old_layout): + def check(paths, new_path): + if force_old_layout: + assert paths.default_install_location == paths.base.old_install_path + else: + assert paths.default_install_location == str(new_path) -def _unconditional_path_override_checks(tmp_path, base_paths_generator): # "config:locations:home" variable overrides the above (even if there # are no installs there and there are installs in the old location) spack_home_cfg_prefix = _ensure_dir(tmp_path / "spack-home2") spack.config.set("config:locations:home", spack_home_cfg_prefix) p2 = SpackPaths(base_paths_generator()) - assert p2.default_install_location == str( - pathlib.Path(spack_home_cfg_prefix) / ".local" / "share" / "spack" / "installs" - ) + check(p2, pathlib.Path(spack_home_cfg_prefix) / ".local" / "share" / "spack" / "installs") # "config:locations:data" overrides the above spack_data_prefix = _ensure_dir(tmp_path / "spack-data") spack.config.set("config:locations:data", spack_data_prefix) p3 = SpackPaths(base_paths_generator()) - assert p3.default_install_location == str(pathlib.Path(spack_data_prefix) / "installs") + check(p3, pathlib.Path(spack_data_prefix) / "installs") # SPACK_HOME env variable overrides the above (even if there # are no installs there and there are installs in the old location) spack_home_env_prefix = _ensure_dir(tmp_path / "spack-home1") os.environ["SPACK_HOME"] = spack_home_env_prefix p1 = SpackPaths(base_paths_generator()) - assert p1.default_install_location == str( - pathlib.Path(spack_home_env_prefix) / ".local" / "share" / "spack" / "installs" - ) + check(p1, pathlib.Path(spack_home_env_prefix) / ".local" / "share" / "spack" / "installs") # Check that $SPACK_DATA_HOME overrides all the above spack_data_home = _ensure_dir(tmp_path / "spack_data_home") os.environ["SPACK_DATA_HOME"] = spack_data_home p5 = SpackPaths(base_paths_generator()) - assert p5.default_install_location == str(pathlib.Path(spack_data_home) / "installs") + check(p5, pathlib.Path(spack_data_home) / "installs") # Disable all location-based env vars: this will then defer # to using "config:locations:data" spack.config.set("config:locations:disable_env", True) p6 = SpackPaths(base_paths_generator()) - assert p6.default_install_location == str(pathlib.Path(spack_data_prefix) / "installs") + check(p6, pathlib.Path(spack_data_prefix) / "installs") def test_system_config_path_is_overridable(working_env, tmp_path): From 66dd597bc5e39f5ce068e8f6393fb39407f56710 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 16 Apr 2026 13:35:34 -0700 Subject: [PATCH 422/506] change function name (no longer accurate) Signed-off-by: Peter Scheibel --- lib/spack/spack/test/paths.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 22d3b6c1ba60d8..3180c233b2fdba 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -53,7 +53,7 @@ def paths_base_empty_old_install(): p4 = SpackPaths(paths_base_empty_old_install()) assert p4.default_install_location == str(pathlib.Path(xdg_data_home) / "spack" / "installs") - _unconditional_path_override_checks(tmp_path, paths_base_empty_old_install, False) + _install_path_checks(tmp_path, paths_base_empty_old_install, False) def test_install_location_old_installs_exist(working_env, tmp_path, mutable_config, set_home): @@ -99,10 +99,10 @@ def paths_base_nonempty_old_install(): (pathlib.Path(xdg_installs_location) / "afile").touch() assert p4.default_install_location == nonempty_dir - _unconditional_path_override_checks(tmp_path, paths_base_nonempty_old_install, True) + _install_path_checks(tmp_path, paths_base_nonempty_old_install, True) -def _unconditional_path_override_checks(tmp_path, base_paths_generator, force_old_layout): +def _install_path_checks(tmp_path, base_paths_generator, force_old_layout): def check(paths, new_path): if force_old_layout: assert paths.default_install_location == paths.base.old_install_path From 4e3ad241972f9caa82de0618937aa281f8574301 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 16 Apr 2026 17:12:11 -0700 Subject: [PATCH 423/506] init logic for backwards_compat from/to Signed-off-by: Peter Scheibel --- etc/spack/include.yaml | 3 +++ lib/spack/spack/config.py | 38 +++++++++++++++++++++++++++++++ lib/spack/spack/schema/include.py | 9 ++++++++ lib/spack/spack/test/config.py | 24 +++++++++++++++++++ 4 files changed, 74 insertions(+) diff --git a/etc/spack/include.yaml b/etc/spack/include.yaml index b8ab4ebfeb3b8b..b827bd02c232b8 100644 --- a/etc/spack/include.yaml +++ b/etc/spack/include.yaml @@ -3,6 +3,9 @@ include: - name: "user" path_override_env_var: SPACK_USER_CONFIG_PATH path: "~/.config/spack/" + backwards_compat: + from: "~/.spack/" + to: "~/.config/spack/" optional: true prefer_modify: true when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 1933afb2b906b0..1710012c41be88 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1256,6 +1256,32 @@ def paths(self) -> List[str]: raise NotImplementedError("must be implemented in derived classes") +_yaml_exts = ".yaml", ".yml" +is_yaml = lambda x: (x.suffix in _yaml_exts) + + +import shutil + + +def there_are_yaml_files_in(_dir): + for path in pathlib.Path(_dir).rglob("*"): + if is_yaml(path): + return True + return False + + +def copy_yaml_files(_from, to): + _from = pathlib.Path(_from) + to = pathlib.Path(to) + to.mkdir(parents=True) + for path in _from.rglob("*"): + if is_yaml(path): + rel_from = path.relative_to(_from) + abs_to = to / rel_from + abs_to.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, abs_to) + + class IncludePath(OptionalInclude): path: str sha256: str @@ -1273,6 +1299,18 @@ def __init__(self, entry: dict): context = f"{context_prefix}{path}" self.path = substitute_include_path(path, context) + backwards_compat = entry.get("backwards_compat", {}) + if backwards_compat: + _from = substitute_include_path(backwards_compat.get("from"), context) + to = substitute_include_path(backwards_compat.get("to"), context) + if not os.path.exists(_from): + os.mkdir(to) + elif to == self.path: + if not os.path.exists(to): + if there_are_yaml_files_in(_from): + tty.debug(f"Generating initial config in {to} from {_from}") + copy_yaml_files(_from, to) + self.sha256 = entry.get("sha256", "") self.remote = "sha256" in entry self.destination = None diff --git a/lib/spack/spack/schema/include.py b/lib/spack/spack/schema/include.py index 1cc1fcb4c13e4a..d62b9297beeff3 100644 --- a/lib/spack/spack/schema/include.py +++ b/lib/spack/spack/schema/include.py @@ -44,6 +44,15 @@ "raw form). Supports file, ftp, http, https schemes and " "Spack/environment variables", }, + "backwards_compat": { + "type": "object", + "description": "For default scopes, old paths where they may be " + "found, if they have moved", + "properties": { + "from": {"type": "string"}, + "to": {"type": "string"}, + } + }, "sha256": { "type": "string", "description": "Required SHA256 hash for remote URLs to verify " diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 340d35205fab4d..08b2080ef38235 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1715,6 +1715,30 @@ def test_included_path_conditional_bad_when( assert not scopes +def test_copy_backwards_compat( + tmp_path: pathlib.Path +): + src_dir = tmp_path / "src" + sub_dir = src_dir / "subdir" + sub_dir.mkdir(parents=True) + (sub_dir / "example.yaml").touch() + + to_dir = tmp_path / "to" + + entry = { + "path": str(to_dir), + "optional": True, + "backwards_compat": { + "from": str(src_dir), + "to": str(to_dir), + } + } + include = spack.config.included_path(entry) + + assert to_dir.exists() + assert (to_dir / "subdir" / "example.yaml").exists() + + def test_included_path_conditional_success(tmp_path: pathlib.Path, mock_low_high_config): path = tmp_path / "local" path.mkdir() From eb4da1c7a56377be4061c677f7500861fb7aa5a6 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 16 Apr 2026 18:56:56 -0700 Subject: [PATCH 424/506] competing processes case Signed-off-by: Peter Scheibel --- lib/spack/spack/config.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 1710012c41be88..7dc6409658a8cf 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1273,7 +1273,21 @@ def there_are_yaml_files_in(_dir): def copy_yaml_files(_from, to): _from = pathlib.Path(_from) to = pathlib.Path(to) - to.mkdir(parents=True) + tmpdir = pathlib.Path(tempfile.mkdtemp(prefix=f".{to.name}.tmp-", dir=to.parent)) + + try: + _copy_yaml_files(_from, tmpdir) + except Exception as e: + shutil.rmtree(tmpdir, ignore_errors=True) + raise ConfigFileError(str(e)) + + try: + tmpdir.rename(to) + except FileExistsError: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def _copy_yaml_files(_from, to): for path in _from.rglob("*"): if is_yaml(path): rel_from = path.relative_to(_from) From 04f67ad38ca69349d2db1186d58b4b7e4c51c459 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 16 Apr 2026 18:59:50 -0700 Subject: [PATCH 425/506] style edit Signed-off-by: Peter Scheibel --- lib/spack/spack/config.py | 4 +--- lib/spack/spack/schema/include.py | 5 +---- lib/spack/spack/test/config.py | 11 +++-------- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 7dc6409658a8cf..bac8a2ed92c880 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -33,6 +33,7 @@ import os.path import pathlib import re +import shutil import sys import tempfile from collections import defaultdict @@ -1260,9 +1261,6 @@ def paths(self) -> List[str]: is_yaml = lambda x: (x.suffix in _yaml_exts) -import shutil - - def there_are_yaml_files_in(_dir): for path in pathlib.Path(_dir).rglob("*"): if is_yaml(path): diff --git a/lib/spack/spack/schema/include.py b/lib/spack/spack/schema/include.py index d62b9297beeff3..ae4330e9c1d7a6 100644 --- a/lib/spack/spack/schema/include.py +++ b/lib/spack/spack/schema/include.py @@ -48,10 +48,7 @@ "type": "object", "description": "For default scopes, old paths where they may be " "found, if they have moved", - "properties": { - "from": {"type": "string"}, - "to": {"type": "string"}, - } + "properties": {"from": {"type": "string"}, "to": {"type": "string"}}, }, "sha256": { "type": "string", diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 08b2080ef38235..6da53a51715050 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1715,9 +1715,7 @@ def test_included_path_conditional_bad_when( assert not scopes -def test_copy_backwards_compat( - tmp_path: pathlib.Path -): +def test_copy_backwards_compat(tmp_path: pathlib.Path): src_dir = tmp_path / "src" sub_dir = src_dir / "subdir" sub_dir.mkdir(parents=True) @@ -1728,12 +1726,9 @@ def test_copy_backwards_compat( entry = { "path": str(to_dir), "optional": True, - "backwards_compat": { - "from": str(src_dir), - "to": str(to_dir), - } + "backwards_compat": {"from": str(src_dir), "to": str(to_dir)}, } - include = spack.config.included_path(entry) + spack.config.included_path(entry) assert to_dir.exists() assert (to_dir / "subdir" / "example.yaml").exists() From 88b52fe74de43c0bf252c1f14ed96688ef5eaa0e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 16 Apr 2026 22:49:01 -0700 Subject: [PATCH 426/506] remove warning: old user config scope location is automatically copied --- lib/spack/spack/main.py | 2 -- lib/spack/spack/paths.py | 30 ---------------------------- lib/spack/spack/test/config.py | 36 ---------------------------------- 3 files changed, 68 deletions(-) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index c8eedc696277ba..dbbccf600d51b1 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -1034,8 +1034,6 @@ def add_environment_scope(): ) setup_main_options(args) - spack.paths.locations._warn_unused_old_config() - # ------------------------------------------------------------------------ # Things that require configuration should go below here # ------------------------------------------------------------------------ diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 5205690810796c..08966cdb48ba19 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -16,7 +16,6 @@ from functools import partial import spack.config as config -import spack.llnl.util.tty as tty import spack.paths_base as paths_base @@ -158,35 +157,6 @@ def default_install_location(self): self.data_home_provenance, ) - def _warn_unused_old_config(self, _show=True): - user_scope = [s for s in config.CONFIG.active_scopes if s.name == "user"] - - if not user_scope: - return - - # Note: does not require spack.util.path.canonicalize_path because the - # default user scope path only uses "~" - default_path = pathlib.Path("~/.config/spack").expanduser().resolve() - scope_path = pathlib.Path(getattr(user_scope[0], "path", "")).expanduser().resolve() - if scope_path != default_path: - # User overrode the scope path, they meant it - return - - old_path = pathlib.Path("~/.spack").expanduser() - if dir_is_occupied(old_path): - msg = ( - f"Bypassing config in '{old_path}' in favor of '{default_path}'.\n" - f" Detected config in old location: '{default_path}'.\n" - " Set `SPACK_USER_CONFIG_PATH` or modify your include.yaml" - f" to use '{old_path}'\n" - f" or move files from '{old_path}' to '{default_path}'" - " to suppress this warning." - ) - if _show: - tty.warn(msg) - else: - return msg - @property def default_envs_path(self): return self._decide_old_or_new_location( diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 6da53a51715050..a39ecee36257f8 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -19,8 +19,6 @@ import spack.environment as ev import spack.error import spack.llnl.util.filesystem as fs -import spack.paths -import spack.paths_base import spack.platforms import spack.schema.compilers import spack.schema.config @@ -2104,37 +2102,3 @@ def test_config_invalid_scope(mock_low_high_config): err = "Must be one of \\['low', 'high'\\]" # noqa: W605 with pytest.raises(ValueError, match=err): spack.config.CONFIG.get_config_filename("noscope", "nosection") - - -def test_unused_old_cfg_warning(set_home, tmp_path_factory): - base_prefix = tmp_path_factory.mktemp("home-prefix") - home_prefix = tmp_path_factory.mktemp("home-prefix") - set_home(str(home_prefix)) - - default_usr_scope_dir = home_prefix / ".config" / "spack" - test_cfg1 = [spack.config.DirectoryConfigScope("user", str(default_usr_scope_dir))] - with spack.config.use_configuration(*test_cfg1): - # Start with the default user scope pointing to the 1.2 location, - # in ~/.config/spack. ~/.spack is empty, so no warning should be - # generated - p1 = spack.paths.SpackPaths(spack.paths_base.SpackPathsBase(str(base_prefix))) - assert not p1._warn_unused_old_config(_show=False) - - # Now make it so ~/.spack is not empty: a warning should be - # generated - (home_prefix / ".spack").mkdir() - (home_prefix / ".spack" / "some-file").touch() - warning = p1._warn_unused_old_config(_show=False) - assert warning and "Detected config in old location" in warning - - # If the scope isn't called "user" -> no problem - test_cfg2 = [spack.config.DirectoryConfigScope("customuser", str(default_usr_scope_dir))] - with spack.config.use_configuration(*test_cfg2): - assert not p1._warn_unused_old_config(_show=False) - - # If the scope is called "user" but was edited to point at the old location - # -> no problem - custom_usr_scope_dir = home_prefix / ".scope" - test_cfg3 = [spack.config.DirectoryConfigScope("user", str(custom_usr_scope_dir))] - with spack.config.use_configuration(*test_cfg3): - assert not p1._warn_unused_old_config(_show=False) From ff0e9e419720283284f3ecb52d172aa26f3dbed5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 16 Apr 2026 22:59:22 -0700 Subject: [PATCH 427/506] extra test Signed-off-by: Peter Scheibel --- lib/spack/spack/test/config.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index a39ecee36257f8..f6b6dc18a472dc 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1713,12 +1713,13 @@ def test_included_path_conditional_bad_when( assert not scopes -def test_copy_backwards_compat(tmp_path: pathlib.Path): +def test_backwards_compat_use_old(tmp_path: pathlib.Path): src_dir = tmp_path / "src" sub_dir = src_dir / "subdir" sub_dir.mkdir(parents=True) (sub_dir / "example.yaml").touch() + # Path doesn't exist yet to_dir = tmp_path / "to" entry = { @@ -1732,6 +1733,26 @@ def test_copy_backwards_compat(tmp_path: pathlib.Path): assert (to_dir / "subdir" / "example.yaml").exists() +def test_backwards_compat_skip_old(tmp_path: pathlib.Path): + src_dir = tmp_path / "src" + sub_dir = src_dir / "subdir" + sub_dir.mkdir(parents=True) + (sub_dir / "example.yaml").touch() + + to_dir = tmp_path / "to" + to_dir.mkdir() + + entry = { + "path": str(to_dir), + "optional": True, + "backwards_compat": {"from": str(src_dir), "to": str(to_dir)}, + } + spack.config.included_path(entry) + + assert to_dir.exists() + assert not (to_dir / "subdir" / "example.yaml").exists() + + def test_included_path_conditional_success(tmp_path: pathlib.Path, mock_low_high_config): path = tmp_path / "local" path.mkdir() From 640031be1450bd7aa8fb52fdbed862873ab09031 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 16 Apr 2026 23:04:12 -0700 Subject: [PATCH 428/506] consolidation of tests plus docstrings Signed-off-by: Peter Scheibel --- lib/spack/spack/test/config.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index f6b6dc18a472dc..d42e79ecbee1c5 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1713,7 +1713,8 @@ def test_included_path_conditional_bad_when( assert not scopes -def test_backwards_compat_use_old(tmp_path: pathlib.Path): +@pytest.fixture +def backwards_compat_setup(tmp_path: pathlib.Path): src_dir = tmp_path / "src" sub_dir = src_dir / "subdir" sub_dir.mkdir(parents=True) @@ -1727,28 +1728,25 @@ def test_backwards_compat_use_old(tmp_path: pathlib.Path): "optional": True, "backwards_compat": {"from": str(src_dir), "to": str(to_dir)}, } - spack.config.included_path(entry) + yield src_dir, to_dir, entry + + +def test_backwards_compat_use_old(backwards_compat_setup): + """If target dir doesn't exist, make it and copy over yaml files""" + src_dir, to_dir, entry = backwards_compat_setup + + spack.config.included_path(entry) assert to_dir.exists() assert (to_dir / "subdir" / "example.yaml").exists() -def test_backwards_compat_skip_old(tmp_path: pathlib.Path): - src_dir = tmp_path / "src" - sub_dir = src_dir / "subdir" - sub_dir.mkdir(parents=True) - (sub_dir / "example.yaml").touch() +def test_backwards_compat_skip_old(backwards_compat_setup): + """If target dir exists, copy over nothing from source""" + src_dir, to_dir, entry = backwards_compat_setup - to_dir = tmp_path / "to" to_dir.mkdir() - - entry = { - "path": str(to_dir), - "optional": True, - "backwards_compat": {"from": str(src_dir), "to": str(to_dir)}, - } spack.config.included_path(entry) - assert to_dir.exists() assert not (to_dir / "subdir" / "example.yaml").exists() From 9b117f841b2cc819d1577a592fb63f36068fb9d2 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 16 Apr 2026 23:06:35 -0700 Subject: [PATCH 429/506] doc tweak Signed-off-by: Peter Scheibel --- lib/spack/spack/test/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index d42e79ecbee1c5..96c67f515f8212 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1733,7 +1733,7 @@ def backwards_compat_setup(tmp_path: pathlib.Path): def test_backwards_compat_use_old(backwards_compat_setup): - """If target dir doesn't exist, make it and copy over yaml files""" + """If target dir doesn't exist, make it and copy over yaml files.""" src_dir, to_dir, entry = backwards_compat_setup spack.config.included_path(entry) @@ -1742,7 +1742,7 @@ def test_backwards_compat_use_old(backwards_compat_setup): def test_backwards_compat_skip_old(backwards_compat_setup): - """If target dir exists, copy over nothing from source""" + """If target dir exists, copy over nothing from source.""" src_dir, to_dir, entry = backwards_compat_setup to_dir.mkdir() From 923a38f173f14129bec357da74bb82c950d70f38 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 16 Apr 2026 23:33:50 -0700 Subject: [PATCH 430/506] more test consolidation Signed-off-by: Peter Scheibel --- lib/spack/spack/test/paths.py | 60 ++++++++++------------------------- 1 file changed, 17 insertions(+), 43 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 3180c233b2fdba..0fd35a43a181aa 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -40,20 +40,7 @@ def paths_base_empty_old_install(): set_home(home_prefix) - p1 = SpackPaths(paths_base_empty_old_install()) - assert p1.default_install_location == str( - pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs" - ) - - spack.config.set("config:locations", {}) - - # $XDG_DATA_HOME overrides the default - xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") - os.environ["XDG_DATA_HOME"] = xdg_data_home - p4 = SpackPaths(paths_base_empty_old_install()) - assert p4.default_install_location == str(pathlib.Path(xdg_data_home) / "spack" / "installs") - - _install_path_checks(tmp_path, paths_base_empty_old_install, False) + _install_path_checks(tmp_path, paths_base_empty_old_install, home_prefix, False) def test_install_location_old_installs_exist(working_env, tmp_path, mutable_config, set_home): @@ -72,45 +59,32 @@ def paths_base_nonempty_old_install(): set_home(home_prefix) - # The new default installs dir is ignored if the old install - # location has anything in it - p1 = SpackPaths(paths_base_nonempty_old_install()) - assert p1.default_install_location == nonempty_dir + _install_path_checks(tmp_path, paths_base_nonempty_old_install, home_prefix, True) + + +def _install_path_checks(tmp_path, base_paths_generator, home_prefix, force_old_layout): + def check(paths, new_path): + if force_old_layout: + assert paths.default_install_location == paths.base.old_install_path + else: + assert paths.default_install_location == str(new_path) - # This is continues to hold even if the new default install - # directory has installs in it new_default_installs_dir = _ensure_dir( pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs" ) (pathlib.Path(new_default_installs_dir) / "afile").touch() - assert p1.default_install_location == nonempty_dir - - spack.config.set("config:locations", {}) + p0 = SpackPaths(base_paths_generator()) + check(p0, pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs") - # XDG_DATA_HOME does not override the old install location if it - # if there are installs in the old location + # $XDG_DATA_HOME overrides the default xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") os.environ["XDG_DATA_HOME"] = xdg_data_home - p4 = SpackPaths(paths_base_nonempty_old_install()) - xdg_installs_location = _ensure_dir(pathlib.Path(xdg_data_home) / "spack" / "installs") - assert p4.default_install_location == nonempty_dir - - # ... XDG_DATA_HOME still does not override the old install location - (pathlib.Path(xdg_installs_location) / "afile").touch() - assert p4.default_install_location == nonempty_dir - - _install_path_checks(tmp_path, paths_base_nonempty_old_install, True) - + p7 = SpackPaths(base_paths_generator()) + check(p7, pathlib.Path(xdg_data_home) / "spack" / "installs") -def _install_path_checks(tmp_path, base_paths_generator, force_old_layout): - def check(paths, new_path): - if force_old_layout: - assert paths.default_install_location == paths.base.old_install_path - else: - assert paths.default_install_location == str(new_path) + spack.config.set("config:locations", {}) - # "config:locations:home" variable overrides the above (even if there - # are no installs there and there are installs in the old location) + # "config:locations:home" variable overrides the above spack_home_cfg_prefix = _ensure_dir(tmp_path / "spack-home2") spack.config.set("config:locations:home", spack_home_cfg_prefix) p2 = SpackPaths(base_paths_generator()) From 7fe797b00b2fa52d42e568ec50fa054d0ac7cb4e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 16 Apr 2026 23:56:27 -0700 Subject: [PATCH 431/506] partial handling of full use of old scheme (still need to use old gpg paths in default config) Signed-off-by: Peter Scheibel --- etc/spack/defaults/base/config.yaml | 2 +- lib/spack/spack/paths.py | 8 ++++++++ lib/spack/spack/paths_base.py | 1 + lib/spack/spack/util/path.py | 2 ++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index dacb5c92e1bd0f..1812370cfa5d51 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -35,7 +35,7 @@ config: - $spack/share/spack/templates # Directory where licenses should be located - license_dir: $data_home/licenses + license_dir: $default_license_dir # Temporary locations Spack can try to use for builds. # diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 08966cdb48ba19..f6e5ae931e80a1 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -166,6 +166,13 @@ def default_envs_path(self): self.data_home_provenance, ) + @property + def default_licenses_dir(self): + if self.old_layout: + return self.base.old_licenses_dir + else: + return os.path.join(self.data_home, "licenses") + @property def reports_path(self): #: junit, cdash, etc. reports about builds @@ -333,6 +340,7 @@ def detect_old_spack_layout(paths: paths_base.SpackPathsBase): (paths.old_fetch_cache_path, []), (paths.old_gpg_path, []), (paths.old_gpg_keys_path, ["README.md"]), + (paths.old_licenses_path, []), ] for x, y in checks: if dir_is_occupied(x, except_for=set(y)): diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index 7106fb6763c8f9..fe89f94b266956 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -54,6 +54,7 @@ def __init__(self, _prefix=None): self.old_fetch_cache_path = os.path.join(self.var_path, "cache") self.old_gpg_path = os.path.join(self.prefix, "opt", "spack", "gpg") self.old_gpg_keys_path = os.path.join(self.var_path, "gpg") + self.old_licenses_path = os.path.join(self.etc_path, "licenses") #: User configuration location self.user_config_path = os.path.expanduser( diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 78f9d18b3d9f4c..c11d22f696d3be 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -81,6 +81,8 @@ def replacements(): "user_cache_path": lambda: paths.user_cache_path, "default_install_root": lambda: paths.default_install_location, "default_envs_root": lambda: paths.default_envs_path, + "default_license_dir": lambda: paths.default_license_dir, + # TODO: for now need analogous values for gpg keys "data_home": lambda: paths.data_home, "cache_home": lambda: paths.cache_home, "state_home": lambda: paths.state_home, From 412c5d7f3d82cf9a26958d2fe75f2e92ac8b92a1 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 17 Apr 2026 00:06:39 -0700 Subject: [PATCH 432/506] existence check Signed-off-by: Peter Scheibel --- lib/spack/spack/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index bac8a2ed92c880..9c332a8c059745 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1316,7 +1316,9 @@ def __init__(self, entry: dict): _from = substitute_include_path(backwards_compat.get("from"), context) to = substitute_include_path(backwards_compat.get("to"), context) if not os.path.exists(_from): - os.mkdir(to) + # We'll initialize the empty dir and use it from now on (even if + # config is written into the old location later) + pathlib.Path(to).mkdir(exist_ok=True) elif to == self.path: if not os.path.exists(to): if there_are_yaml_files_in(_from): From 59376125061dc615c8d762abbe7f0ded2b93136d Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 17 Apr 2026 00:23:52 -0700 Subject: [PATCH 433/506] var name mismatch --- lib/spack/spack/paths.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index f6e5ae931e80a1..5c7010bdc6831a 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -167,9 +167,9 @@ def default_envs_path(self): ) @property - def default_licenses_dir(self): + def default_license_dir(self): if self.old_layout: - return self.base.old_licenses_dir + return self.base.old_licenses_path else: return os.path.join(self.data_home, "licenses") From 6b88bb65467c7162bc06632761166ce75cc2505c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 17 Apr 2026 10:15:39 -0700 Subject: [PATCH 434/506] dir existence --- lib/spack/spack/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 9c332a8c059745..8d0a59f2f0b9b1 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1271,6 +1271,7 @@ def there_are_yaml_files_in(_dir): def copy_yaml_files(_from, to): _from = pathlib.Path(_from) to = pathlib.Path(to) + to.parent.mkdir(parents=True, exist_ok=True) tmpdir = pathlib.Path(tempfile.mkdtemp(prefix=f".{to.name}.tmp-", dir=to.parent)) try: From fb1d38f3be298f67433b4d541020fbe0d23389ef Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 17 Apr 2026 10:31:08 -0700 Subject: [PATCH 435/506] parents must exist Signed-off-by: Peter Scheibel --- lib/spack/spack/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 8d0a59f2f0b9b1..2015520aadcfe5 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1319,7 +1319,7 @@ def __init__(self, entry: dict): if not os.path.exists(_from): # We'll initialize the empty dir and use it from now on (even if # config is written into the old location later) - pathlib.Path(to).mkdir(exist_ok=True) + pathlib.Path(to).mkdir(parents=True, exist_ok=True) elif to == self.path: if not os.path.exists(to): if there_are_yaml_files_in(_from): From a6d251914dc02d72e9ebdf27f08d45f6d20b7eca Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 17 Apr 2026 11:28:17 -0700 Subject: [PATCH 436/506] rm redundant imports Signed-off-by: Peter Scheibel --- lib/spack/spack/test/paths.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 0fd35a43a181aa..be4c92205eceee 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -232,7 +232,6 @@ def test_location_vars_that_use_other_location_vars( def test_license_dir_config(mutable_config, mock_packages, tmp_path, monkeypatch, set_home): """Ensure license directory is customizable""" - import spack.config import spack.package_base import spack.repo @@ -286,8 +285,6 @@ def test_child_proc_sanity_xdg_based_paths(tmp_path, set_home, monkeypatch): set_home(home_prefix) - import spack.paths - pbtest = SpackPathsBase(base_prefix) pbtest.old_install_path = empty_dir ptest = SpackPaths(pbtest) From ed204b476cc4361b860985c14ea182fdbedeffac Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 17 Apr 2026 13:49:23 -0700 Subject: [PATCH 437/506] new style format on changes to this PR Signed-off-by: Peter Scheibel --- lib/spack/spack/config.py | 2 +- lib/spack/spack/paths.py | 1 - lib/spack/spack/test/cmd/style.py | 3 +-- lib/spack/spack/test/paths.py | 6 +++--- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 9d337af6c800d0..d0ea159e453f2e 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1259,7 +1259,7 @@ def paths(self) -> List[str]: _yaml_exts = ".yaml", ".yml" -is_yaml = lambda x: (x.suffix in _yaml_exts) +is_yaml = lambda x: x.suffix in _yaml_exts def there_are_yaml_files_in(_dir): diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index bb4ef5e8897a7e..b606dce27de0d1 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -79,7 +79,6 @@ def _unset_path_vars(env): class SpackPaths: - relative_state_home = os.path.join(".local", "state") relative_data_home = os.path.join(".local", "share") relative_cache_home = ".cache" diff --git a/lib/spack/spack/test/cmd/style.py b/lib/spack/spack/test/cmd/style.py index af8242890d09ec..845f44e4bbfef3 100644 --- a/lib/spack/spack/test/cmd/style.py +++ b/lib/spack/spack/test/cmd/style.py @@ -410,7 +410,6 @@ def test_case_sensitive_imports(tmp_path: pathlib.Path): def test_pkg_imports(): assert ( - spack.cmd.style._module_part(pathlib.Path(paths.prefix), "spack.pkg.builtin.boost") - is None + spack.cmd.style._module_part(pathlib.Path(paths.prefix), "spack.pkg.builtin.boost") is None ) assert spack.cmd.style._module_part(pathlib.Path(paths.prefix), "spack.pkg") is None diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index be4c92205eceee..52ba6f643e2154 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -269,9 +269,9 @@ def __call__(self): os.environ["XDG_DATA_HOME"] = "/made-up-value-that-shouldnt-matter" expected = str(pathlib.Path(self.home_prefix) / ".local" / "share" / "spack" / "installs") - assert ( - spack.paths.locations.default_install_location == expected - ), f"Expected {expected}\nGot {spack.paths.locations.default_install_location}" + assert spack.paths.locations.default_install_location == expected, ( + f"Expected {expected}\nGot {spack.paths.locations.default_install_location}" + ) def test_child_proc_sanity_xdg_based_paths(tmp_path, set_home, monkeypatch): From da903e433bc623ab70a3ea8c881ae4daf376d7b9 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 17 Apr 2026 15:43:55 -0700 Subject: [PATCH 438/506] update path state subprocess transmission --- lib/spack/spack/new_installer.py | 6 +++++- lib/spack/spack/paths.py | 16 +++++++++++++--- lib/spack/spack/subprocess_context.py | 8 ++++---- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/new_installer.py b/lib/spack/spack/new_installer.py index 4162a0238e83b1..fb284884dc270e 100644 --- a/lib/spack/spack/new_installer.py +++ b/lib/spack/spack/new_installer.py @@ -70,6 +70,7 @@ import spack.llnl.util.filesystem as fs import spack.llnl.util.tty import spack.llnl.util.tty.color +import spack.paths import spack.paths_base import spack.report import spack.spec @@ -328,11 +329,13 @@ class GlobalState: but excludes the Spack environment, which is slow to serialize and should not be needed during the build.""" - __slots__ = ("store", "config", "monkey_patches", "spack_working_dir", "repo_cache") + __slots__ = ("store", "config", "monkey_patches", "spack_working_dir", "paths_state") def __init__(self): + paths_state = spack.paths.freeze() if multiprocessing.get_start_method() == "fork": return + self.paths_state = paths_state self.config = spack.config.CONFIG.ensure_unwrapped() self.store = spack.store.STORE self.monkey_patches = spack.subprocess_context.TestPatches.create() @@ -349,6 +352,7 @@ def restore(self): opener.urlopen._instance = None s3_client_cache.clear() return + spack.paths.restore(self.paths_state) spack.store.STORE = self.store spack.config.CONFIG = self.config self.monkey_patches.restore() diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index b606dce27de0d1..4af2c32a0f14e4 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -360,6 +360,16 @@ def freeze(): Note that for this reason, the private variables like _data_home are important for more than just caching. """ - locations.state_home - locations.data_home - locations.cache_home + return { + "state_home": locations.state_home, + "data_home": locations.data_home, + "cache_home": locations.cache_home, + "old_layout": locations.old_layout, + } + + +def restore(bundled_state): + locations._state_home = bundled_state["state_home"] + locations._data_home = bundled_state["data_home"] + locations._cache_home = bundled_state["cache_home"] + locations.old_layout = bundled_state["old_layout"] diff --git a/lib/spack/spack/subprocess_context.py b/lib/spack/spack/subprocess_context.py index 3dd2e11e2fd5f8..87b5ebc94f0018 100644 --- a/lib/spack/spack/subprocess_context.py +++ b/lib/spack/spack/subprocess_context.py @@ -95,14 +95,14 @@ def __init__( ) -> None: ctx = ctx or multiprocessing.get_context() self.is_forked = ctx.get_start_method() == "fork" - # Note: freezing before is sufficient for fork. Spawn starts - # must also freeze after - spack.paths.freeze() + # Make sure paths have been resolved before fork + paths_state = spack.paths.freeze() if self.is_forked: return from spack.environment import active_environment + self.paths_state = paths_state self.config = spack.config.CONFIG.ensure_unwrapped() self.platform = spack.platforms.host self.store = spack.store.STORE @@ -117,7 +117,7 @@ def restore(self): spack.platforms.host = self.platform spack.store.STORE = self.store self.test_patches.restore() - spack.paths.freeze() + spack.paths.restore(self.paths_state) if self.env: from spack.environment import activate From 52a22207cb570269aaeb0d344becb8fc7207cba7 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 17 Apr 2026 16:09:07 -0700 Subject: [PATCH 439/506] update doc section about spack implicit layout toggling --- lib/spack/docs/where_spack_writes_data.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/spack/docs/where_spack_writes_data.rst b/lib/spack/docs/where_spack_writes_data.rst index b4127f57ef5366..24c9d83bd433b7 100644 --- a/lib/spack/docs/where_spack_writes_data.rst +++ b/lib/spack/docs/where_spack_writes_data.rst @@ -28,6 +28,18 @@ You can control this in the following ways: In the absence of any Spack-specific settings, Spack will respect [XDG](https://specifications.freedesktop.org/basedir/latest/) environment variables controlling the home directory for specific types of data. +Spack's older layout, and pulling newer versions of Spack +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Spack previously stored many important pieces of data in the Spack prefix: + +* Installs in ``$spack/opt/spack`` +* Environments in ``$spack/var/spack/environments`` +* GPG keys in ``$spack/opt/spack/gpg`` + +If Spack detects this old layout in use, it will continue to use it. +Targeted config settings like ``config:install_tree:root`` will override this, but not other environment variables or general configuration (i.e. anything described below this section). + .. _config-file-data-variables: Spack-specific variables controlling data location From d8fc3ab84878abd9cddd8719abeead8ab7568f40 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 17 Apr 2026 17:42:07 -0700 Subject: [PATCH 440/506] ci generate encourages installing inside spack root Signed-off-by: Peter Scheibel --- lib/spack/spack/ci/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/ci/common.py b/lib/spack/spack/ci/common.py index 581ce99b5704a0..02bd832a864f92 100644 --- a/lib/spack/spack/ci/common.py +++ b/lib/spack/spack/ci/common.py @@ -579,6 +579,7 @@ def generate_ir(self): { "build-job": { "script": [ + "spack config add 'config:install_tree:root:$spack/installs'", "cd {env_dir}", "spack env activate --without-view .", "spack spec /$SPACK_JOB_SPEC_DAG_HASH", From 2a50e5fd39d3b5386385e478d2e9d8fe8d451b19 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 17 Apr 2026 17:59:52 -0700 Subject: [PATCH 441/506] update test based on new default command Signed-off-by: Peter Scheibel --- lib/spack/spack/test/cmd/ci.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 63e107510fb318..9f8d2e730ec400 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -333,10 +333,11 @@ def test_ci_generate_with_custom_settings( "git checkout ${SPACK_REF}", "popd", ] - assert ci_obj["script"][1].startswith("cd ") - ci_obj["script"][1] = "cd ENV" + assert ci_obj["script"][2].startswith("cd ") + ci_obj["script"][2] = "cd ENV" assert ci_obj["script"] == [ "spack -d ci rebuild", + "spack config add 'config:install_tree:root:$spack/installs'", "cd ENV", "spack env activate --without-view .", "spack spec /$SPACK_JOB_SPEC_DAG_HASH", From 44a9671ed1d89fd1c2cf2d662743e3ac4d7561a5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 13 May 2026 15:22:19 -0700 Subject: [PATCH 442/506] update command completion (was reverted for merge from develop) --- share/spack/spack-completion.bash | 6 +++--- share/spack/spack-completion.fish | 12 +++++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 315bc0edf75237..d687f3c88aca19 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -394,7 +394,7 @@ SPACK_ALIASES="concretise:concretize;containerise:containerize;rm:remove" _spack() { if $list_options then - SPACK_COMPREPLY="--color -v --verbose -k --insecure -b --bootstrap -V --version -h --help -H --all-help -c --config -C --config-scope -e --env -D --env-dir -E --no-env --use-env-repo -d --debug -t --backtrace --pdb --timestamp -m --mock --print-shell-vars --stacktrace -l --enable-locks -L --disable-locks -p --profile --profile-file --sorted-profile --lines" + SPACK_COMPREPLY="--color -v --verbose -k --insecure -b --bootstrap -V --version -h --help -H --all-help -c --config -C --config-scope -e --env -D --env-dir -E --no-env --use-env-repo -d --debug -t --backtrace --pdb --timestamp -m --mock --print-shell-vars --stacktrace --warn-writes-into-spack -l --enable-locks -L --disable-locks -p --profile --profile-file --sorted-profile --lines" else SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse logs maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" fi @@ -677,7 +677,7 @@ _spack_buildcache_migrate() { _spack_cd() { if $list_options then - SPACK_COMPREPLY="-h --help -m --module-dir -r --spack-root -i --install-dir -p --package-dir --repo --packages -P -s --stage-dir -S --stages -c --source-dir -b --build-dir -e --env -v --view --first" + SPACK_COMPREPLY="-h --help -m --module-dir -r --spack-root -i --install-dir --install-root -p --package-dir --repo --packages -P -s --stage-dir -S --stages -c --source-dir -b --build-dir -e --env -v --view --first" else _all_packages fi @@ -1406,7 +1406,7 @@ _spack_load() { _spack_location() { if $list_options then - SPACK_COMPREPLY="-h --help -m --module-dir -r --spack-root -i --install-dir -p --package-dir --repo --packages -P -s --stage-dir -S --stages -c --source-dir -b --build-dir -e --env -v --view --first" + SPACK_COMPREPLY="-h --help -m --module-dir -r --spack-root -i --install-dir --install-root -p --package-dir --repo --packages -P -s --stage-dir -S --stages -c --source-dir -b --build-dir -e --env -v --view --first" else _all_packages fi diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 5a0a496923cdf7..426a1ed4ea7055 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -346,7 +346,7 @@ complete -c spack --erase # Everything below here is auto-generated. # spack -set -g __fish_spack_optspecs_spack color= v/verbose k/insecure b/bootstrap V/version h/help H/all-help c/config= C/config-scope= e/env= D/env-dir= E/no-env use-env-repo d/debug t/backtrace pdb timestamp m/mock print-shell-vars= stacktrace l/enable-locks L/disable-locks p/profile profile-file= sorted-profile= lines= +set -g __fish_spack_optspecs_spack color= v/verbose k/insecure b/bootstrap V/version h/help H/all-help c/config= C/config-scope= e/env= D/env-dir= E/no-env use-env-repo d/debug t/backtrace pdb timestamp m/mock print-shell-vars= stacktrace warn-writes-into-spack l/enable-locks L/disable-locks p/profile profile-file= sorted-profile= lines= complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a add -d 'add a spec to an environment' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a arch -d 'print architecture information about this machine' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a audit -d 'audit configuration files, packages, etc.' @@ -466,6 +466,8 @@ complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -f -a complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -d 'print info needed by setup-env.*sh' complete -c spack -n '__fish_spack_using_command ' -l stacktrace -f -a stacktrace complete -c spack -n '__fish_spack_using_command ' -l stacktrace -d 'add stacktraces to all printed statements' +complete -c spack -n '__fish_spack_using_command ' -l warn-writes-into-spack -f -a warn_writes_into_spack +complete -c spack -n '__fish_spack_using_command ' -l warn-writes-into-spack -d 'Warn when Spack tries to write into its own prefix' complete -c spack -n '__fish_spack_using_command ' -s l -l enable-locks -f -a locks complete -c spack -n '__fish_spack_using_command ' -s l -l enable-locks -d 'use filesystem locking (default)' complete -c spack -n '__fish_spack_using_command ' -s L -l disable-locks -f -a locks @@ -921,7 +923,7 @@ complete -c spack -n '__fish_spack_using_command buildcache migrate' -s y -l yes complete -c spack -n '__fish_spack_using_command buildcache migrate' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request' # spack cd -set -g __fish_spack_optspecs_spack_cd h/help m/module-dir r/spack-root i/install-dir p/package-dir repo= s/stage-dir S/stages c/source-dir b/build-dir e/env= v/view= first +set -g __fish_spack_optspecs_spack_cd h/help m/module-dir r/spack-root i/install-dir install-root p/package-dir repo= s/stage-dir S/stages c/source-dir b/build-dir e/env= v/view= first complete -c spack -n '__fish_spack_using_command_pos_remainder 0 cd' -f -k -a '(__fish_spack_specs)' complete -c spack -n '__fish_spack_using_command cd' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command cd' -s h -l help -d 'show this help message and exit' @@ -931,6 +933,8 @@ complete -c spack -n '__fish_spack_using_command cd' -s r -l spack-root -f -a sp complete -c spack -n '__fish_spack_using_command cd' -s r -l spack-root -d 'spack installation root' complete -c spack -n '__fish_spack_using_command cd' -s i -l install-dir -f -a install_dir complete -c spack -n '__fish_spack_using_command cd' -s i -l install-dir -d 'install prefix for spec (spec need not be installed)' +complete -c spack -n '__fish_spack_using_command cd' -l install-root -f -a install_root +complete -c spack -n '__fish_spack_using_command cd' -l install-root -d 'where spack installs specs' complete -c spack -n '__fish_spack_using_command cd' -s p -l package-dir -f -a package_dir complete -c spack -n '__fish_spack_using_command cd' -s p -l package-dir -d 'directory enclosing a spec'"'"'s package.py file' complete -c spack -n '__fish_spack_using_command cd' -l repo -l packages -s P -r -f -a repo @@ -2262,7 +2266,7 @@ complete -c spack -n '__fish_spack_using_command load' -l list -f -a list complete -c spack -n '__fish_spack_using_command load' -l list -d 'show loaded packages: same as ``spack find --loaded``' # spack location -set -g __fish_spack_optspecs_spack_location h/help m/module-dir r/spack-root i/install-dir p/package-dir repo= s/stage-dir S/stages c/source-dir b/build-dir e/env= v/view= first +set -g __fish_spack_optspecs_spack_location h/help m/module-dir r/spack-root i/install-dir install-root p/package-dir repo= s/stage-dir S/stages c/source-dir b/build-dir e/env= v/view= first complete -c spack -n '__fish_spack_using_command_pos_remainder 0 location' -f -k -a '(__fish_spack_specs)' complete -c spack -n '__fish_spack_using_command location' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command location' -s h -l help -d 'show this help message and exit' @@ -2272,6 +2276,8 @@ complete -c spack -n '__fish_spack_using_command location' -s r -l spack-root -f complete -c spack -n '__fish_spack_using_command location' -s r -l spack-root -d 'spack installation root' complete -c spack -n '__fish_spack_using_command location' -s i -l install-dir -f -a install_dir complete -c spack -n '__fish_spack_using_command location' -s i -l install-dir -d 'install prefix for spec (spec need not be installed)' +complete -c spack -n '__fish_spack_using_command location' -l install-root -f -a install_root +complete -c spack -n '__fish_spack_using_command location' -l install-root -d 'where spack installs specs' complete -c spack -n '__fish_spack_using_command location' -s p -l package-dir -f -a package_dir complete -c spack -n '__fish_spack_using_command location' -s p -l package-dir -d 'directory enclosing a spec'"'"'s package.py file' complete -c spack -n '__fish_spack_using_command location' -l repo -l packages -s P -r -f -a repo From 60b8e7da87c22c8a4110600b119d07515e4f6c91 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 May 2026 11:21:01 -0700 Subject: [PATCH 443/506] user_cache_path uses old if new is not present; config uses old if new is not present --- etc/spack/include.yaml | 4 +--- lib/spack/spack/config.py | 23 ++++++++++----------- lib/spack/spack/paths.py | 33 ++++++++++++++++++++++++++++--- lib/spack/spack/paths_base.py | 7 ++++++- lib/spack/spack/schema/include.py | 3 +-- lib/spack/spack/test/config.py | 17 +++++++--------- 6 files changed, 55 insertions(+), 32 deletions(-) diff --git a/etc/spack/include.yaml b/etc/spack/include.yaml index b827bd02c232b8..066b227ad8567f 100644 --- a/etc/spack/include.yaml +++ b/etc/spack/include.yaml @@ -3,9 +3,7 @@ include: - name: "user" path_override_env_var: SPACK_USER_CONFIG_PATH path: "~/.config/spack/" - backwards_compat: - from: "~/.spack/" - to: "~/.config/spack/" + backwards_compat: "~/.spack/" optional: true prefer_modify: true when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index d0ea159e453f2e..44a6c7a1881a94 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1311,21 +1311,18 @@ def __init__(self, entry: dict): context_prefix = f"({self.name}) " if self.name else "" context = f"{context_prefix}{path}" - self.path = substitute_include_path(path, context) - backwards_compat = entry.get("backwards_compat", {}) + new_path = substitute_include_path(path, context) + old_path = None + backwards_compat = entry.get("backwards_compat", None) if backwards_compat: - _from = substitute_include_path(backwards_compat.get("from"), context) - to = substitute_include_path(backwards_compat.get("to"), context) - if not os.path.exists(_from): - # We'll initialize the empty dir and use it from now on (even if - # config is written into the old location later) - pathlib.Path(to).mkdir(parents=True, exist_ok=True) - elif to == self.path: - if not os.path.exists(to): - if there_are_yaml_files_in(_from): - tty.debug(f"Generating initial config in {to} from {_from}") - copy_yaml_files(_from, to) + old_path = substitute_include_path(backwards_compat, context) + + if old_path and os.path.exists(old_path) and not os.path.exists(new_path): + self.path = old_path + # TODO: warn + else: + self.path = new_path self.sha256 = entry.get("sha256", "") self.remote = "sha256" in entry diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 4af2c32a0f14e4..1824452670bbb0 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -69,6 +69,11 @@ class Spack_vars(Enum): user_cache_path = "SPACK_USER_CACHE_PATH" home = "SPACK_HOME" + @classmethod + def new_layout(cls): + # Exclude SPACK_USER_CACHE_PATH + return [Spack_vars.state_home, Spack_vars.data_home, Spack_vars.cache_home, Spack_vars.home] + # This is for tests that want to clean the environment of XDG_ variables that # affect spack behavior. Note that this will not influence install_test.py's @@ -94,17 +99,31 @@ def __init__(self, base): os.path.expanduser("~"), SpackPaths.relative_data_home, "spack" ) - self.old_layout = detect_old_spack_layout(base) + self.old_layout_detected = detect_old_spack_layout(base) @property def state_home(self): if not self._state_home: - self._state_home, _ = self.resolve_a_home( + state_home, _ = self.resolve_a_home( ["SPACK_STATE_HOME", "SPACK_USER_CACHE_PATH"], "state", SpackPaths.relative_state_home, "XDG_STATE_HOME", ) + + if new_layout_enforced(): + self._state_home = state_home + elif "SPACK_USER_CACHE_PATH" in os.environ: + # If we're here, SPACK_STATE_HOME is not in os.environ + self._state_home = os.environ.get("SPACK_USER_CACHE_PATH") + # TODO: print warning + elif dir_is_occupied(self.base.old_default_dot_spack): + self._state_home = self.base.old_default_dot_spack + # TODO: make sure package_repos_path returns the right thing + # TODO: print warning + else: + self._state_home = state_home + return self._state_home @property @@ -325,12 +344,20 @@ def env_check(env_vars, provenance, rel=None): def _decide_old_or_new_location( self, old_location, new_location, default_new_location, provenance ): - if self.old_layout: + if new_layout_enforced(): + return new_location + if self.old_layout_detected: return old_location else: return new_location +def new_layout_enforced(): + first = any(x.value in os.environ for x in Spack_vars.new_layout()) + second = bool(config.get("config:locations")) + return first or second + + def detect_old_spack_layout(paths: paths_base.SpackPathsBase): checks = [ # It's important if this directory is occupied but we have a separate diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py index fe89f94b266956..177f76e6b83f18 100644 --- a/lib/spack/spack/paths_base.py +++ b/lib/spack/spack/paths_base.py @@ -56,9 +56,14 @@ def __init__(self, _prefix=None): self.old_gpg_keys_path = os.path.join(self.var_path, "gpg") self.old_licenses_path = os.path.join(self.etc_path, "licenses") + expanded_home = os.path.expanduser("~") + + # If this exists, this was the location for configs and for the package repository + self.old_default_dot_spack = os.path.join(expanded_home, ".spack") + #: User configuration location self.user_config_path = os.path.expanduser( - os.getenv("SPACK_USER_CONFIG_PATH") or os.path.join("~", ".config", "spack") + os.getenv("SPACK_USER_CONFIG_PATH") or os.path.join(expanded_home, ".config", "spack") ) #: System configuration location diff --git a/lib/spack/spack/schema/include.py b/lib/spack/spack/schema/include.py index 5e72a73191feaa..5f06a0dbbd0d29 100644 --- a/lib/spack/spack/schema/include.py +++ b/lib/spack/spack/schema/include.py @@ -46,10 +46,9 @@ "Spack/environment variables", }, "backwards_compat": { - "type": "object", + "type": "string", "description": "For default scopes, old paths where they may be " "found, if they have moved", - "properties": {"from": {"type": "string"}, "to": {"type": "string"}}, }, "sha256": { "type": "string", diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 585251c9614ba0..d5ca00bebe1518 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1716,9 +1716,8 @@ def test_included_path_conditional_bad_when( @pytest.fixture def backwards_compat_setup(tmp_path: pathlib.Path): src_dir = tmp_path / "src" - sub_dir = src_dir / "subdir" - sub_dir.mkdir(parents=True) - (sub_dir / "example.yaml").touch() + src_dir.mkdir() + (src_dir / "example.yaml").touch() # Path doesn't exist yet to_dir = tmp_path / "to" @@ -1726,7 +1725,7 @@ def backwards_compat_setup(tmp_path: pathlib.Path): entry = { "path": str(to_dir), "optional": True, - "backwards_compat": {"from": str(src_dir), "to": str(to_dir)}, + "backwards_compat": str(src_dir) } yield src_dir, to_dir, entry @@ -1736,9 +1735,8 @@ def test_backwards_compat_use_old(backwards_compat_setup): """If target dir doesn't exist, make it and copy over yaml files.""" src_dir, to_dir, entry = backwards_compat_setup - spack.config.included_path(entry) - assert to_dir.exists() - assert (to_dir / "subdir" / "example.yaml").exists() + x = spack.config.included_path(entry) + assert x.paths == [str(src_dir)] def test_backwards_compat_skip_old(backwards_compat_setup): @@ -1746,9 +1744,8 @@ def test_backwards_compat_skip_old(backwards_compat_setup): src_dir, to_dir, entry = backwards_compat_setup to_dir.mkdir() - spack.config.included_path(entry) - assert to_dir.exists() - assert not (to_dir / "subdir" / "example.yaml").exists() + x = spack.config.included_path(entry) + assert x.paths == [str(to_dir)] def test_included_path_conditional_success(tmp_path: pathlib.Path, mock_low_high_config): From bd207fd173c3f3e238b4960056a48b8dda195ad5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 May 2026 11:43:30 -0700 Subject: [PATCH 444/506] rm utility code (now unused) --- lib/spack/spack/config.py | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 44a6c7a1881a94..9604bc3d22e64f 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1258,44 +1258,6 @@ def paths(self) -> List[str]: raise NotImplementedError("must be implemented in derived classes") -_yaml_exts = ".yaml", ".yml" -is_yaml = lambda x: x.suffix in _yaml_exts - - -def there_are_yaml_files_in(_dir): - for path in pathlib.Path(_dir).rglob("*"): - if is_yaml(path): - return True - return False - - -def copy_yaml_files(_from, to): - _from = pathlib.Path(_from) - to = pathlib.Path(to) - to.parent.mkdir(parents=True, exist_ok=True) - tmpdir = pathlib.Path(tempfile.mkdtemp(prefix=f".{to.name}.tmp-", dir=to.parent)) - - try: - _copy_yaml_files(_from, tmpdir) - except Exception as e: - shutil.rmtree(tmpdir, ignore_errors=True) - raise ConfigFileError(str(e)) - - try: - tmpdir.rename(to) - except FileExistsError: - shutil.rmtree(tmpdir, ignore_errors=True) - - -def _copy_yaml_files(_from, to): - for path in _from.rglob("*"): - if is_yaml(path): - rel_from = path.relative_to(_from) - abs_to = to / rel_from - abs_to.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(path, abs_to) - - class IncludePath(OptionalInclude): path: str sha256: str From 463116d08b29e2c5abf76b650ffcce92f51f019d Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 May 2026 13:26:29 -0700 Subject: [PATCH 445/506] new migrate command --- lib/spack/spack/cmd/migrate.py | 144 +++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 lib/spack/spack/cmd/migrate.py diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py new file mode 100644 index 00000000000000..6be7496584f5af --- /dev/null +++ b/lib/spack/spack/cmd/migrate.py @@ -0,0 +1,144 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import argparse +import os +import shutil + +import spack.llnl.util.tty as tty +from spack.paths_base import locations as paths_base + +description = "migrate user config and cache from old to new locations" +section = "config" +level = "long" + + +def setup_parser(subparser: argparse.ArgumentParser) -> None: + subparser.add_argument( + "-n", + "--dry-run", + action="store_true", + help="show what would be migrated without actually moving files", + ) + + +def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: + """Migrate user config and package repositories from ~/.spack to new locations. + + This command migrates: + - User config files: ~/.spack/*.yaml -> ~/.config/spack/ + - Package repositories: ~/.spack/package_repos -> ~/.local/state/spack/package_repos + """ + old_location = os.path.expanduser("~/.spack") + new_config_location = paths_base.user_config_path + new_state_location = os.path.join(os.path.expanduser("~"), ".local", "state", "spack") + + # Check if old location exists + if not os.path.exists(old_location): + tty.die(f"Old configuration location does not exist: {old_location}") + + # Track what we'll migrate + migrations = [] + errors = [] + + # 1. Check for config files to migrate (*.yaml and *.yml files in ~/.spack/) + config_files = [] + if os.path.isdir(old_location): + for item in os.listdir(old_location): + if item.endswith(".yaml") or item.endswith(".yml"): + config_files.append(item) + + if config_files: + # Check if new config location already has config files + if os.path.exists(new_config_location): + existing_configs = [ + f for f in os.listdir(new_config_location) + if f.endswith(".yaml") or f.endswith(".yml") + ] + if existing_configs: + errors.append( + f"New config location already contains config files: {new_config_location}\n" + f" Existing files: {', '.join(existing_configs)}" + ) + + if not errors: + migrations.append( + ("config", config_files, old_location, new_config_location) + ) + + # 2. Check for package repositories to migrate + old_package_repos = os.path.join(old_location, "package_repos") + new_package_repos = os.path.join(new_state_location, "package_repos") + + if os.path.exists(old_package_repos) and os.path.isdir(old_package_repos): + # Check if there's anything in it + repos = os.listdir(old_package_repos) + if repos: + # Check if new location already has package repositories + if os.path.exists(new_package_repos): + existing_repos = os.listdir(new_package_repos) + if existing_repos: + errors.append( + f"New package repository location already exists: {new_package_repos}\n" + f" Existing repos: {', '.join(existing_repos)}" + ) + + if not errors: + migrations.append( + ("package_repos", repos, old_package_repos, new_package_repos) + ) + + # Report errors if any + if errors: + tty.die("Cannot migrate due to conflicts:\n " + "\n ".join(errors)) + + # Report what we found + if not migrations: + tty.msg("Nothing to migrate - no config files or package repositories found in ~/.spack") + return + + # Show what will be migrated + tty.msg("Will migrate the following:") + for migration_type, items, src, dst in migrations: + if migration_type == "config": + tty.msg(f"\n Config files from {src}/ to {dst}/:") + for item in items: + tty.msg(f" - {item}") + elif migration_type == "package_repos": + tty.msg(f"\n Package repositories from {src}/ to {dst}/:") + for item in items: + tty.msg(f" - {item}") + + if args.dry_run: + tty.msg("\nDry run - no files were moved") + return + + # Perform migrations + tty.msg("\nMigrating files...") + + for migration_type, items, src, dst in migrations: + # Ensure destination directory exists + os.makedirs(dst, exist_ok=True) + + if migration_type == "config": + # Move config files + for item in items: + src_path = os.path.join(src, item) + dst_path = os.path.join(dst, item) + tty.debug(f"Moving {src_path} -> {dst_path}") + shutil.move(src_path, dst_path) + tty.msg(f" Migrated {len(items)} config file(s) to {dst}") + + elif migration_type == "package_repos": + # Move entire package_repos directory contents + for item in items: + src_path = os.path.join(src, item) + dst_path = os.path.join(dst, item) + tty.debug(f"Moving {src_path} -> {dst_path}") + shutil.move(src_path, dst_path) + tty.msg(f" Migrated {len(items)} package repositor(y|ies) to {dst}") + + tty.msg("\nMigration complete!") + tty.msg(f" Config location: {new_config_location}") + tty.msg(f" State location: {new_state_location}") From f9aa204343df95bd7cd963a724c2f25e0748afcd Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 May 2026 15:20:13 -0700 Subject: [PATCH 446/506] spack migrate command can backup/restore dot-spack dir --- lib/spack/spack/cmd/migrate.py | 166 +++++++++++++++++++++++++-------- lib/spack/spack/paths.py | 5 + 2 files changed, 130 insertions(+), 41 deletions(-) diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py index 6be7496584f5af..3016eb3bec343c 100644 --- a/lib/spack/spack/cmd/migrate.py +++ b/lib/spack/spack/cmd/migrate.py @@ -7,6 +7,7 @@ import shutil import spack.llnl.util.tty as tty +from spack.paths import locations as paths from spack.paths_base import locations as paths_base description = "migrate user config and cache from old to new locations" @@ -21,6 +22,52 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: action="store_true", help="show what would be migrated without actually moving files", ) + subparser.add_argument( + "--backup", + action="store_true", + help="move entire ~/.spack directory to backup location after migration", + ) + subparser.add_argument( + "--restore-old-configs", + action="store_true", + help="restore ~/.spack from backup location", + ) + + +def restore_old_configs(args: argparse.Namespace) -> None: + """Restore ~/.spack from backup location.""" + old_location = os.path.expanduser("~/.spack") + + # Check both the current backup location and the default one + backup_locations = [paths.dotspack_backup] + default_backup = os.path.join(paths.default_data_home, "dotspack_backup") + if default_backup != paths.dotspack_backup: + backup_locations.append(default_backup) + + # Find which backup location exists + backup_location = None + for loc in backup_locations: + if os.path.exists(loc): + backup_location = loc + break + + if not backup_location: + tty.die( + f"No backup found. Checked:\n" + + "\n".join(f" - {loc}" for loc in backup_locations) + ) + + # Check if ~/.spack already exists + if os.path.exists(old_location): + tty.die(f"Cannot restore: {old_location} already exists") + + if args.dry_run: + tty.msg(f"Would restore from {backup_location} to {old_location}") + return + + tty.msg(f"Restoring from {backup_location} to {old_location}...") + shutil.copytree(backup_location, old_location) + tty.msg("Restore complete!") def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: @@ -30,14 +77,24 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: - User config files: ~/.spack/*.yaml -> ~/.config/spack/ - Package repositories: ~/.spack/package_repos -> ~/.local/state/spack/package_repos """ + # Handle restore mode + if args.restore_old_configs: + restore_old_configs(args) + return + old_location = os.path.expanduser("~/.spack") new_config_location = paths_base.user_config_path new_state_location = os.path.join(os.path.expanduser("~"), ".local", "state", "spack") + backup_location = paths.dotspack_backup # Check if old location exists if not os.path.exists(old_location): tty.die(f"Old configuration location does not exist: {old_location}") + # Check if backup already exists + if args.backup and os.path.exists(backup_location): + tty.die(f"Backup location already exists: {backup_location}") + # Track what we'll migrate migrations = [] errors = [] @@ -91,54 +148,81 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: # Report errors if any if errors: - tty.die("Cannot migrate due to conflicts:\n " + "\n ".join(errors)) + if args.backup: + # If --backup is specified and there are conflicts, skip migration and just backup + tty.msg("Migration conflicts detected (files already in new locations):") + for error in errors: + tty.msg(f" {error}") + tty.msg(f"\nSkipping migration, will only backup ~/.spack to {backup_location}") + migrations = [] # Clear migrations, just do backup + else: + tty.die("Cannot migrate due to conflicts:\n " + "\n ".join(errors)) # Report what we found if not migrations: - tty.msg("Nothing to migrate - no config files or package repositories found in ~/.spack") - return + if args.backup: + tty.msg("Nothing to migrate - no config files or package repositories found in ~/.spack") + tty.msg(f"Will still backup ~/.spack to {backup_location}") + else: + tty.msg("Nothing to migrate - no config files or package repositories found in ~/.spack") + return # Show what will be migrated - tty.msg("Will migrate the following:") - for migration_type, items, src, dst in migrations: - if migration_type == "config": - tty.msg(f"\n Config files from {src}/ to {dst}/:") - for item in items: - tty.msg(f" - {item}") - elif migration_type == "package_repos": - tty.msg(f"\n Package repositories from {src}/ to {dst}/:") - for item in items: - tty.msg(f" - {item}") + if migrations: + tty.msg("Will migrate the following:") + for migration_type, items, src, dst in migrations: + if migration_type == "config": + tty.msg(f"\n Config files from {src}/ to {dst}/:") + for item in items: + tty.msg(f" - {item}") + elif migration_type == "package_repos": + tty.msg(f"\n Package repositories from {src}/ to {dst}/:") + for item in items: + tty.msg(f" - {item}") + + if args.backup: + tty.msg(f"\nAfter migration, will backup entire ~/.spack to {backup_location}") if args.dry_run: - tty.msg("\nDry run - no files were moved") + tty.msg("\nDry run - no files were copied or moved") return # Perform migrations - tty.msg("\nMigrating files...") - - for migration_type, items, src, dst in migrations: - # Ensure destination directory exists - os.makedirs(dst, exist_ok=True) - - if migration_type == "config": - # Move config files - for item in items: - src_path = os.path.join(src, item) - dst_path = os.path.join(dst, item) - tty.debug(f"Moving {src_path} -> {dst_path}") - shutil.move(src_path, dst_path) - tty.msg(f" Migrated {len(items)} config file(s) to {dst}") - - elif migration_type == "package_repos": - # Move entire package_repos directory contents - for item in items: - src_path = os.path.join(src, item) - dst_path = os.path.join(dst, item) - tty.debug(f"Moving {src_path} -> {dst_path}") - shutil.move(src_path, dst_path) - tty.msg(f" Migrated {len(items)} package repositor(y|ies) to {dst}") - - tty.msg("\nMigration complete!") - tty.msg(f" Config location: {new_config_location}") - tty.msg(f" State location: {new_state_location}") + if migrations: + tty.msg("\nMigrating files...") + + for migration_type, items, src, dst in migrations: + # Ensure destination directory exists + os.makedirs(dst, exist_ok=True) + + if migration_type == "config": + # Copy config files + for item in items: + src_path = os.path.join(src, item) + dst_path = os.path.join(dst, item) + tty.debug(f"Copying {src_path} -> {dst_path}") + shutil.copy2(src_path, dst_path) + tty.msg(f" Migrated {len(items)} config file(s) to {dst}") + + elif migration_type == "package_repos": + # Copy entire package_repos directory contents + for item in items: + src_path = os.path.join(src, item) + dst_path = os.path.join(dst, item) + tty.debug(f"Copying {src_path} -> {dst_path}") + if os.path.isdir(src_path): + shutil.copytree(src_path, dst_path) + else: + shutil.copy2(src_path, dst_path) + tty.msg(f" Migrated {len(items)} package repositor(y|ies) to {dst}") + + tty.msg("\nMigration complete!") + tty.msg(f" Config location: {new_config_location}") + tty.msg(f" State location: {new_state_location}") + + # Handle backup + if args.backup: + tty.msg(f"\nBacking up ~/.spack to {backup_location}...") + os.makedirs(os.path.dirname(backup_location), exist_ok=True) + shutil.move(old_location, backup_location) + tty.msg(f"Backup complete! Original ~/.spack moved to {backup_location}") diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 1824452670bbb0..ef496ec7994783 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -229,6 +229,11 @@ def package_repos_path(self): #: default location where remote package repositories are cloned return os.path.join(self.state_home, "package_repos") + @property + def dotspack_backup(self): + #: backup location for old ~/.spack directory during migration + return os.path.join(self.data_home, "dotspack_backup") + @property def gpg_path(self): return self._decide_old_or_new_location( From 77c2c4512ea882d41a60bf28bb5ca82d1f3237a8 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 May 2026 16:10:18 -0700 Subject: [PATCH 447/506] tweak state_home/user_cache_path logic --- lib/spack/spack/paths.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index ef496ec7994783..7cc73916b204f7 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -116,11 +116,12 @@ def state_home(self): elif "SPACK_USER_CACHE_PATH" in os.environ: # If we're here, SPACK_STATE_HOME is not in os.environ self._state_home = os.environ.get("SPACK_USER_CACHE_PATH") - # TODO: print warning + elif dir_is_occupied(state_home): + self._state_home = state_home elif dir_is_occupied(self.base.old_default_dot_spack): self._state_home = self.base.old_default_dot_spack - # TODO: make sure package_repos_path returns the right thing - # TODO: print warning + # TODO: generate a warning if this dir is occupied (doesn't necessarily + # have to happen here) else: self._state_home = state_home From bab6f996f2ff779273d94be7ffb867768f25568e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 May 2026 17:09:56 -0700 Subject: [PATCH 448/506] consolidation of logic; less verbose messages --- lib/spack/spack/cmd/migrate.py | 72 ++++++++++++---------------------- 1 file changed, 24 insertions(+), 48 deletions(-) diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py index 3016eb3bec343c..73bb80810630e5 100644 --- a/lib/spack/spack/cmd/migrate.py +++ b/lib/spack/spack/cmd/migrate.py @@ -17,20 +17,20 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( - "-n", "--dry-run", action="store_true", help="show what would be migrated without actually moving files", ) subparser.add_argument( - "--backup", + "--clear", action="store_true", - help="move entire ~/.spack directory to backup location after migration", + help="move entire ~/.spack directory to backup location:" + "use this if no other instances need this old location", ) subparser.add_argument( "--restore-old-configs", action="store_true", - help="restore ~/.spack from backup location", + help="restore ~/.spack from backup location (after --clear)", ) @@ -92,7 +92,7 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: tty.die(f"Old configuration location does not exist: {old_location}") # Check if backup already exists - if args.backup and os.path.exists(backup_location): + if args.clear and os.path.exists(backup_location): tty.die(f"Backup location already exists: {backup_location}") # Track what we'll migrate @@ -107,7 +107,6 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: config_files.append(item) if config_files: - # Check if new config location already has config files if os.path.exists(new_config_location): existing_configs = [ f for f in os.listdir(new_config_location) @@ -146,51 +145,31 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: ("package_repos", repos, old_package_repos, new_package_repos) ) - # Report errors if any if errors: - if args.backup: - # If --backup is specified and there are conflicts, skip migration and just backup - tty.msg("Migration conflicts detected (files already in new locations):") - for error in errors: - tty.msg(f" {error}") - tty.msg(f"\nSkipping migration, will only backup ~/.spack to {backup_location}") - migrations = [] # Clear migrations, just do backup - else: - tty.die("Cannot migrate due to conflicts:\n " + "\n ".join(errors)) - - # Report what we found - if not migrations: - if args.backup: - tty.msg("Nothing to migrate - no config files or package repositories found in ~/.spack") - tty.msg(f"Will still backup ~/.spack to {backup_location}") - else: - tty.msg("Nothing to migrate - no config files or package repositories found in ~/.spack") - return + tty.msg("Migration conflicts detected (files already in new locations):") + for error in errors: + tty.msg(f" {error}") + tty.msg(f"\nSkipping migration") + migrations = [] + elif not migrations: + tty.msg("Nothing to migrate - no config files or package repositories found in ~/.spack") # Show what will be migrated - if migrations: - tty.msg("Will migrate the following:") - for migration_type, items, src, dst in migrations: - if migration_type == "config": - tty.msg(f"\n Config files from {src}/ to {dst}/:") - for item in items: - tty.msg(f" - {item}") - elif migration_type == "package_repos": - tty.msg(f"\n Package repositories from {src}/ to {dst}/:") - for item in items: - tty.msg(f" - {item}") - - if args.backup: - tty.msg(f"\nAfter migration, will backup entire ~/.spack to {backup_location}") - if args.dry_run: - tty.msg("\nDry run - no files were copied or moved") + if migrations: + tty.msg("Would migrate the following:") + for migration_type, items, src, dst in migrations: + if migration_type == "config": + tty.msg(f"\n Config files from {src}/ to {dst}/:") + for item in items: + tty.msg(f" - {item}") + elif migration_type == "package_repos": + tty.msg(f"\n Package repositories from {src}/ to {dst}/:") + for item in items: + tty.msg(f" - {item}") return - # Perform migrations if migrations: - tty.msg("\nMigrating files...") - for migration_type, items, src, dst in migrations: # Ensure destination directory exists os.makedirs(dst, exist_ok=True) @@ -202,7 +181,6 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: dst_path = os.path.join(dst, item) tty.debug(f"Copying {src_path} -> {dst_path}") shutil.copy2(src_path, dst_path) - tty.msg(f" Migrated {len(items)} config file(s) to {dst}") elif migration_type == "package_repos": # Copy entire package_repos directory contents @@ -214,14 +192,12 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: shutil.copytree(src_path, dst_path) else: shutil.copy2(src_path, dst_path) - tty.msg(f" Migrated {len(items)} package repositor(y|ies) to {dst}") tty.msg("\nMigration complete!") tty.msg(f" Config location: {new_config_location}") tty.msg(f" State location: {new_state_location}") - # Handle backup - if args.backup: + if args.clear: tty.msg(f"\nBacking up ~/.spack to {backup_location}...") os.makedirs(os.path.dirname(backup_location), exist_ok=True) shutil.move(old_location, backup_location) From 3d63d4b6c85dd3d9963f9f585bddebb4db1c48ce Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 May 2026 17:12:06 -0700 Subject: [PATCH 449/506] minor edit --- lib/spack/spack/cmd/migrate.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py index 73bb80810630e5..07792b1c6ad39c 100644 --- a/lib/spack/spack/cmd/migrate.py +++ b/lib/spack/spack/cmd/migrate.py @@ -154,8 +154,8 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: elif not migrations: tty.msg("Nothing to migrate - no config files or package repositories found in ~/.spack") - # Show what will be migrated if args.dry_run: + # Show what will be migrated if migrations: tty.msg("Would migrate the following:") for migration_type, items, src, dst in migrations: @@ -175,7 +175,6 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: os.makedirs(dst, exist_ok=True) if migration_type == "config": - # Copy config files for item in items: src_path = os.path.join(src, item) dst_path = os.path.join(dst, item) @@ -183,7 +182,6 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: shutil.copy2(src_path, dst_path) elif migration_type == "package_repos": - # Copy entire package_repos directory contents for item in items: src_path = os.path.join(src, item) dst_path = os.path.join(dst, item) @@ -198,7 +196,6 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: tty.msg(f" State location: {new_state_location}") if args.clear: - tty.msg(f"\nBacking up ~/.spack to {backup_location}...") os.makedirs(os.path.dirname(backup_location), exist_ok=True) shutil.move(old_location, backup_location) tty.msg(f"Backup complete! Original ~/.spack moved to {backup_location}") From 3c620eea9495b886514353be94444878652e38fc Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 May 2026 17:28:23 -0700 Subject: [PATCH 450/506] skip clear when there are errors --- lib/spack/spack/cmd/migrate.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py index 07792b1c6ad39c..f8ba853c1cf8e3 100644 --- a/lib/spack/spack/cmd/migrate.py +++ b/lib/spack/spack/cmd/migrate.py @@ -149,8 +149,10 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: tty.msg("Migration conflicts detected (files already in new locations):") for error in errors: tty.msg(f" {error}") - tty.msg(f"\nSkipping migration") - migrations = [] + tty.msg(f"\nSkipping migration and backup/clear due to conflicts") + # Exit early here regardless of --clear (we shouldn't move .spack if + # we couldn't copy out the components we want) + return elif not migrations: tty.msg("Nothing to migrate - no config files or package repositories found in ~/.spack") From c46bce276463d500843c67e8ec86eac7036c4932 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 May 2026 18:11:10 -0700 Subject: [PATCH 451/506] warn about use of old dot spack --- lib/spack/spack/main.py | 58 +++++++++++++++++++++++++++++++++++++++- lib/spack/spack/paths.py | 3 +-- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 04e46fbfef0601..d60c3f4d38b10f 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -927,6 +927,60 @@ def add_command_line_scopes( cfg.push_scope(scope, priority=ConfigScopePriority.CUSTOM) +def _warn_about_old_dotspack(): + """Warn if ~/.spack exists and is in use (not explicitly configured).""" + old_dotspack = os.path.expanduser("~/.spack") + + # Don't warn if it doesn't exist + if not os.path.exists(old_dotspack): + return + + # Check if explicitly configured via config:locations + if spack.config.get("config:locations"): + # Any config:locations setting means explicit configuration, don't warn + return + + # Check if explicitly configured via environment variables + explicit_env_vars = [ + "SPACK_HOME", + "SPACK_DATA_HOME", + "SPACK_STATE_HOME", + "SPACK_CACHE_HOME", + "SPACK_USER_CACHE_PATH", + ] + if any(var in os.environ for var in explicit_env_vars): + # Explicitly configured, don't warn + return + + # Helper to check if old_dotspack is a prefix + def uses_old_dotspack(path): + return path == old_dotspack or path.startswith(old_dotspack + os.sep) + + # Check if any config scope is using ~/.spack (means explicit configuration) + for scope in spack.config.CONFIG.scopes.values(): + if hasattr(scope, "path") and uses_old_dotspack(scope.path): + # Explicitly configured via a config scope, don't warn + return + + from spack.paths import locations as paths + + # Check which paths are using ~/.spack + parts = [] + if uses_old_dotspack(paths.state_home): + parts.append("user cache path is in use") + if uses_old_dotspack(paths.data_home): + parts.append("data home is in use") + if uses_old_dotspack(paths.cache_home): + parts.append("cache home is in use") + + # Only warn if something is using it + if not parts: + return + + usage = " and ".join(parts) + tty.warn(f"Old config/user-cache-path in `~/.spack` detected ({usage}). Run `spack migrate`") + + def _main(argv=None): """Logic for the main entry point for the Spack command. @@ -1054,7 +1108,9 @@ def add_environment_scope(): bootstrap_context = bootstrap.ensure_bootstrap_configuration() with bootstrap_context: - return finish_parse_and_run(parser, cmd_name, args, env_format_error) + result = finish_parse_and_run(parser, cmd_name, args, env_format_error) + _warn_about_old_dotspack() + return result def finish_parse_and_run(parser, cmd_name, main_args, env_format_error): diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 7cc73916b204f7..5c7f76a8ffad81 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -120,8 +120,7 @@ def state_home(self): self._state_home = state_home elif dir_is_occupied(self.base.old_default_dot_spack): self._state_home = self.base.old_default_dot_spack - # TODO: generate a warning if this dir is occupied (doesn't necessarily - # have to happen here) + # TODO [default state home case in old ~/.spack] else: self._state_home = state_home From 56d92af905919c39b35f4f337f44c43c3067d738 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 May 2026 18:54:40 -0700 Subject: [PATCH 452/506] more-robust warnings (more careful about when to warn) --- lib/spack/spack/config.py | 6 ++++- lib/spack/spack/main.py | 50 ++++++++++++++++++--------------------- lib/spack/spack/paths.py | 4 +++- 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 9604bc3d22e64f..430bf5fdd6b79d 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -161,6 +161,7 @@ def __init__(self, name: str, included: bool = False) -> None: self.sections = syaml.syaml_dict() self.prefer_modify = False self.included = included + self.backwards_compat_fallback = False #: included configuration scopes self._included_scopes: Optional[List["ConfigScope"]] = None @@ -1280,9 +1281,10 @@ def __init__(self, entry: dict): if backwards_compat: old_path = substitute_include_path(backwards_compat, context) + self.backwards_compat_fallback = False if old_path and os.path.exists(old_path) and not os.path.exists(new_path): self.path = old_path - # TODO: warn + self.backwards_compat_fallback = True else: self.path = new_path @@ -1335,6 +1337,8 @@ def scopes(self, parent_scope: ConfigScope) -> List[ConfigScope]: scope = self._scope(self.path, self.destination, parent_scope) if scope is not None: + if self.backwards_compat_fallback: + scope.backwards_compat_fallback = True self._scopes = [scope] return self._scopes diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index d60c3f4d38b10f..901eb0cbc58f34 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -935,50 +935,46 @@ def _warn_about_old_dotspack(): if not os.path.exists(old_dotspack): return - # Check if explicitly configured via config:locations - if spack.config.get("config:locations"): - # Any config:locations setting means explicit configuration, don't warn - return - - # Check if explicitly configured via environment variables - explicit_env_vars = [ - "SPACK_HOME", - "SPACK_DATA_HOME", - "SPACK_STATE_HOME", - "SPACK_CACHE_HOME", - "SPACK_USER_CACHE_PATH", - ] - if any(var in os.environ for var in explicit_env_vars): - # Explicitly configured, don't warn - return - # Helper to check if old_dotspack is a prefix def uses_old_dotspack(path): return path == old_dotspack or path.startswith(old_dotspack + os.sep) # Check if any config scope is using ~/.spack (means explicit configuration) + reasons = [] for scope in spack.config.CONFIG.scopes.values(): if hasattr(scope, "path") and uses_old_dotspack(scope.path): # Explicitly configured via a config scope, don't warn - return + if scope.backwards_compat_fallback: + reasons.append(f"Used by config scope: {scope.name}") + else: + # A config scope explicitly targets ~/.spack + return from spack.paths import locations as paths - # Check which paths are using ~/.spack - parts = [] + explicit_path = False if uses_old_dotspack(paths.state_home): - parts.append("user cache path is in use") + if paths.default_state_home_dot_spack: + reasons.append("User cache path fallback") + else: + explicit_path = True if uses_old_dotspack(paths.data_home): - parts.append("data home is in use") + explicit_path = True if uses_old_dotspack(paths.cache_home): - parts.append("cache home is in use") + explicit_path = True - # Only warn if something is using it - if not parts: + if explicit_path: return - usage = " and ".join(parts) - tty.warn(f"Old config/user-cache-path in `~/.spack` detected ({usage}). Run `spack migrate`") + msg = "Old config/user-cache-path in `~/.spack`" + if reasons: + msg += " - it is currently in use:" + for reason in reasons: + msg += f"\n\t{reason}" + else: + msg += " - it is not currently used by this spack instance." + msg += ("\nIf all spack instances are >= 1.2, you can use" + " `spack migrate --clear` to silence this warning") def _main(argv=None): diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 5c7f76a8ffad81..3bdb8e22346a37 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -111,6 +111,8 @@ def state_home(self): "XDG_STATE_HOME", ) + self.default_state_home_dot_spack = False + if new_layout_enforced(): self._state_home = state_home elif "SPACK_USER_CACHE_PATH" in os.environ: @@ -120,7 +122,7 @@ def state_home(self): self._state_home = state_home elif dir_is_occupied(self.base.old_default_dot_spack): self._state_home = self.base.old_default_dot_spack - # TODO [default state home case in old ~/.spack] + self.default_state_home_dot_spack = True else: self._state_home = state_home From fc41f0c3a21c38ee9242da4fb6603457622f23df Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 May 2026 20:09:30 -0700 Subject: [PATCH 453/506] forgot the part about printing the warning --- lib/spack/spack/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 901eb0cbc58f34..6874a06c36e83c 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -975,6 +975,7 @@ def uses_old_dotspack(path): msg += " - it is not currently used by this spack instance." msg += ("\nIf all spack instances are >= 1.2, you can use" " `spack migrate --clear` to silence this warning") + tty.warn(msg) def _main(argv=None): From a4d608861a8b6734b7696ce3cf65000770ed2f0a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 May 2026 20:12:50 -0700 Subject: [PATCH 454/506] style edits --- lib/spack/spack/cmd/migrate.py | 18 +++++++----------- lib/spack/spack/config.py | 1 - lib/spack/spack/main.py | 6 ++++-- lib/spack/spack/paths.py | 7 ++++++- lib/spack/spack/test/config.py | 6 +----- 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py index f8ba853c1cf8e3..f474a11e386ab2 100644 --- a/lib/spack/spack/cmd/migrate.py +++ b/lib/spack/spack/cmd/migrate.py @@ -25,7 +25,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: "--clear", action="store_true", help="move entire ~/.spack directory to backup location:" - "use this if no other instances need this old location", + "use this if no other instances need this old location", ) subparser.add_argument( "--restore-old-configs", @@ -53,8 +53,7 @@ def restore_old_configs(args: argparse.Namespace) -> None: if not backup_location: tty.die( - f"No backup found. Checked:\n" - + "\n".join(f" - {loc}" for loc in backup_locations) + "No backup found. Checked:\n" + "\n".join(f" - {loc}" for loc in backup_locations) ) # Check if ~/.spack already exists @@ -109,7 +108,8 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: if config_files: if os.path.exists(new_config_location): existing_configs = [ - f for f in os.listdir(new_config_location) + f + for f in os.listdir(new_config_location) if f.endswith(".yaml") or f.endswith(".yml") ] if existing_configs: @@ -119,9 +119,7 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: ) if not errors: - migrations.append( - ("config", config_files, old_location, new_config_location) - ) + migrations.append(("config", config_files, old_location, new_config_location)) # 2. Check for package repositories to migrate old_package_repos = os.path.join(old_location, "package_repos") @@ -141,15 +139,13 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: ) if not errors: - migrations.append( - ("package_repos", repos, old_package_repos, new_package_repos) - ) + migrations.append(("package_repos", repos, old_package_repos, new_package_repos)) if errors: tty.msg("Migration conflicts detected (files already in new locations):") for error in errors: tty.msg(f" {error}") - tty.msg(f"\nSkipping migration and backup/clear due to conflicts") + tty.msg("\nSkipping migration and backup/clear due to conflicts") # Exit early here regardless of --clear (we shouldn't move .spack if # we couldn't copy out the components we want) return diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 430bf5fdd6b79d..3f1c346b9dd383 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -34,7 +34,6 @@ import os.path import pathlib import re -import shutil import sys import tempfile from collections import defaultdict diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 6874a06c36e83c..1a852e62729368 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -973,8 +973,10 @@ def uses_old_dotspack(path): msg += f"\n\t{reason}" else: msg += " - it is not currently used by this spack instance." - msg += ("\nIf all spack instances are >= 1.2, you can use" - " `spack migrate --clear` to silence this warning") + msg += ( + "\nIf all spack instances are >= 1.2, you can use" + " `spack migrate --clear` to silence this warning" + ) tty.warn(msg) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 3bdb8e22346a37..25a0739443eac3 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -72,7 +72,12 @@ class Spack_vars(Enum): @classmethod def new_layout(cls): # Exclude SPACK_USER_CACHE_PATH - return [Spack_vars.state_home, Spack_vars.data_home, Spack_vars.cache_home, Spack_vars.home] + return [ + Spack_vars.state_home, + Spack_vars.data_home, + Spack_vars.cache_home, + Spack_vars.home, + ] # This is for tests that want to clean the environment of XDG_ variables that diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index d5ca00bebe1518..81384030216005 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -1722,11 +1722,7 @@ def backwards_compat_setup(tmp_path: pathlib.Path): # Path doesn't exist yet to_dir = tmp_path / "to" - entry = { - "path": str(to_dir), - "optional": True, - "backwards_compat": str(src_dir) - } + entry = {"path": str(to_dir), "optional": True, "backwards_compat": str(src_dir)} yield src_dir, to_dir, entry From 7897f67cba23a020f60016ef4b10cbaddbe135fb Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 May 2026 20:42:33 -0700 Subject: [PATCH 455/506] better error tracing --- lib/spack/spack/test/paths.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 52ba6f643e2154..f1e9afa69339df 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -63,24 +63,24 @@ def paths_base_nonempty_old_install(): def _install_path_checks(tmp_path, base_paths_generator, home_prefix, force_old_layout): - def check(paths, new_path): + def check(paths, new_path, msg): if force_old_layout: - assert paths.default_install_location == paths.base.old_install_path + assert paths.default_install_location == paths.base.old_install_path, msg else: - assert paths.default_install_location == str(new_path) + assert paths.default_install_location == str(new_path), msg new_default_installs_dir = _ensure_dir( pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs" ) (pathlib.Path(new_default_installs_dir) / "afile").touch() p0 = SpackPaths(base_paths_generator()) - check(p0, pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs") + check(p0, pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs", "p0: default location") # $XDG_DATA_HOME overrides the default xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") os.environ["XDG_DATA_HOME"] = xdg_data_home p7 = SpackPaths(base_paths_generator()) - check(p7, pathlib.Path(xdg_data_home) / "spack" / "installs") + check(p7, pathlib.Path(xdg_data_home) / "spack" / "installs", "p7: XDG_DATA_HOME override") spack.config.set("config:locations", {}) @@ -88,32 +88,32 @@ def check(paths, new_path): spack_home_cfg_prefix = _ensure_dir(tmp_path / "spack-home2") spack.config.set("config:locations:home", spack_home_cfg_prefix) p2 = SpackPaths(base_paths_generator()) - check(p2, pathlib.Path(spack_home_cfg_prefix) / ".local" / "share" / "spack" / "installs") + check(p2, pathlib.Path(spack_home_cfg_prefix) / ".local" / "share" / "spack" / "installs", "p2: config:locations:home override") # "config:locations:data" overrides the above spack_data_prefix = _ensure_dir(tmp_path / "spack-data") spack.config.set("config:locations:data", spack_data_prefix) p3 = SpackPaths(base_paths_generator()) - check(p3, pathlib.Path(spack_data_prefix) / "installs") + check(p3, pathlib.Path(spack_data_prefix) / "installs", "p3: config:locations:data override") # SPACK_HOME env variable overrides the above (even if there # are no installs there and there are installs in the old location) spack_home_env_prefix = _ensure_dir(tmp_path / "spack-home1") os.environ["SPACK_HOME"] = spack_home_env_prefix p1 = SpackPaths(base_paths_generator()) - check(p1, pathlib.Path(spack_home_env_prefix) / ".local" / "share" / "spack" / "installs") + check(p1, pathlib.Path(spack_home_env_prefix) / ".local" / "share" / "spack" / "installs", "p1: SPACK_HOME override") # Check that $SPACK_DATA_HOME overrides all the above spack_data_home = _ensure_dir(tmp_path / "spack_data_home") os.environ["SPACK_DATA_HOME"] = spack_data_home p5 = SpackPaths(base_paths_generator()) - check(p5, pathlib.Path(spack_data_home) / "installs") + check(p5, pathlib.Path(spack_data_home) / "installs", "p5: SPACK_DATA_HOME override") # Disable all location-based env vars: this will then defer # to using "config:locations:data" spack.config.set("config:locations:disable_env", True) p6 = SpackPaths(base_paths_generator()) - check(p6, pathlib.Path(spack_data_prefix) / "installs") + check(p6, pathlib.Path(spack_data_prefix) / "installs", "p6: disable_env defers to config:locations:data") def test_system_config_path_is_overridable(working_env, tmp_path): From 28ee85143c0d28dbdf92ff310693bc4f4b82af31 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 May 2026 21:02:01 -0700 Subject: [PATCH 456/506] fix one test (explicit new-style location settings force new layout usage even if old layout exists) --- lib/spack/spack/test/paths.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index f1e9afa69339df..94d67157a122c4 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -63,24 +63,27 @@ def paths_base_nonempty_old_install(): def _install_path_checks(tmp_path, base_paths_generator, home_prefix, force_old_layout): - def check(paths, new_path, msg): + def checka(paths, new_path, msg): if force_old_layout: assert paths.default_install_location == paths.base.old_install_path, msg else: assert paths.default_install_location == str(new_path), msg + def checkb(paths, new_path, msg): + assert paths.default_install_location == str(new_path), msg + new_default_installs_dir = _ensure_dir( pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs" ) (pathlib.Path(new_default_installs_dir) / "afile").touch() p0 = SpackPaths(base_paths_generator()) - check(p0, pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs", "p0: default location") + checka(p0, pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs", "p0: default location") # $XDG_DATA_HOME overrides the default xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") os.environ["XDG_DATA_HOME"] = xdg_data_home p7 = SpackPaths(base_paths_generator()) - check(p7, pathlib.Path(xdg_data_home) / "spack" / "installs", "p7: XDG_DATA_HOME override") + checka(p7, pathlib.Path(xdg_data_home) / "spack" / "installs", "p7: XDG_DATA_HOME override") spack.config.set("config:locations", {}) @@ -88,32 +91,32 @@ def check(paths, new_path, msg): spack_home_cfg_prefix = _ensure_dir(tmp_path / "spack-home2") spack.config.set("config:locations:home", spack_home_cfg_prefix) p2 = SpackPaths(base_paths_generator()) - check(p2, pathlib.Path(spack_home_cfg_prefix) / ".local" / "share" / "spack" / "installs", "p2: config:locations:home override") + checkb(p2, pathlib.Path(spack_home_cfg_prefix) / ".local" / "share" / "spack" / "installs", "p2: config:locations:home override") # "config:locations:data" overrides the above spack_data_prefix = _ensure_dir(tmp_path / "spack-data") spack.config.set("config:locations:data", spack_data_prefix) p3 = SpackPaths(base_paths_generator()) - check(p3, pathlib.Path(spack_data_prefix) / "installs", "p3: config:locations:data override") + checkb(p3, pathlib.Path(spack_data_prefix) / "installs", "p3: config:locations:data override") # SPACK_HOME env variable overrides the above (even if there # are no installs there and there are installs in the old location) spack_home_env_prefix = _ensure_dir(tmp_path / "spack-home1") os.environ["SPACK_HOME"] = spack_home_env_prefix p1 = SpackPaths(base_paths_generator()) - check(p1, pathlib.Path(spack_home_env_prefix) / ".local" / "share" / "spack" / "installs", "p1: SPACK_HOME override") + checkb(p1, pathlib.Path(spack_home_env_prefix) / ".local" / "share" / "spack" / "installs", "p1: SPACK_HOME override") # Check that $SPACK_DATA_HOME overrides all the above spack_data_home = _ensure_dir(tmp_path / "spack_data_home") os.environ["SPACK_DATA_HOME"] = spack_data_home p5 = SpackPaths(base_paths_generator()) - check(p5, pathlib.Path(spack_data_home) / "installs", "p5: SPACK_DATA_HOME override") + checkb(p5, pathlib.Path(spack_data_home) / "installs", "p5: SPACK_DATA_HOME override") # Disable all location-based env vars: this will then defer # to using "config:locations:data" spack.config.set("config:locations:disable_env", True) p6 = SpackPaths(base_paths_generator()) - check(p6, pathlib.Path(spack_data_prefix) / "installs", "p6: disable_env defers to config:locations:data") + checkb(p6, pathlib.Path(spack_data_prefix) / "installs", "p6: disable_env defers to config:locations:data") def test_system_config_path_is_overridable(working_env, tmp_path): From 4010af9751a4fefd70ec761a3a6efca558ba65b9 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 May 2026 21:02:43 -0700 Subject: [PATCH 457/506] style edit --- lib/spack/spack/test/paths.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 94d67157a122c4..636669574b7a47 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -77,7 +77,11 @@ def checkb(paths, new_path, msg): ) (pathlib.Path(new_default_installs_dir) / "afile").touch() p0 = SpackPaths(base_paths_generator()) - checka(p0, pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs", "p0: default location") + checka( + p0, + pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs", + "p0: default location", + ) # $XDG_DATA_HOME overrides the default xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") @@ -91,7 +95,11 @@ def checkb(paths, new_path, msg): spack_home_cfg_prefix = _ensure_dir(tmp_path / "spack-home2") spack.config.set("config:locations:home", spack_home_cfg_prefix) p2 = SpackPaths(base_paths_generator()) - checkb(p2, pathlib.Path(spack_home_cfg_prefix) / ".local" / "share" / "spack" / "installs", "p2: config:locations:home override") + checkb( + p2, + pathlib.Path(spack_home_cfg_prefix) / ".local" / "share" / "spack" / "installs", + "p2: config:locations:home override", + ) # "config:locations:data" overrides the above spack_data_prefix = _ensure_dir(tmp_path / "spack-data") @@ -104,7 +112,11 @@ def checkb(paths, new_path, msg): spack_home_env_prefix = _ensure_dir(tmp_path / "spack-home1") os.environ["SPACK_HOME"] = spack_home_env_prefix p1 = SpackPaths(base_paths_generator()) - checkb(p1, pathlib.Path(spack_home_env_prefix) / ".local" / "share" / "spack" / "installs", "p1: SPACK_HOME override") + checkb( + p1, + pathlib.Path(spack_home_env_prefix) / ".local" / "share" / "spack" / "installs", + "p1: SPACK_HOME override", + ) # Check that $SPACK_DATA_HOME overrides all the above spack_data_home = _ensure_dir(tmp_path / "spack_data_home") @@ -116,7 +128,11 @@ def checkb(paths, new_path, msg): # to using "config:locations:data" spack.config.set("config:locations:disable_env", True) p6 = SpackPaths(base_paths_generator()) - checkb(p6, pathlib.Path(spack_data_prefix) / "installs", "p6: disable_env defers to config:locations:data") + checkb( + p6, + pathlib.Path(spack_data_prefix) / "installs", + "p6: disable_env defers to config:locations:data", + ) def test_system_config_path_is_overridable(working_env, tmp_path): From ce3cce851e709361709788245adaf0e78305c128 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 14 May 2026 21:24:33 -0700 Subject: [PATCH 458/506] fix second test; add TODO --- lib/spack/spack/paths.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 25a0739443eac3..4f0515e0534cd4 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -403,12 +403,13 @@ def freeze(): "state_home": locations.state_home, "data_home": locations.data_home, "cache_home": locations.cache_home, - "old_layout": locations.old_layout, + "old_layout_detected": locations.old_layout_detected, } + # TODO: also need to propagate new_layout_enforced() def restore(bundled_state): locations._state_home = bundled_state["state_home"] locations._data_home = bundled_state["data_home"] locations._cache_home = bundled_state["cache_home"] - locations.old_layout = bundled_state["old_layout"] + locations.old_layout_detected = bundled_state["old_layout_detected"] From 0c6317934b13d8a1ecb81e6c0795fa190cd4a764 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 15 May 2026 15:04:35 -0700 Subject: [PATCH 459/506] new command bash/etc. autocompletion --- share/spack/spack-completion.bash | 6 +++++- share/spack/spack-completion.fish | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index d687f3c88aca19..1b00018df52255 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -396,7 +396,7 @@ _spack() { then SPACK_COMPREPLY="--color -v --verbose -k --insecure -b --bootstrap -V --version -h --help -H --all-help -c --config -C --config-scope -e --env -D --env-dir -E --no-env --use-env-repo -d --debug -t --backtrace --pdb --timestamp -m --mock --print-shell-vars --stacktrace --warn-writes-into-spack -l --enable-locks -L --disable-locks -p --profile --profile-file --sorted-profile --lines" else - SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse logs maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" + SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse logs maintainers make-installer mark migrate mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" fi } @@ -1457,6 +1457,10 @@ _spack_mark() { fi } +_spack_migrate() { + SPACK_COMPREPLY="-h --help --dry-run --clear --restore-old-configs" +} + _spack_mirror() { if $list_options then diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 426a1ed4ea7055..971012ab465073 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -398,6 +398,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a logs -d 'print ou complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a maintainers -d 'get information about package maintainers' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a make-installer -d 'generate Windows installer' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a mark -d 'mark packages as explicitly or implicitly installed' +complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a migrate -d 'migrate user config and cache from old to new locations' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a mirror -d 'manage mirrors (source and binary)' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a module -d 'generate/manage module files' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a patch -d 'patch expanded sources in preparation for install' @@ -2357,6 +2358,17 @@ complete -c spack -n '__fish_spack_using_command mark' -s e -l explicit -d 'mark complete -c spack -n '__fish_spack_using_command mark' -s i -l implicit -f -a implicit complete -c spack -n '__fish_spack_using_command mark' -s i -l implicit -d 'mark packages as implicitly installed' +# spack migrate +set -g __fish_spack_optspecs_spack_migrate h/help dry-run clear restore-old-configs +complete -c spack -n '__fish_spack_using_command migrate' -s h -l help -f -a help +complete -c spack -n '__fish_spack_using_command migrate' -s h -l help -d 'show this help message and exit' +complete -c spack -n '__fish_spack_using_command migrate' -l dry-run -f -a dry_run +complete -c spack -n '__fish_spack_using_command migrate' -l dry-run -d 'show what would be migrated without actually moving files' +complete -c spack -n '__fish_spack_using_command migrate' -l clear -f -a clear +complete -c spack -n '__fish_spack_using_command migrate' -l clear -d 'move entire ~/.spack directory to backup location:use this if no other instances need this old location' +complete -c spack -n '__fish_spack_using_command migrate' -l restore-old-configs -f -a restore_old_configs +complete -c spack -n '__fish_spack_using_command migrate' -l restore-old-configs -d 'restore ~/.spack from backup location (after --clear)' + # spack mirror set -g __fish_spack_optspecs_spack_mirror h/help n/no-checksum complete -c spack -n '__fish_spack_using_command_pos 0 mirror' -f -a create -d 'create a directory to be used as a spack mirror, and fill it with package archives' From 9c389a202d12b31d07f7979feafb629e3350f39e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 15 May 2026 15:16:37 -0700 Subject: [PATCH 460/506] subproc transmit extra state --- lib/spack/spack/paths.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 4f0515e0534cd4..5a7971aaf30d2c 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -105,6 +105,13 @@ def __init__(self, base): ) self.old_layout_detected = detect_old_spack_layout(base) + self._new_layout_enforced = None + + @property + def new_layout_enforced(self): + if not self._new_layout_enforced: + self._new_layout_enforced = new_layout_enforced() + return self._new_layout_enforced @property def state_home(self): @@ -404,8 +411,8 @@ def freeze(): "data_home": locations.data_home, "cache_home": locations.cache_home, "old_layout_detected": locations.old_layout_detected, + "new_layout_enforced": new_layout_enforced(), } - # TODO: also need to propagate new_layout_enforced() def restore(bundled_state): @@ -413,3 +420,4 @@ def restore(bundled_state): locations._data_home = bundled_state["data_home"] locations._cache_home = bundled_state["cache_home"] locations.old_layout_detected = bundled_state["old_layout_detected"] + locations._new_layout_enforced = bundled_state["new_layout_enforced"] From d1fa8fc577e41e8adc8970fa58b2c89f79e15c2a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 15 May 2026 15:28:39 -0700 Subject: [PATCH 461/506] fix some reference errors --- lib/spack/spack/paths.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 5a7971aaf30d2c..9d0ec0fa258fe3 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -109,7 +109,7 @@ def __init__(self, base): @property def new_layout_enforced(self): - if not self._new_layout_enforced: + if self._new_layout_enforced is None: self._new_layout_enforced = new_layout_enforced() return self._new_layout_enforced @@ -125,7 +125,7 @@ def state_home(self): self.default_state_home_dot_spack = False - if new_layout_enforced(): + if self.new_layout_enforced: self._state_home = state_home elif "SPACK_USER_CACHE_PATH" in os.environ: # If we're here, SPACK_STATE_HOME is not in os.environ @@ -363,7 +363,7 @@ def env_check(env_vars, provenance, rel=None): def _decide_old_or_new_location( self, old_location, new_location, default_new_location, provenance ): - if new_layout_enforced(): + if self.new_layout_enforced: return new_location if self.old_layout_detected: return old_location From 440a2a1306b6805c0f052ba6a97c8bcab92ca851 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 15 May 2026 15:56:25 -0700 Subject: [PATCH 462/506] reference error --- lib/spack/spack/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 9d0ec0fa258fe3..8d47a04ee564eb 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -201,7 +201,7 @@ def default_envs_path(self): @property def default_license_dir(self): - if self.old_layout: + if self.old_layout_detected: return self.base.old_licenses_path else: return os.path.join(self.data_home, "licenses") From cabb9ab5c7678448e0aa6e094b69fa0d6ea025e3 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 15 May 2026 17:17:30 -0700 Subject: [PATCH 463/506] test of state_home/user_cache_path --- lib/spack/spack/test/paths.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 636669574b7a47..652850921d5e14 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -135,6 +135,31 @@ def checkb(paths, new_path, msg): ) +def test_state_home(working_env, tmp_path, mutable_config, set_home): + base_prefix = _ensure_dir(tmp_path / "spack-root") + home_prefix = _ensure_dir(tmp_path / "home-prefix") + + set_home(home_prefix) + pb = SpackPathsBase(base_prefix) + + p0 = SpackPaths(pb) + # if neither old nor new dir is occupied, choose new + assert p0.state_home == str(pathlib.Path(home_prefix) / ".local" / "state" / "spack") + + old_user_cache_path = pathlib.Path(home_prefix) / ".spack" + old_user_cache_path.mkdir(parents=True) + (old_user_cache_path / "afile").touch() + p1 = SpackPaths(pb) + # if new dir does not exist, and old dir does, choose old + assert p1.state_home == str(old_user_cache_path) + + os.environ["SPACK_DATA_HOME"] = base_prefix + p2 = SpackPaths(pb) + # even though new dir does not exist, and old dir does, if the user sets a variable + # like SPACK_DATA_HOME use the new default location for state_home/user_cache_path + assert p2.state_home == str(pathlib.Path(home_prefix) / ".local" / "state" / "spack") + + def test_system_config_path_is_overridable(working_env, tmp_path): redirect_syscfg_path = str(pathlib.Path(tmp_path) / "redirected_syscfg") os.environ["SPACK_SYSTEM_CONFIG_PATH"] = redirect_syscfg_path From 40846b34423c07354dff7567a8c1e1e4ae8fafa4 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 15 May 2026 17:36:24 -0700 Subject: [PATCH 464/506] update state_home behavior: choose old dotspack even if SPACK_DATA_HOME is set --- lib/spack/spack/paths.py | 12 ++++++++---- lib/spack/spack/test/paths.py | 18 ++++++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 8d47a04ee564eb..09e9b3576c6f6a 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -125,11 +125,15 @@ def state_home(self): self.default_state_home_dot_spack = False - if self.new_layout_enforced: + def cfg_state_home(): + return config.get("config:locations:home", None) or config.get("config:locations:state", None) + + def env_state_home(): + disable_env = config.get("config:locations:disable_env", False) + return not disable_env and any(x in os.environ for x in ["SPACK_USER_CACHE_PATH", "SPACK_STATE_HOME", "SPACK_HOME"]) + + if env_state_home() or cfg_state_home(): self._state_home = state_home - elif "SPACK_USER_CACHE_PATH" in os.environ: - # If we're here, SPACK_STATE_HOME is not in os.environ - self._state_home = os.environ.get("SPACK_USER_CACHE_PATH") elif dir_is_occupied(state_home): self._state_home = state_home elif dir_is_occupied(self.base.old_default_dot_spack): diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index 652850921d5e14..da0becf1f56a4e 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -142,9 +142,11 @@ def test_state_home(working_env, tmp_path, mutable_config, set_home): set_home(home_prefix) pb = SpackPathsBase(base_prefix) + new_user_cache_path = str(pathlib.Path(home_prefix) / ".local" / "state" / "spack") + p0 = SpackPaths(pb) # if neither old nor new dir is occupied, choose new - assert p0.state_home == str(pathlib.Path(home_prefix) / ".local" / "state" / "spack") + assert p0.state_home == new_user_cache_path old_user_cache_path = pathlib.Path(home_prefix) / ".spack" old_user_cache_path.mkdir(parents=True) @@ -155,9 +157,17 @@ def test_state_home(working_env, tmp_path, mutable_config, set_home): os.environ["SPACK_DATA_HOME"] = base_prefix p2 = SpackPaths(pb) - # even though new dir does not exist, and old dir does, if the user sets a variable - # like SPACK_DATA_HOME use the new default location for state_home/user_cache_path - assert p2.state_home == str(pathlib.Path(home_prefix) / ".local" / "state" / "spack") + # While setting any new-style location env var or config:locations value + # relocates installs, gpg keys, downloads, etc. It does not make spack + # choose ~/.local/state/spack over ~/.spack + assert p2.state_home == str(old_user_cache_path) + + os.environ["SPACK_HOME"] = home_prefix + p3 = SpackPaths(pb) + # ... SPACK_HOME is special amongst these variables though, because by + # definition it is supposed to relocate everything except for config + # (caches included) + assert p3.state_home == new_user_cache_path def test_system_config_path_is_overridable(working_env, tmp_path): From f14b24d1b385edf4a082b21c4da86d0f3dbc8ea2 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 15 May 2026 17:37:34 -0700 Subject: [PATCH 465/506] style update --- lib/spack/spack/paths.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 09e9b3576c6f6a..f5a5971ad6d136 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -126,12 +126,17 @@ def state_home(self): self.default_state_home_dot_spack = False def cfg_state_home(): - return config.get("config:locations:home", None) or config.get("config:locations:state", None) + return config.get("config:locations:home", None) or config.get( + "config:locations:state", None + ) def env_state_home(): disable_env = config.get("config:locations:disable_env", False) - return not disable_env and any(x in os.environ for x in ["SPACK_USER_CACHE_PATH", "SPACK_STATE_HOME", "SPACK_HOME"]) - + return not disable_env and any( + x in os.environ + for x in ["SPACK_USER_CACHE_PATH", "SPACK_STATE_HOME", "SPACK_HOME"] + ) + if env_state_home() or cfg_state_home(): self._state_home = state_home elif dir_is_occupied(state_home): From 22bee19c746c92c93770065e27bd377d860316e3 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sat, 16 May 2026 15:46:32 -0700 Subject: [PATCH 466/506] initial tests for migrate --- lib/spack/spack/test/cmd/migrate.py | 295 ++++++++++++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 lib/spack/spack/test/cmd/migrate.py diff --git a/lib/spack/spack/test/cmd/migrate.py b/lib/spack/spack/test/cmd/migrate.py new file mode 100644 index 00000000000000..1fbd68817efeb2 --- /dev/null +++ b/lib/spack/spack/test/cmd/migrate.py @@ -0,0 +1,295 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import pathlib + +import pytest + +import spack.cmd.migrate +import spack.main +import spack.paths +import spack.paths_base +from spack.paths import SpackPaths +from spack.paths_base import SpackPathsBase + +migrate = spack.main.SpackCommand("migrate") + + +@pytest.fixture(autouse=True) +def clear_env_vars(working_env): + spack.paths._unset_path_vars(os.environ) + + +def create_dotspack_files(base_path): + """Create minimal test files under a base path to simulate ~/.spack structure. + + Args: + base_path: Path-like object or string where files should be created + + Returns: + dict: mapping of created file/dir paths to their expected content/structure + """ + base = pathlib.Path(base_path) + + # Create config files with verifiable content + config_yaml = base / "config.yaml" + config_yaml.write_text("config:\n build_jobs: 8\n") + + packages_yaml = base / "packages.yaml" + packages_yaml.write_text("packages:\n all:\n compiler: [gcc]\n") + + # Create package_repos directory with a test repo + package_repos_dir = base / "package_repos" + package_repos_dir.mkdir(parents=True, exist_ok=True) + + test_repo = package_repos_dir / "test_repo" + test_repo.mkdir(parents=True, exist_ok=True) + + # Add a minimal repo.yaml to the test repo + repo_yaml = test_repo / "repo.yaml" + repo_yaml.write_text("repo:\n namespace: test\n") + + # Add a packages directory + packages_dir = test_repo / "packages" + packages_dir.mkdir(parents=True, exist_ok=True) + + # Return mapping of what we created + return { + "config_files": ["config.yaml", "packages.yaml"], + "package_repos": ["test_repo"], + "test_repo_structure": { + "repo.yaml": "repo:\n namespace: test\n", + "packages": {}, # directory + } + } + + +def verify_files_copied(source_base, dest_base, created_files): + """Verify that files from source_base have been copied to dest_base. + + Args: + source_base: Path-like source directory + dest_base: Path-like destination directory + created_files: dict returned from create_dotspack_files + + Returns: + bool: True if all expected files exist in dest_base with correct content + """ + source = pathlib.Path(source_base) + dest = pathlib.Path(dest_base) + + # Check if dest directory exists + if not dest.exists(): + print(f"Destination directory does not exist: {dest}") + return False + + # Check config files + for config_file in created_files["config_files"]: + source_file = source / config_file + dest_file = dest / config_file + + if not dest_file.exists(): + print(f"Expected file does not exist: {dest_file}") + print(f"Files in {dest}: {list(dest.iterdir()) if dest.exists() else 'dir does not exist'}") + return False + + # Verify content matches + if source_file.read_text() != dest_file.read_text(): + print(f"Content mismatch for {config_file}") + return False + + return True + + +def verify_package_repos_copied(source_repos_base, dest_repos_base, created_files): + """Verify that package repos have been copied correctly. + + Args: + source_repos_base: Path-like source package_repos directory + dest_repos_base: Path-like destination package_repos directory + created_files: dict returned from create_dotspack_files + + Returns: + bool: True if all expected repos exist in dest with correct structure + """ + source_base = pathlib.Path(source_repos_base) + dest_base = pathlib.Path(dest_repos_base) + + for repo_name in created_files["package_repos"]: + source_repo = source_base / repo_name + dest_repo = dest_base / repo_name + + if not dest_repo.exists(): + return False + + # Check for repo.yaml + source_repo_yaml = source_repo / "repo.yaml" + dest_repo_yaml = dest_repo / "repo.yaml" + + if not dest_repo_yaml.exists(): + return False + + if source_repo_yaml.read_text() != dest_repo_yaml.read_text(): + return False + + # Check for packages directory + if not (dest_repo / "packages").is_dir(): + return False + + return True + + +def test_migrate_basic(tmp_path, set_home, monkeypatch, mutable_config): + """Test basic migrate: copies files to new locations, preserves old location.""" + # Set up directories + spack_root = tmp_path / "spack-root" + spack_root.mkdir() + home = tmp_path / "home" + home.mkdir() + + # Set home BEFORE creating paths objects + set_home(str(home)) + + # Create ~/.spack with test files + dotspack = home / ".spack" + dotspack.mkdir() + created = create_dotspack_files(dotspack) + + # Set expected new locations + new_config = home / ".config" / "spack" + new_state = home / ".local" / "state" / "spack" + + # Create fresh SpackPaths object and patch the module + base_paths = SpackPathsBase(str(spack_root)) + paths = SpackPaths(base_paths) + monkeypatch.setattr(spack.paths, "locations", paths) + monkeypatch.setattr(spack.paths_base, "locations", base_paths) + + # Also need to patch the cmd.migrate module which has already imported these + monkeypatch.setattr(spack.cmd.migrate, "paths", paths) + monkeypatch.setattr(spack.cmd.migrate, "paths_base", base_paths) + + # Run migrate + migrate() + + # Verify config files were copied + assert verify_files_copied(dotspack, new_config, created) + + # Verify package repos were copied + old_repos = dotspack / "package_repos" + new_repos = new_state / "package_repos" + assert verify_package_repos_copied(old_repos, new_repos, created) + + # Verify old location still exists (not deleted) + assert dotspack.exists() + for config_file in created["config_files"]: + assert (dotspack / config_file).exists() + + +def test_migrate_with_clear(tmp_path, set_home, monkeypatch, mutable_config): + """Test migrate --clear: copies files and removes ~/.spack.""" + # Set up directories + spack_root = tmp_path / "spack-root" + spack_root.mkdir() + home = tmp_path / "home" + home.mkdir() + + # Set home BEFORE creating paths objects + set_home(str(home)) + + # Create ~/.spack with test files + dotspack = home / ".spack" + dotspack.mkdir() + created = create_dotspack_files(dotspack) + + # Set expected new locations + new_config = home / ".config" / "spack" + new_state = home / ".local" / "state" / "spack" + + # Create fresh SpackPaths object and patch the module + base_paths = SpackPathsBase(str(spack_root)) + paths = SpackPaths(base_paths) + monkeypatch.setattr(spack.paths, "locations", paths) + monkeypatch.setattr(spack.paths_base, "locations", base_paths) + + # Also need to patch the cmd.migrate module which has already imported these + monkeypatch.setattr(spack.cmd.migrate, "paths", paths) + monkeypatch.setattr(spack.cmd.migrate, "paths_base", base_paths) + + # The backup location is now paths.dotspack_backup + backup_location = pathlib.Path(paths.dotspack_backup) + + # Run migrate with --clear + migrate("--clear") + + # Verify config files were copied + assert verify_files_copied(backup_location, new_config, created) + + # Verify package repos were copied + old_repos = backup_location / "package_repos" + new_repos = new_state / "package_repos" + assert verify_package_repos_copied(old_repos, new_repos, created) + + # Verify old location no longer exists + assert not dotspack.exists() + + # Verify backup exists + assert backup_location.exists() + + +def test_migrate_then_clear(tmp_path, set_home, monkeypatch, mutable_config): + """Test migrate, then migrate --clear separately.""" + # Set up directories + spack_root = tmp_path / "spack-root" + spack_root.mkdir() + home = tmp_path / "home" + home.mkdir() + + # Set home BEFORE creating paths objects + set_home(str(home)) + + # Create ~/.spack with test files + dotspack = home / ".spack" + dotspack.mkdir() + created = create_dotspack_files(dotspack) + + # Set expected new locations + new_config = home / ".config" / "spack" + new_state = home / ".local" / "state" / "spack" + + # Create fresh SpackPaths object and patch the module + base_paths = SpackPathsBase(str(spack_root)) + paths = SpackPaths(base_paths) + monkeypatch.setattr(spack.paths, "locations", paths) + monkeypatch.setattr(spack.paths_base, "locations", base_paths) + + # Also need to patch the cmd.migrate module which has already imported these + monkeypatch.setattr(spack.cmd.migrate, "paths", paths) + monkeypatch.setattr(spack.cmd.migrate, "paths_base", base_paths) + + # Run migrate (without --clear) + migrate() + + # Verify files were copied + assert verify_files_copied(dotspack, new_config, created) + old_repos = dotspack / "package_repos" + new_repos = new_state / "package_repos" + assert verify_package_repos_copied(old_repos, new_repos, created) + + # Verify old location still exists + assert dotspack.exists() + + # Now run migrate --clear + backup_location = pathlib.Path(paths.dotspack_backup) + + migrate("--clear") + + # Verify old location is gone + assert not dotspack.exists() + + # Verify backup exists with the original content + assert backup_location.exists() + for config_file in created["config_files"]: + assert (backup_location / config_file).exists() From 4be31a21d46b96c6e2e717a09caf5f81532a2175 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sat, 16 May 2026 16:17:03 -0700 Subject: [PATCH 467/506] 1 more test, tests now passing --- lib/spack/spack/cmd/migrate.py | 38 ++++++++++++++++ lib/spack/spack/test/cmd/migrate.py | 68 +++++++++++++++++++++++++++-- 2 files changed, 102 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py index f474a11e386ab2..c1fea9b027fd2a 100644 --- a/lib/spack/spack/cmd/migrate.py +++ b/lib/spack/spack/cmd/migrate.py @@ -27,6 +27,18 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: help="move entire ~/.spack directory to backup location:" "use this if no other instances need this old location", ) + subparser.add_argument( + "--clear-only", + action="store_true", + help="only move ~/.spack to backup without migrating files " + "(useful after running migrate without --clear)", + ) + subparser.add_argument( + "--replace", + action="store_true", + help="replace existing files in new locations (use with --clear " + "if you forgot to use --clear on first run)", + ) subparser.add_argument( "--restore-old-configs", action="store_true", @@ -86,6 +98,32 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: new_state_location = os.path.join(os.path.expanduser("~"), ".local", "state", "spack") backup_location = paths.dotspack_backup + # Handle --clear-only mode + if args.clear_only: + if not os.path.exists(old_location): + tty.die(f"Old configuration location does not exist: {old_location}") + if os.path.exists(backup_location): + tty.die(f"Backup location already exists: {backup_location}") + + if args.dry_run: + tty.msg(f"Would move {old_location} to {backup_location}") + return + + os.makedirs(os.path.dirname(backup_location), exist_ok=True) + shutil.move(old_location, backup_location) + tty.msg(f"Backup complete! Original ~/.spack moved to {backup_location}") + return + + # Handle --replace: delete destination directories before migrating + if args.replace: + if os.path.exists(new_config_location): + tty.msg(f"Removing existing config location: {new_config_location}") + shutil.rmtree(new_config_location) + new_package_repos = os.path.join(new_state_location, "package_repos") + if os.path.exists(new_package_repos): + tty.msg(f"Removing existing package repos: {new_package_repos}") + shutil.rmtree(new_package_repos) + # Check if old location exists if not os.path.exists(old_location): tty.die(f"Old configuration location does not exist: {old_location}") diff --git a/lib/spack/spack/test/cmd/migrate.py b/lib/spack/spack/test/cmd/migrate.py index 1fbd68817efeb2..0743d88a2efe3c 100644 --- a/lib/spack/spack/test/cmd/migrate.py +++ b/lib/spack/spack/test/cmd/migrate.py @@ -239,8 +239,8 @@ def test_migrate_with_clear(tmp_path, set_home, monkeypatch, mutable_config): assert backup_location.exists() -def test_migrate_then_clear(tmp_path, set_home, monkeypatch, mutable_config): - """Test migrate, then migrate --clear separately.""" +def test_migrate_then_clear_replace(tmp_path, set_home, monkeypatch, mutable_config): + """Test migrate, then migrate --clear --replace to clean up.""" # Set up directories spack_root = tmp_path / "spack-root" spack_root.mkdir() @@ -281,10 +281,10 @@ def test_migrate_then_clear(tmp_path, set_home, monkeypatch, mutable_config): # Verify old location still exists assert dotspack.exists() - # Now run migrate --clear + # Now run migrate --clear --replace (to remove existing files and clear ~/.spack) backup_location = pathlib.Path(paths.dotspack_backup) - migrate("--clear") + migrate("--clear", "--replace") # Verify old location is gone assert not dotspack.exists() @@ -293,3 +293,63 @@ def test_migrate_then_clear(tmp_path, set_home, monkeypatch, mutable_config): assert backup_location.exists() for config_file in created["config_files"]: assert (backup_location / config_file).exists() + + +def test_migrate_then_clear_only(tmp_path, set_home, monkeypatch, mutable_config): + """Test migrate, then migrate --clear-only to just remove ~/.spack.""" + # Set up directories + spack_root = tmp_path / "spack-root" + spack_root.mkdir() + home = tmp_path / "home" + home.mkdir() + + # Set home BEFORE creating paths objects + set_home(str(home)) + + # Create ~/.spack with test files + dotspack = home / ".spack" + dotspack.mkdir() + created = create_dotspack_files(dotspack) + + # Set expected new locations + new_config = home / ".config" / "spack" + new_state = home / ".local" / "state" / "spack" + + # Create fresh SpackPaths object and patch the module + base_paths = SpackPathsBase(str(spack_root)) + paths = SpackPaths(base_paths) + monkeypatch.setattr(spack.paths, "locations", paths) + monkeypatch.setattr(spack.paths_base, "locations", base_paths) + + # Also need to patch the cmd.migrate module which has already imported these + monkeypatch.setattr(spack.cmd.migrate, "paths", paths) + monkeypatch.setattr(spack.cmd.migrate, "paths_base", base_paths) + + # Run migrate (without --clear) + migrate() + + # Verify files were copied + assert verify_files_copied(dotspack, new_config, created) + old_repos = dotspack / "package_repos" + new_repos = new_state / "package_repos" + assert verify_package_repos_copied(old_repos, new_repos, created) + + # Verify old location still exists + assert dotspack.exists() + + # Now run migrate --clear-only (just move ~/.spack without re-migrating) + backup_location = pathlib.Path(paths.dotspack_backup) + + migrate("--clear-only") + + # Verify old location is gone + assert not dotspack.exists() + + # Verify backup exists with the original content + assert backup_location.exists() + for config_file in created["config_files"]: + assert (backup_location / config_file).exists() + + # Verify new locations still have the files (weren't touched) + assert verify_files_copied(backup_location, new_config, created) + assert verify_package_repos_copied(backup_location / "package_repos", new_repos, created) From 0c4e8d17e40bf4dc9b58319a7b916f9803e3c157 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sun, 17 May 2026 18:41:43 -0700 Subject: [PATCH 468/506] update 'spack migrate' options --- share/spack/spack-completion.bash | 2 +- share/spack/spack-completion.fish | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 1b00018df52255..e3ecd9e6870526 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -1458,7 +1458,7 @@ _spack_mark() { } _spack_migrate() { - SPACK_COMPREPLY="-h --help --dry-run --clear --restore-old-configs" + SPACK_COMPREPLY="-h --help --dry-run --clear --clear-only --replace --restore-old-configs" } _spack_mirror() { diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 971012ab465073..90ac230bd2bfe8 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -2359,13 +2359,17 @@ complete -c spack -n '__fish_spack_using_command mark' -s i -l implicit -f -a im complete -c spack -n '__fish_spack_using_command mark' -s i -l implicit -d 'mark packages as implicitly installed' # spack migrate -set -g __fish_spack_optspecs_spack_migrate h/help dry-run clear restore-old-configs +set -g __fish_spack_optspecs_spack_migrate h/help dry-run clear clear-only replace restore-old-configs complete -c spack -n '__fish_spack_using_command migrate' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command migrate' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command migrate' -l dry-run -f -a dry_run complete -c spack -n '__fish_spack_using_command migrate' -l dry-run -d 'show what would be migrated without actually moving files' complete -c spack -n '__fish_spack_using_command migrate' -l clear -f -a clear complete -c spack -n '__fish_spack_using_command migrate' -l clear -d 'move entire ~/.spack directory to backup location:use this if no other instances need this old location' +complete -c spack -n '__fish_spack_using_command migrate' -l clear-only -f -a clear_only +complete -c spack -n '__fish_spack_using_command migrate' -l clear-only -d 'only move ~/.spack to backup without migrating files (useful after running migrate without --clear)' +complete -c spack -n '__fish_spack_using_command migrate' -l replace -f -a replace +complete -c spack -n '__fish_spack_using_command migrate' -l replace -d 'replace existing files in new locations (use with --clear if you forgot to use --clear on first run)' complete -c spack -n '__fish_spack_using_command migrate' -l restore-old-configs -f -a restore_old_configs complete -c spack -n '__fish_spack_using_command migrate' -l restore-old-configs -d 'restore ~/.spack from backup location (after --clear)' From 5cfa5b9b435ab0532029b81b296adc972c7af600 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sun, 17 May 2026 18:45:34 -0700 Subject: [PATCH 469/506] style checks --- lib/spack/spack/test/cmd/migrate.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/cmd/migrate.py b/lib/spack/spack/test/cmd/migrate.py index 0743d88a2efe3c..2b2073729d788c 100644 --- a/lib/spack/spack/test/cmd/migrate.py +++ b/lib/spack/spack/test/cmd/migrate.py @@ -62,7 +62,7 @@ def create_dotspack_files(base_path): "test_repo_structure": { "repo.yaml": "repo:\n namespace: test\n", "packages": {}, # directory - } + }, } @@ -92,7 +92,10 @@ def verify_files_copied(source_base, dest_base, created_files): if not dest_file.exists(): print(f"Expected file does not exist: {dest_file}") - print(f"Files in {dest}: {list(dest.iterdir()) if dest.exists() else 'dir does not exist'}") + if dest.exists(): + print(f"Files in {dest}: {list(dest.iterdir())}") + else: + print(f"{dest} does not exist") return False # Verify content matches From 0ef0e890df717c31b133d334eda1703e4d141057 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sun, 17 May 2026 18:52:37 -0700 Subject: [PATCH 470/506] comment cleanup --- lib/spack/spack/test/cmd/migrate.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/lib/spack/spack/test/cmd/migrate.py b/lib/spack/spack/test/cmd/migrate.py index 2b2073729d788c..1fae8d36601c62 100644 --- a/lib/spack/spack/test/cmd/migrate.py +++ b/lib/spack/spack/test/cmd/migrate.py @@ -146,13 +146,11 @@ def verify_package_repos_copied(source_repos_base, dest_repos_base, created_file def test_migrate_basic(tmp_path, set_home, monkeypatch, mutable_config): """Test basic migrate: copies files to new locations, preserves old location.""" - # Set up directories spack_root = tmp_path / "spack-root" spack_root.mkdir() home = tmp_path / "home" home.mkdir() - # Set home BEFORE creating paths objects set_home(str(home)) # Create ~/.spack with test files @@ -160,7 +158,6 @@ def test_migrate_basic(tmp_path, set_home, monkeypatch, mutable_config): dotspack.mkdir() created = create_dotspack_files(dotspack) - # Set expected new locations new_config = home / ".config" / "spack" new_state = home / ".local" / "state" / "spack" @@ -174,13 +171,11 @@ def test_migrate_basic(tmp_path, set_home, monkeypatch, mutable_config): monkeypatch.setattr(spack.cmd.migrate, "paths", paths) monkeypatch.setattr(spack.cmd.migrate, "paths_base", base_paths) - # Run migrate migrate() # Verify config files were copied assert verify_files_copied(dotspack, new_config, created) - # Verify package repos were copied old_repos = dotspack / "package_repos" new_repos = new_state / "package_repos" assert verify_package_repos_copied(old_repos, new_repos, created) @@ -193,13 +188,11 @@ def test_migrate_basic(tmp_path, set_home, monkeypatch, mutable_config): def test_migrate_with_clear(tmp_path, set_home, monkeypatch, mutable_config): """Test migrate --clear: copies files and removes ~/.spack.""" - # Set up directories spack_root = tmp_path / "spack-root" spack_root.mkdir() home = tmp_path / "home" home.mkdir() - # Set home BEFORE creating paths objects set_home(str(home)) # Create ~/.spack with test files @@ -207,7 +200,6 @@ def test_migrate_with_clear(tmp_path, set_home, monkeypatch, mutable_config): dotspack.mkdir() created = create_dotspack_files(dotspack) - # Set expected new locations new_config = home / ".config" / "spack" new_state = home / ".local" / "state" / "spack" @@ -224,13 +216,11 @@ def test_migrate_with_clear(tmp_path, set_home, monkeypatch, mutable_config): # The backup location is now paths.dotspack_backup backup_location = pathlib.Path(paths.dotspack_backup) - # Run migrate with --clear migrate("--clear") # Verify config files were copied assert verify_files_copied(backup_location, new_config, created) - # Verify package repos were copied old_repos = backup_location / "package_repos" new_repos = new_state / "package_repos" assert verify_package_repos_copied(old_repos, new_repos, created) @@ -244,13 +234,11 @@ def test_migrate_with_clear(tmp_path, set_home, monkeypatch, mutable_config): def test_migrate_then_clear_replace(tmp_path, set_home, monkeypatch, mutable_config): """Test migrate, then migrate --clear --replace to clean up.""" - # Set up directories spack_root = tmp_path / "spack-root" spack_root.mkdir() home = tmp_path / "home" home.mkdir() - # Set home BEFORE creating paths objects set_home(str(home)) # Create ~/.spack with test files @@ -272,10 +260,8 @@ def test_migrate_then_clear_replace(tmp_path, set_home, monkeypatch, mutable_con monkeypatch.setattr(spack.cmd.migrate, "paths", paths) monkeypatch.setattr(spack.cmd.migrate, "paths_base", base_paths) - # Run migrate (without --clear) migrate() - # Verify files were copied assert verify_files_copied(dotspack, new_config, created) old_repos = dotspack / "package_repos" new_repos = new_state / "package_repos" @@ -300,13 +286,11 @@ def test_migrate_then_clear_replace(tmp_path, set_home, monkeypatch, mutable_con def test_migrate_then_clear_only(tmp_path, set_home, monkeypatch, mutable_config): """Test migrate, then migrate --clear-only to just remove ~/.spack.""" - # Set up directories spack_root = tmp_path / "spack-root" spack_root.mkdir() home = tmp_path / "home" home.mkdir() - # Set home BEFORE creating paths objects set_home(str(home)) # Create ~/.spack with test files @@ -328,10 +312,8 @@ def test_migrate_then_clear_only(tmp_path, set_home, monkeypatch, mutable_config monkeypatch.setattr(spack.cmd.migrate, "paths", paths) monkeypatch.setattr(spack.cmd.migrate, "paths_base", base_paths) - # Run migrate (without --clear) migrate() - # Verify files were copied assert verify_files_copied(dotspack, new_config, created) old_repos = dotspack / "package_repos" new_repos = new_state / "package_repos" From e8b206e1e2685f79f70932317f806727e47be8af Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sun, 17 May 2026 19:01:01 -0700 Subject: [PATCH 471/506] extract repeat code into fixture --- lib/spack/spack/test/cmd/migrate.py | 145 +++++++++------------------- 1 file changed, 43 insertions(+), 102 deletions(-) diff --git a/lib/spack/spack/test/cmd/migrate.py b/lib/spack/spack/test/cmd/migrate.py index 1fae8d36601c62..e03c0eb4e4a1e3 100644 --- a/lib/spack/spack/test/cmd/migrate.py +++ b/lib/spack/spack/test/cmd/migrate.py @@ -22,6 +22,41 @@ def clear_env_vars(working_env): spack.paths._unset_path_vars(os.environ) +@pytest.fixture +def migrate_setup(tmp_path, set_home, monkeypatch, mutable_config): + """Set up common test environment for migrate tests. + + Yields: + tuple: (dotspack, created, new_config, new_state, paths) + """ + spack_root = tmp_path / "spack-root" + spack_root.mkdir() + home = tmp_path / "home" + home.mkdir() + + set_home(str(home)) + + # Create ~/.spack with test files + dotspack = home / ".spack" + dotspack.mkdir() + created = create_dotspack_files(dotspack) + + new_config = home / ".config" / "spack" + new_state = home / ".local" / "state" / "spack" + + # Create fresh SpackPaths object and patch the module + base_paths = SpackPathsBase(str(spack_root)) + paths = SpackPaths(base_paths) + monkeypatch.setattr(spack.paths, "locations", paths) + monkeypatch.setattr(spack.paths_base, "locations", base_paths) + + # Also need to patch the cmd.migrate module which has already imported these + monkeypatch.setattr(spack.cmd.migrate, "paths", paths) + monkeypatch.setattr(spack.cmd.migrate, "paths_base", base_paths) + + yield (dotspack, created, new_config, new_state, paths) + + def create_dotspack_files(base_path): """Create minimal test files under a base path to simulate ~/.spack structure. @@ -144,32 +179,9 @@ def verify_package_repos_copied(source_repos_base, dest_repos_base, created_file return True -def test_migrate_basic(tmp_path, set_home, monkeypatch, mutable_config): +def test_migrate_basic(migrate_setup): """Test basic migrate: copies files to new locations, preserves old location.""" - spack_root = tmp_path / "spack-root" - spack_root.mkdir() - home = tmp_path / "home" - home.mkdir() - - set_home(str(home)) - - # Create ~/.spack with test files - dotspack = home / ".spack" - dotspack.mkdir() - created = create_dotspack_files(dotspack) - - new_config = home / ".config" / "spack" - new_state = home / ".local" / "state" / "spack" - - # Create fresh SpackPaths object and patch the module - base_paths = SpackPathsBase(str(spack_root)) - paths = SpackPaths(base_paths) - monkeypatch.setattr(spack.paths, "locations", paths) - monkeypatch.setattr(spack.paths_base, "locations", base_paths) - - # Also need to patch the cmd.migrate module which has already imported these - monkeypatch.setattr(spack.cmd.migrate, "paths", paths) - monkeypatch.setattr(spack.cmd.migrate, "paths_base", base_paths) + dotspack, created, new_config, new_state, paths = migrate_setup migrate() @@ -186,32 +198,9 @@ def test_migrate_basic(tmp_path, set_home, monkeypatch, mutable_config): assert (dotspack / config_file).exists() -def test_migrate_with_clear(tmp_path, set_home, monkeypatch, mutable_config): +def test_migrate_with_clear(migrate_setup): """Test migrate --clear: copies files and removes ~/.spack.""" - spack_root = tmp_path / "spack-root" - spack_root.mkdir() - home = tmp_path / "home" - home.mkdir() - - set_home(str(home)) - - # Create ~/.spack with test files - dotspack = home / ".spack" - dotspack.mkdir() - created = create_dotspack_files(dotspack) - - new_config = home / ".config" / "spack" - new_state = home / ".local" / "state" / "spack" - - # Create fresh SpackPaths object and patch the module - base_paths = SpackPathsBase(str(spack_root)) - paths = SpackPaths(base_paths) - monkeypatch.setattr(spack.paths, "locations", paths) - monkeypatch.setattr(spack.paths_base, "locations", base_paths) - - # Also need to patch the cmd.migrate module which has already imported these - monkeypatch.setattr(spack.cmd.migrate, "paths", paths) - monkeypatch.setattr(spack.cmd.migrate, "paths_base", base_paths) + dotspack, created, new_config, new_state, paths = migrate_setup # The backup location is now paths.dotspack_backup backup_location = pathlib.Path(paths.dotspack_backup) @@ -232,33 +221,9 @@ def test_migrate_with_clear(tmp_path, set_home, monkeypatch, mutable_config): assert backup_location.exists() -def test_migrate_then_clear_replace(tmp_path, set_home, monkeypatch, mutable_config): +def test_migrate_then_clear_replace(migrate_setup): """Test migrate, then migrate --clear --replace to clean up.""" - spack_root = tmp_path / "spack-root" - spack_root.mkdir() - home = tmp_path / "home" - home.mkdir() - - set_home(str(home)) - - # Create ~/.spack with test files - dotspack = home / ".spack" - dotspack.mkdir() - created = create_dotspack_files(dotspack) - - # Set expected new locations - new_config = home / ".config" / "spack" - new_state = home / ".local" / "state" / "spack" - - # Create fresh SpackPaths object and patch the module - base_paths = SpackPathsBase(str(spack_root)) - paths = SpackPaths(base_paths) - monkeypatch.setattr(spack.paths, "locations", paths) - monkeypatch.setattr(spack.paths_base, "locations", base_paths) - - # Also need to patch the cmd.migrate module which has already imported these - monkeypatch.setattr(spack.cmd.migrate, "paths", paths) - monkeypatch.setattr(spack.cmd.migrate, "paths_base", base_paths) + dotspack, created, new_config, new_state, paths = migrate_setup migrate() @@ -284,33 +249,9 @@ def test_migrate_then_clear_replace(tmp_path, set_home, monkeypatch, mutable_con assert (backup_location / config_file).exists() -def test_migrate_then_clear_only(tmp_path, set_home, monkeypatch, mutable_config): +def test_migrate_then_clear_only(migrate_setup): """Test migrate, then migrate --clear-only to just remove ~/.spack.""" - spack_root = tmp_path / "spack-root" - spack_root.mkdir() - home = tmp_path / "home" - home.mkdir() - - set_home(str(home)) - - # Create ~/.spack with test files - dotspack = home / ".spack" - dotspack.mkdir() - created = create_dotspack_files(dotspack) - - # Set expected new locations - new_config = home / ".config" / "spack" - new_state = home / ".local" / "state" / "spack" - - # Create fresh SpackPaths object and patch the module - base_paths = SpackPathsBase(str(spack_root)) - paths = SpackPaths(base_paths) - monkeypatch.setattr(spack.paths, "locations", paths) - monkeypatch.setattr(spack.paths_base, "locations", base_paths) - - # Also need to patch the cmd.migrate module which has already imported these - monkeypatch.setattr(spack.cmd.migrate, "paths", paths) - monkeypatch.setattr(spack.cmd.migrate, "paths_base", base_paths) + dotspack, created, new_config, new_state, paths = migrate_setup migrate() From 7c4c99d24b6d718d3e7b80526359e3fb295188a5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sun, 17 May 2026 19:42:14 -0700 Subject: [PATCH 472/506] more-meaningful docstrings for 'spack migrate' tests --- lib/spack/spack/test/cmd/migrate.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/test/cmd/migrate.py b/lib/spack/spack/test/cmd/migrate.py index e03c0eb4e4a1e3..0367090c9051e5 100644 --- a/lib/spack/spack/test/cmd/migrate.py +++ b/lib/spack/spack/test/cmd/migrate.py @@ -180,7 +180,12 @@ def verify_package_repos_copied(source_repos_base, dest_repos_base, created_file def test_migrate_basic(migrate_setup): - """Test basic migrate: copies files to new locations, preserves old location.""" + """`spack migrate` with no additional arguments moves config files + out of ~/.spack into ~/.config/spack, moves package repositories + from ~/.spack/package_repos into ~/.local/state/spack/package_repos, + and does not delete any files in ~/.spack (e.g. if some spack + instances are not getting updated to 1.2). + """ dotspack, created, new_config, new_state, paths = migrate_setup migrate() @@ -199,7 +204,9 @@ def test_migrate_basic(migrate_setup): def test_migrate_with_clear(migrate_setup): - """Test migrate --clear: copies files and removes ~/.spack.""" + """Test --clear: does everything `spack migrate` does, plus + removes ~/.spack (moves it into a backup location in + $data_home/dotspack_backup).""" dotspack, created, new_config, new_state, paths = migrate_setup # The backup location is now paths.dotspack_backup @@ -222,7 +229,13 @@ def test_migrate_with_clear(migrate_setup): def test_migrate_then_clear_replace(migrate_setup): - """Test migrate, then migrate --clear --replace to clean up.""" + """A user who runs `spack migrate` but forgets --clear should + run `spack migrate --clear --replace`, which deletes everything + in ~/.config/spack and ~/.local/state/spack/package_repos + (--replace is telling Spack it's OK to do this, because + otherwise configs in the old/new locations may have diverged + and Spack has no notion of how to merge them). + """ dotspack, created, new_config, new_state, paths = migrate_setup migrate() @@ -250,7 +263,11 @@ def test_migrate_then_clear_replace(migrate_setup): def test_migrate_then_clear_only(migrate_setup): - """Test migrate, then migrate --clear-only to just remove ~/.spack.""" + """If the user runs `spack migrate` but forgets --clear, then + using `spack migrate --clear-only` is telling spack it's ok + to get rid of `~/.spack` (it won't try coping files out of + ~/.spack, which would trigger a collision if `spack migrate` + copied anything into ~/.config/spack, for example).""" dotspack, created, new_config, new_state, paths = migrate_setup migrate() From 81bd1d5611c6a1c55edeea9ebc95841041ba7416 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sun, 17 May 2026 20:05:13 -0700 Subject: [PATCH 473/506] add a file describing this feature --- shared-spack.txt | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 shared-spack.txt diff --git a/shared-spack.txt b/shared-spack.txt new file mode 100644 index 00000000000000..e76b21f3b2e2e2 --- /dev/null +++ b/shared-spack.txt @@ -0,0 +1,57 @@ +# Shared spack + +## Moving Spack-generated artifacts outside of $spack + +The biggest blocker to creating a shared spack instance is that Spack +will default to writing data into its own prefix. For new clones of +Spack, this PR moves all generated files out of the cloned directory +and into ~. This includes: + +1. installs +2. where environments are stored +3. cached downloads +4. gpg keys +5. where modules are generated + +## XDG compliance + +Spack by default stores user_cache_path and the user config scope in +~/.spack. + +- We want new spack instances to start using XDG-compliant locations + for these +- For the data moving out of $spack described in the previous section, + we want to move it into XDG-compliant locations + +## Old/new spack instance behavior: smoothing transition + +- If no new spack clones are generated and only old spack clones + exist, then each of them will continue using their old install + location inside of their respective $spack directories + (including after they pull this PR). +- If a new clone starts writing into ~ and an old spack instance + git pulls this logic, it will still keep using its own $spack + directory for installs, environments, etc. +- If an old spack instance pulls this PR and then sets any new- + style location variable (an env var like SPACK_HOME, or a config + variable like config:locations:data), spack assumes that it + should use new-style defaults. +- Both old and new spack instances will use ~/.spack for config + and as a user_cache_path if they detect it and if the new + locations (~/.config/spack and ~/.local/state/spack) are not + detected +- A warning will be generated when ~/.spack is detected, telling + the user to run `spack migrate` (a new command provided with + this PR). + - However, this PR also contains logic to detect when the user + is explicitly using ~/.spack (e.g. if a config scope + explicitly targets ~/.spack as a backup, or if a user + sets SPACK_USER_CACHE_PATH to ~/.spack) and does not warn + in that case either. + +## Data categories with lots of data + +1-3 are large. Before this PR there were config options to relocate +each of these individually (and those still take precedence). This +PR offers additional mechanisms to set a single env var or config +option in order to relocate all of them together. \ No newline at end of file From 8a4614afce3da42b5466c257fb2d62da1a7a1adc Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sun, 17 May 2026 20:28:55 -0700 Subject: [PATCH 474/506] update feature description w/reference to up-to-date doc (but docs in general still need updates) --- shared-spack.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/shared-spack.txt b/shared-spack.txt index e76b21f3b2e2e2..eacedb97bbc001 100644 --- a/shared-spack.txt +++ b/shared-spack.txt @@ -54,4 +54,11 @@ Spack by default stores user_cache_path and the user config scope in 1-3 are large. Before this PR there were config options to relocate each of these individually (and those still take precedence). This PR offers additional mechanisms to set a single env var or config -option in order to relocate all of them together. \ No newline at end of file +option in order to relocate all of them together. + +## New mechanisms for controlling data location + +This is described in detail in where_spack_writes_data.rst (that +section is focused on how Spack behaves going forward - questions +about how existing spack instances can keep using their old data +is described here). From e75d2b47ee9a972281b7e6bdddd5b110c1cdc2cd Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sun, 17 May 2026 20:36:34 -0700 Subject: [PATCH 475/506] other style check --- lib/spack/spack/test/cmd/migrate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/test/cmd/migrate.py b/lib/spack/spack/test/cmd/migrate.py index 0367090c9051e5..a4bed1941f35cd 100644 --- a/lib/spack/spack/test/cmd/migrate.py +++ b/lib/spack/spack/test/cmd/migrate.py @@ -7,6 +7,7 @@ import pytest +import spack import spack.cmd.migrate import spack.main import spack.paths From f7663673f61a2daf82b546ee2b8092aa7c4c3f84 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 18 May 2026 10:05:26 -0700 Subject: [PATCH 476/506] fix docs issue --- lib/spack/spack/cmd/migrate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py index c1fea9b027fd2a..f4904d6cb51fb7 100644 --- a/lib/spack/spack/cmd/migrate.py +++ b/lib/spack/spack/cmd/migrate.py @@ -85,7 +85,7 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: """Migrate user config and package repositories from ~/.spack to new locations. This command migrates: - - User config files: ~/.spack/*.yaml -> ~/.config/spack/ + - User config files: ~/.spack/...yaml -> ~/.config/spack/ - Package repositories: ~/.spack/package_repos -> ~/.local/state/spack/package_repos """ # Handle restore mode From daf0f119274e878b1d18af87dd85a4277c90e162 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 19 May 2026 09:55:59 -0700 Subject: [PATCH 477/506] fix some issues with check for new-style location defaults; add clarifications to shared-spack.txt --- lib/spack/spack/paths.py | 10 +++++++--- shared-spack.txt | 11 ++++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index f5a5971ad6d136..d2b762f05334f1 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -381,9 +381,13 @@ def _decide_old_or_new_location( def new_layout_enforced(): - first = any(x.value in os.environ for x in Spack_vars.new_layout()) - second = bool(config.get("config:locations")) - return first or second + cfg = config.get("config:locations", {}) + for x in ["home", "data", "state", "cache"]: + if cfg.get(x, None): + return True + + use_env = not bool(cfg.get("disable_env", False)) + return use_env and any(x.value in os.environ for x in Spack_vars.new_layout()) def detect_old_spack_layout(paths: paths_base.SpackPathsBase): diff --git a/shared-spack.txt b/shared-spack.txt index eacedb97bbc001..bc5475d9de2b57 100644 --- a/shared-spack.txt +++ b/shared-spack.txt @@ -30,12 +30,17 @@ Spack by default stores user_cache_path and the user config scope in location inside of their respective $spack directories (including after they pull this PR). - If a new clone starts writing into ~ and an old spack instance - git pulls this logic, it will still keep using its own $spack - directory for installs, environments, etc. + git pulls this logic, the old instance will still keep using its + own $spack directory for installs, environments, etc. - If an old spack instance pulls this PR and then sets any new- style location variable (an env var like SPACK_HOME, or a config variable like config:locations:data), spack assumes that it - should use new-style defaults. + should use new-style defaults for *all* types of data. + - e.g. setting STATE_HOME would make an older spack instance + use installs in ~/.local/share/spack/installs rather than + in $spack/opt/spack + - A user can avoid env vars messing with their older spack + install by adding config:locations:disable_env:true - Both old and new spack instances will use ~/.spack for config and as a user_cache_path if they detect it and if the new locations (~/.config/spack and ~/.local/state/spack) are not From fc567c85df8a3fa145f22916a6ae620bb7d1f17e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 19 May 2026 17:04:39 -0700 Subject: [PATCH 478/506] add section on all-or-nothing behavior --- shared-spack.txt | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/shared-spack.txt b/shared-spack.txt index bc5475d9de2b57..ab35a876a679fc 100644 --- a/shared-spack.txt +++ b/shared-spack.txt @@ -1,6 +1,6 @@ -# Shared spack +Shared spack -## Moving Spack-generated artifacts outside of $spack +# Moving Spack-generated artifacts outside of $spack The biggest blocker to creating a shared spack instance is that Spack will default to writing data into its own prefix. For new clones of @@ -12,8 +12,9 @@ and into ~. This includes: 3. cached downloads 4. gpg keys 5. where modules are generated +6. licenses -## XDG compliance +# XDG compliance Spack by default stores user_cache_path and the user config scope in ~/.spack. @@ -23,7 +24,7 @@ Spack by default stores user_cache_path and the user config scope in - For the data moving out of $spack described in the previous section, we want to move it into XDG-compliant locations -## Old/new spack instance behavior: smoothing transition +# Old/new spack instance behavior: smoothing transition - If no new spack clones are generated and only old spack clones exist, then each of them will continue using their old install @@ -54,14 +55,29 @@ Spack by default stores user_cache_path and the user config scope in sets SPACK_USER_CACHE_PATH to ~/.spack) and does not warn in that case either. -## Data categories with lots of data +## General all-or-nothing behavior + +For items that used to live in $spack, they are generally all relocated +together or not at all: + +- e.g. if the old default install tree in $spack/opt/spack contains + installs, then spack also expects to manage environments, gpg keys, + license files, and downloads in their old locations inside of + $spack +- modules is an exception, they are automatically moved into $data_home + (new instances will generate modules in $data_home, and old instances + that pull these changes will start doing that as well) +- The user config scope and the user_cache_path are also excluded from + this (they didn't live in $spack by default though) + +# Data categories with lots of data 1-3 are large. Before this PR there were config options to relocate each of these individually (and those still take precedence). This PR offers additional mechanisms to set a single env var or config option in order to relocate all of them together. -## New mechanisms for controlling data location +# New mechanisms for controlling data location This is described in detail in where_spack_writes_data.rst (that section is focused on how Spack behaves going forward - questions From b075d81b4e2148f29aef1500963dac135d22a64c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 19 May 2026 17:17:02 -0700 Subject: [PATCH 479/506] more updates to doc --- shared-spack.txt | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/shared-spack.txt b/shared-spack.txt index ab35a876a679fc..33750b216ce34d 100644 --- a/shared-spack.txt +++ b/shared-spack.txt @@ -30,16 +30,19 @@ Spack by default stores user_cache_path and the user config scope in exist, then each of them will continue using their old install location inside of their respective $spack directories (including after they pull this PR). -- If a new clone starts writing into ~ and an old spack instance - git pulls this logic, the old instance will still keep using its - own $spack directory for installs, environments, etc. +- If a new clone starts placing installed packages in ~, and an + old spack instance git pulls this logic, the old instance will + still keep using its own $spack directory for installs, + environments, etc. - If an old spack instance pulls this PR and then sets any new- style location variable (an env var like SPACK_HOME, or a config variable like config:locations:data), spack assumes that it should use new-style defaults for *all* types of data. - - e.g. setting STATE_HOME would make an older spack instance - use installs in ~/.local/share/spack/installs rather than - in $spack/opt/spack + - e.g. if an older spack instance git pulls this logic and + sees that SPACK_STATE_HOME is set, then it will use installs + in ~/.local/share/spack/installs rather than in $spack/opt/spack + (even though STATE_HOME does not control where installs are + placed) - A user can avoid env vars messing with their older spack install by adding config:locations:disable_env:true - Both old and new spack instances will use ~/.spack for config @@ -70,13 +73,6 @@ together or not at all: - The user config scope and the user_cache_path are also excluded from this (they didn't live in $spack by default though) -# Data categories with lots of data - -1-3 are large. Before this PR there were config options to relocate -each of these individually (and those still take precedence). This -PR offers additional mechanisms to set a single env var or config -option in order to relocate all of them together. - # New mechanisms for controlling data location This is described in detail in where_spack_writes_data.rst (that From d69de4006cb55a0d699bf5694f74c7731f74ec53 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 19 May 2026 17:52:10 -0700 Subject: [PATCH 480/506] move some PR items that describe current behavior into shared-spack.txt --- shared-spack.txt | 58 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/shared-spack.txt b/shared-spack.txt index 33750b216ce34d..14a2dcefb34935 100644 --- a/shared-spack.txt +++ b/shared-spack.txt @@ -1,5 +1,7 @@ Shared spack +This document is accurate up to commit b075d81b4e + # Moving Spack-generated artifacts outside of $spack The biggest blocker to creating a shared spack instance is that Spack @@ -79,3 +81,59 @@ This is described in detail in where_spack_writes_data.rst (that section is focused on how Spack behaves going forward - questions about how existing spack instances can keep using their old data is described here). + +# avoiding ~ + +Before this PR, you could do + +``` +export SPACK_USER_CACHE_PATH=x +export SPACK_DISABLE_LOCAL_CONFIG=1 +``` + +and nothing would be read from or written to ~ + +With this PR, that would not be enough. Some ways to do that here: + +- If you set `SPACK_HOME=x SPACK_USER_CONFIG_PATH=x/config`, everything + will be written to x +- Or you can do `SPACK_HOME=x SPACK_DISABLE_LOCAL_CONFIG=1` +- If you set SPACK_DATA_HOME or `config:locations:data` outside of ~, + this will relocate everything that takes up significant space (envs, + installs, cached downloads) + +To completely move everything, in an env you can do: + +``` +include:: [] +config: + locations: + home: x +``` + +Users who have write access to $spack can do + +``` +spack config --scope=spack edit include +spack config --scope=spack add config:locations:home:x +``` + +# Crosstalk between spack instances + +* The misc. cache now includes a per-spack component (based on a hash + of the Spack root): `spack_instance_id` + * `config:misc_cache` still exists, so users can share a single + misc cache among multiple instances of spack +* Ideally each individual spack would have a distinct highest-level + write scope, but for now they all share `~/.config/spack` + +# Admin management, pip-installed spack + +- If admins want to provide an upstream to users but avoid using it + themselves, they can use existing `include:` config in the `spack scope`: + - Include a scope that adds `upstreams:x` config + `when: SPACK_ADMIN not in env` + - Include a scope that adds `install_tree:root:x` config + `when: SPACK_ADMIN in env` + - As long as all admins make sure to define this env var, this will + satisfy [5a/b] From 45ec603f13b92b9d1c0e080e2ad2baf1ad231e7e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 20 May 2026 17:19:04 -0700 Subject: [PATCH 481/506] add spack migrate command description --- shared-spack.txt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/shared-spack.txt b/shared-spack.txt index 14a2dcefb34935..432eae0ebb7bdd 100644 --- a/shared-spack.txt +++ b/shared-spack.txt @@ -75,6 +75,25 @@ together or not at all: - The user config scope and the user_cache_path are also excluded from this (they didn't live in $spack by default though) +## `spack migrate` command + +Simplest usage is if you are updating all spack instances to >= 1.2, +in which case you can do: + +`spack migrate --clear` + +- Copies config files from ~/.spack into ~/.config +- Copies package repositories from ~/.spack/package_repos into + ~/.local/state/spack/package_repos +- Moves the entire ~/.spack directory into a backup location into + `$data_home/dotspack_backup` + +If it's descovered that this operation removed something from +~/.spack that an older spack instance needs (i.e. you realized you can't +update all your spack instances), then you can restore it with + +`spack migrate --restore-old-configs` + # New mechanisms for controlling data location This is described in detail in where_spack_writes_data.rst (that From e48b8c1045bdaf2555f6142a9c1a08cbfb4582f8 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 20 May 2026 19:08:11 -0700 Subject: [PATCH 482/506] default_vars for all data_home accessors --- etc/spack/defaults/base/config.yaml | 2 +- etc/spack/defaults/base/modules.yaml | 4 ++-- lib/spack/spack/paths.py | 14 ++++++++++++++ lib/spack/spack/util/path.py | 3 ++- shared-spack.txt | 2 +- 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index 124a3c350bb460..b0e5d8f626d97d 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -80,7 +80,7 @@ config: # Cache directory for already downloaded source tarballs and archived # repositories. This can be purged with `spack clean --downloads`. - source_cache: $data_home/downloads + source_cache: $default_downloads_dir ## Directory where spack managed environments are created and stored diff --git a/etc/spack/defaults/base/modules.yaml b/etc/spack/defaults/base/modules.yaml index c43dcee865abde..c2a23626b5120c 100644 --- a/etc/spack/defaults/base/modules.yaml +++ b/etc/spack/defaults/base/modules.yaml @@ -33,8 +33,8 @@ modules: default: # Where to install modules roots: - tcl: $data_home/modules - lmod: $data_home/lmod + tcl: $default_modules_base/modules + lmod: $default_modules_base/lmod # What type of modules to use ("tcl" and/or "lmod") enable: [] diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index d2b762f05334f1..0aca1cb57e2201 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -215,6 +215,20 @@ def default_license_dir(self): else: return os.path.join(self.data_home, "licenses") + @property + def default_modules_base(self): + if self.old_layout_detected: + return self.base.share_path + else: + return os.path.join(self.data_home) + + @property + def default_downloads_dir(self): + if self.old_layout_detected: + return self.base.old_fetch_cache_path + else: + return os.path.join(self.data_home, "downloads") + @property def reports_path(self): #: junit, cdash, etc. reports about builds diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 9f24e51cdc612d..0e119240b93b91 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -83,7 +83,8 @@ def replacements(): "default_install_root": lambda: paths.default_install_location, "default_envs_root": lambda: paths.default_envs_path, "default_license_dir": lambda: paths.default_license_dir, - # TODO: for now need analogous values for gpg keys + "default_downloads_dir": lambda: paths.default_downloads_dir, + "default_modules_base": lambda: paths.default_modules_base, "data_home": lambda: paths.data_home, "cache_home": lambda: paths.cache_home, "state_home": lambda: paths.state_home, diff --git a/shared-spack.txt b/shared-spack.txt index 432eae0ebb7bdd..ebb3670e65619b 100644 --- a/shared-spack.txt +++ b/shared-spack.txt @@ -130,7 +130,7 @@ config: home: x ``` -Users who have write access to $spack can do +Alternatively, users who have write access to $spack can do ``` spack config --scope=spack edit include From 3c8afef5502433774b6b5d94ef10659d2caeaf40 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 20 May 2026 20:33:50 -0700 Subject: [PATCH 483/506] update shared-spack.txt about new guarantees about writing into ~ --- shared-spack.txt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/shared-spack.txt b/shared-spack.txt index ebb3670e65619b..c429d9d2afceff 100644 --- a/shared-spack.txt +++ b/shared-spack.txt @@ -1,6 +1,6 @@ Shared spack -This document is accurate up to commit b075d81b4e +This document is accurate up to commit e48b8c104 # Moving Spack-generated artifacts outside of $spack @@ -69,9 +69,6 @@ together or not at all: installs, then spack also expects to manage environments, gpg keys, license files, and downloads in their old locations inside of $spack -- modules is an exception, they are automatically moved into $data_home - (new instances will generate modules in $data_home, and old instances - that pull these changes will start doing that as well) - The user config scope and the user_cache_path are also excluded from this (they didn't live in $spack by default though) @@ -110,9 +107,13 @@ export SPACK_USER_CACHE_PATH=x export SPACK_DISABLE_LOCAL_CONFIG=1 ``` -and nothing would be read from or written to ~ +and nothing would be read from or written to ~. -With this PR, that would not be enough. Some ways to do that here: +For an older instance of spack that pulls this PR, this will also be +sufficient to prevent writes into ~ (this depends on e48b8c104). + +For a fresh clone of spack using this PR, additional steps are +required to avoid writing into ~: - If you set `SPACK_HOME=x SPACK_USER_CONFIG_PATH=x/config`, everything will be written to x From 4a9a7e5d6caf27c1e6a2b0724f897956992d88d3 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 May 2026 09:58:44 -0700 Subject: [PATCH 484/506] propagate change to code that was moved in merged updates --- lib/spack/spack/solver/compat.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/spack/spack/solver/compat.py b/lib/spack/spack/solver/compat.py index 07c7747ac6a8a8..fb5820d2ee1259 100644 --- a/lib/spack/spack/solver/compat.py +++ b/lib/spack/spack/solver/compat.py @@ -57,7 +57,6 @@ def _ensure_clingo_or_raise(clingo_mod: ModuleType) -> None: # These are imports that may be problematic at top level (circular imports). They are used # only to provide exhaustive details when erroring due to a broken clingo module. import spack.config - import spack.paths as sp import spack.util.path as sup try: @@ -83,7 +82,7 @@ def _ensure_clingo_or_raise(clingo_mod: ModuleType) -> None: if ( pathlib.Path( sup.canonicalize_path( - spack.config.CONFIG.get("bootstrap:root", sp.default_user_bootstrap_path) + spack.config.CONFIG.get("bootstrap:root", "$state_home/bootstrap") ) ) in pathlib.Path(clingo_mod.__file__).parents From 098090c884edffabcae84ab8e76f3218a01cf171 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 May 2026 10:02:37 -0700 Subject: [PATCH 485/506] rename option --- lib/spack/spack/cmd/migrate.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py index f4904d6cb51fb7..076b92e9ca0eb3 100644 --- a/lib/spack/spack/cmd/migrate.py +++ b/lib/spack/spack/cmd/migrate.py @@ -40,13 +40,13 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: "if you forgot to use --clear on first run)", ) subparser.add_argument( - "--restore-old-configs", + "--restore", action="store_true", help="restore ~/.spack from backup location (after --clear)", ) -def restore_old_configs(args: argparse.Namespace) -> None: +def restore(args: argparse.Namespace) -> None: """Restore ~/.spack from backup location.""" old_location = os.path.expanduser("~/.spack") @@ -89,8 +89,8 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: - Package repositories: ~/.spack/package_repos -> ~/.local/state/spack/package_repos """ # Handle restore mode - if args.restore_old_configs: - restore_old_configs(args) + if args.restore: + restore(args) return old_location = os.path.expanduser("~/.spack") From a935837d0ad97cd5069caba1a45829ac46201186 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 May 2026 10:04:43 -0700 Subject: [PATCH 486/506] doc cleanup --- lib/spack/spack/cmd/migrate.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py index 076b92e9ca0eb3..d4522772a54050 100644 --- a/lib/spack/spack/cmd/migrate.py +++ b/lib/spack/spack/cmd/migrate.py @@ -88,7 +88,6 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: - User config files: ~/.spack/...yaml -> ~/.config/spack/ - Package repositories: ~/.spack/package_repos -> ~/.local/state/spack/package_repos """ - # Handle restore mode if args.restore: restore(args) return @@ -98,7 +97,6 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: new_state_location = os.path.join(os.path.expanduser("~"), ".local", "state", "spack") backup_location = paths.dotspack_backup - # Handle --clear-only mode if args.clear_only: if not os.path.exists(old_location): tty.die(f"Old configuration location does not exist: {old_location}") @@ -124,7 +122,6 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: tty.msg(f"Removing existing package repos: {new_package_repos}") shutil.rmtree(new_package_repos) - # Check if old location exists if not os.path.exists(old_location): tty.die(f"Old configuration location does not exist: {old_location}") @@ -164,10 +161,8 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: new_package_repos = os.path.join(new_state_location, "package_repos") if os.path.exists(old_package_repos) and os.path.isdir(old_package_repos): - # Check if there's anything in it repos = os.listdir(old_package_repos) if repos: - # Check if new location already has package repositories if os.path.exists(new_package_repos): existing_repos = os.listdir(new_package_repos) if existing_repos: From 2f62b20fd20dbed265cd20710c8417caeeb4bd81 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 May 2026 10:34:26 -0700 Subject: [PATCH 487/506] add option to print help to spack migrate --- lib/spack/spack/cmd/migrate.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py index d4522772a54050..bf37088b0c242a 100644 --- a/lib/spack/spack/cmd/migrate.py +++ b/lib/spack/spack/cmd/migrate.py @@ -44,6 +44,11 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: action="store_true", help="restore ~/.spack from backup location (after --clear)", ) + subparser.add_argument( + "--i-need-old-spack", + action="store_true", + help="print help about mixing pre-1.2-Spack and Spack >= 1.2", + ) def restore(args: argparse.Namespace) -> None: @@ -81,6 +86,30 @@ def restore(args: argparse.Namespace) -> None: tty.msg("Restore complete!") +def i_need_old_spack(): + print("""\ +If you're getting a warning about using resources in ~/.spack, and +you have pre-1.2 Spack instances that cannot upgrade. Then it is +recommended that both old and new instances use ~/.spack + +In the new instance, you can do that by adding config: + +config: + locations: + state: ~/.spack + +or setting the SPACK_STATE_HOME/SPACK_USER_CACHE environment variable. + +Alternatively, you can set: + +include: + - name: "user" + path: "~/.spack/" + +(in etc/spack/include.yaml) +""") + + def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: """Migrate user config and package repositories from ~/.spack to new locations. @@ -88,6 +117,10 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: - User config files: ~/.spack/...yaml -> ~/.config/spack/ - Package repositories: ~/.spack/package_repos -> ~/.local/state/spack/package_repos """ + if args.i_need_old_spack: + i_need_old_spack() + return + if args.restore: restore(args) return From d73ebf83cf309933864ff431d16147db922606dc Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 May 2026 10:42:01 -0700 Subject: [PATCH 488/506] add pointer --- lib/spack/spack/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 1a852e62729368..3d170a4fd3bde2 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -976,6 +976,8 @@ def uses_old_dotspack(path): msg += ( "\nIf all spack instances are >= 1.2, you can use" " `spack migrate --clear` to silence this warning" + "\nIf not, run `spack migrate --i-need-old-spack` to" + " see what manual steps you can take to silence this warning" ) tty.warn(msg) From e7b434721dc6909050b561d45ff802e51e0af304 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 May 2026 10:42:19 -0700 Subject: [PATCH 489/506] update command completion --- share/spack/spack-completion.bash | 2 +- share/spack/spack-completion.fish | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index e3ecd9e6870526..0b004f71bf841c 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -1458,7 +1458,7 @@ _spack_mark() { } _spack_migrate() { - SPACK_COMPREPLY="-h --help --dry-run --clear --clear-only --replace --restore-old-configs" + SPACK_COMPREPLY="-h --help --dry-run --clear --clear-only --replace --restore --i-need-old-spack" } _spack_mirror() { diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 90ac230bd2bfe8..4042869d71a9f9 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -2359,7 +2359,7 @@ complete -c spack -n '__fish_spack_using_command mark' -s i -l implicit -f -a im complete -c spack -n '__fish_spack_using_command mark' -s i -l implicit -d 'mark packages as implicitly installed' # spack migrate -set -g __fish_spack_optspecs_spack_migrate h/help dry-run clear clear-only replace restore-old-configs +set -g __fish_spack_optspecs_spack_migrate h/help dry-run clear clear-only replace restore i-need-old-spack complete -c spack -n '__fish_spack_using_command migrate' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command migrate' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command migrate' -l dry-run -f -a dry_run @@ -2370,8 +2370,10 @@ complete -c spack -n '__fish_spack_using_command migrate' -l clear-only -f -a cl complete -c spack -n '__fish_spack_using_command migrate' -l clear-only -d 'only move ~/.spack to backup without migrating files (useful after running migrate without --clear)' complete -c spack -n '__fish_spack_using_command migrate' -l replace -f -a replace complete -c spack -n '__fish_spack_using_command migrate' -l replace -d 'replace existing files in new locations (use with --clear if you forgot to use --clear on first run)' -complete -c spack -n '__fish_spack_using_command migrate' -l restore-old-configs -f -a restore_old_configs -complete -c spack -n '__fish_spack_using_command migrate' -l restore-old-configs -d 'restore ~/.spack from backup location (after --clear)' +complete -c spack -n '__fish_spack_using_command migrate' -l restore -f -a restore +complete -c spack -n '__fish_spack_using_command migrate' -l restore -d 'restore ~/.spack from backup location (after --clear)' +complete -c spack -n '__fish_spack_using_command migrate' -l i-need-old-spack -f -a i_need_old_spack +complete -c spack -n '__fish_spack_using_command migrate' -l i-need-old-spack -d 'print help about mixing pre-1.2-Spack and Spack >= 1.2' # spack mirror set -g __fish_spack_optspecs_spack_mirror h/help n/no-checksum From 0a9f24e10c7aaf2f7f18ce9eb3dfbdbaf4a64ebd Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 May 2026 10:46:50 -0700 Subject: [PATCH 490/506] rm unnecessary local import --- lib/spack/spack/main.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 3d170a4fd3bde2..37bd2d06bbb835 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -950,17 +950,15 @@ def uses_old_dotspack(path): # A config scope explicitly targets ~/.spack return - from spack.paths import locations as paths - explicit_path = False - if uses_old_dotspack(paths.state_home): - if paths.default_state_home_dot_spack: + if uses_old_dotspack(spack.paths.locations.state_home): + if spack.paths.locations.default_state_home_dot_spack: reasons.append("User cache path fallback") else: explicit_path = True - if uses_old_dotspack(paths.data_home): + if uses_old_dotspack(spack.paths.locations.data_home): explicit_path = True - if uses_old_dotspack(paths.cache_home): + if uses_old_dotspack(spack.paths.locations.cache_home): explicit_path = True if explicit_path: From a015388bd21a7ad1d1504bdd9ab67de559084b03 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 May 2026 11:15:42 -0700 Subject: [PATCH 491/506] updated help message --- lib/spack/spack/cmd/migrate.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py index bf37088b0c242a..94f625dcb6708b 100644 --- a/lib/spack/spack/cmd/migrate.py +++ b/lib/spack/spack/cmd/migrate.py @@ -87,26 +87,24 @@ def restore(args: argparse.Namespace) -> None: def i_need_old_spack(): + # Note: I specifically recommend SPACK_USER_CACHE_PATH= rather + # than SPACK_STATE_HOME=..., because if a pre-1.2 instance *is* + # eventually updated, it would take the existence of + # SPACK_STATE_HOME as an indication to use new-style defaults + # for installs etc., and ignore existing installs in + # $spack/opt/spack print("""\ If you're getting a warning about using resources in ~/.spack, and you have pre-1.2 Spack instances that cannot upgrade. Then it is -recommended that both old and new instances use ~/.spack +recommended that both old and new instances use ~/.spack for +configs and the user cache path. -In the new instance, you can do that by adding config: +If a new instance sees SPACK_USER_CACHE_PATH=~/.spack, that will +silence the warning. -config: - locations: - state: ~/.spack - -or setting the SPACK_STATE_HOME/SPACK_USER_CACHE environment variable. - -Alternatively, you can set: - -include: - - name: "user" - path: "~/.spack/" - -(in etc/spack/include.yaml) +Other explicit uses of ~/.spack will also silence this warning +(e.g. setting `config:locations:state:~/.spack`, or pointing +the user scope's `path` to `~/.spack`). """) From 2ea1a093e6c9ba053ca84efe69dae0ad800ceaab Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 May 2026 11:18:36 -0700 Subject: [PATCH 492/506] add TODO --- lib/spack/spack/cmd/migrate.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py index 94f625dcb6708b..90d143ef7a2060 100644 --- a/lib/spack/spack/cmd/migrate.py +++ b/lib/spack/spack/cmd/migrate.py @@ -105,6 +105,11 @@ def i_need_old_spack(): Other explicit uses of ~/.spack will also silence this warning (e.g. setting `config:locations:state:~/.spack`, or pointing the user scope's `path` to `~/.spack`). + +TODO: IMO this could also suggest `spack migrate` (no --clear) to +create a divergent cache/config (and once the new spack instance +is isolated from ~/.spack, stop warning). I have to update the +warn logic to accept this though (which is easy). """) From b7c0da13ad591867d3c1d7387252cdcd733d4238 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 May 2026 11:44:44 -0700 Subject: [PATCH 493/506] style --- lib/spack/spack/cmd/migrate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py index 90d143ef7a2060..1d49c884ea61eb 100644 --- a/lib/spack/spack/cmd/migrate.py +++ b/lib/spack/spack/cmd/migrate.py @@ -105,7 +105,7 @@ def i_need_old_spack(): Other explicit uses of ~/.spack will also silence this warning (e.g. setting `config:locations:state:~/.spack`, or pointing the user scope's `path` to `~/.spack`). - + TODO: IMO this could also suggest `spack migrate` (no --clear) to create a divergent cache/config (and once the new spack instance is isolated from ~/.spack, stop warning). I have to update the From 4286422a5ab5d09502fc54bd16dd28a73bbca666 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 May 2026 16:47:45 -0700 Subject: [PATCH 494/506] debugging output when warning is skipped --- lib/spack/spack/main.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 37bd2d06bbb835..0c317af4069a06 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -933,6 +933,7 @@ def _warn_about_old_dotspack(): # Don't warn if it doesn't exist if not os.path.exists(old_dotspack): + tty.debug("Skip .spack warning: no ~/.spack directory") return # Helper to check if old_dotspack is a prefix @@ -948,6 +949,9 @@ def uses_old_dotspack(path): reasons.append(f"Used by config scope: {scope.name}") else: # A config scope explicitly targets ~/.spack + tty.debug( + f"Skip .spack warning: scope {scope.name} includes ~/.spack: {scope.path}" + ) return explicit_path = False @@ -955,10 +959,20 @@ def uses_old_dotspack(path): if spack.paths.locations.default_state_home_dot_spack: reasons.append("User cache path fallback") else: + tty.debug( + "Skip .spack warning:" + f" state_home includes .spack {spack.paths.locations.state_home}" + ) explicit_path = True if uses_old_dotspack(spack.paths.locations.data_home): + tty.debug( + f"Skip .spack warning: data_home includes .spack {spack.paths.locations.data_home}" + ) explicit_path = True if uses_old_dotspack(spack.paths.locations.cache_home): + tty.debug( + f"Skip .spack warning: cache_home includes .spack {spack.paths.locations.cache_home}" + ) explicit_path = True if explicit_path: From 423fdeed885f592864370de19ed2911988c3e5d3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 23 May 2026 21:06:36 -0700 Subject: [PATCH 495/506] paths: add module shim so spack.paths.X keeps working Reverts the renames forced by the previous `from spack.paths import locations as paths` pattern across ~60 callers by adding a sys.modules swap at the bottom of spack/paths.py. The shim subclasses ModuleType and delegates attribute access on the module to the `locations` singleton (which itself delegates to paths_base for static attributes via SpackPaths.__getattr__). Net result: every consumer can keep `import spack.paths` and `spack.paths.X`, regardless of whether X is a static path (prefix, bin_path) or a config-dependent one (state_home, default_install_location). Behavior is unchanged. Uses a sys.modules swap rather than module-level __getattr__ because that's PEP 562 (Python 3.7+) and we still support 3.6. Comment notes the simpler replacement once that minimum bumps. Reduces overall PR diff by ~325 lines. --- lib/spack/spack/binary_distribution.py | 4 +-- lib/spack/spack/ci/__init__.py | 10 +++--- lib/spack/spack/cmd/__init__.py | 4 +-- lib/spack/spack/cmd/clean.py | 4 +-- lib/spack/spack/cmd/commands.py | 10 +++--- lib/spack/spack/cmd/edit.py | 12 +++---- lib/spack/spack/cmd/gpg.py | 4 +-- lib/spack/spack/cmd/install.py | 4 +-- lib/spack/spack/cmd/license.py | 8 ++--- lib/spack/spack/cmd/location.py | 6 ++-- lib/spack/spack/cmd/make_installer.py | 4 +-- lib/spack/spack/cmd/migrate.py | 10 +++--- lib/spack/spack/cmd/style.py | 16 ++++----- lib/spack/spack/cmd/tutorial.py | 6 ++-- lib/spack/spack/cmd/unit_test.py | 12 ++++--- lib/spack/spack/environment/depfile.py | 4 +-- lib/spack/spack/environment/environment.py | 4 +-- lib/spack/spack/modules/common.py | 4 +-- lib/spack/spack/paths.py | 19 +++++++++++ lib/spack/spack/repo.py | 8 +++-- lib/spack/spack/reporters/cdash.py | 4 +-- lib/spack/spack/spec.py | 4 +-- lib/spack/spack/store.py | 8 ++--- lib/spack/spack/test/builder.py | 4 +-- lib/spack/spack/test/ci.py | 10 +++--- lib/spack/spack/test/cmd/blame.py | 8 ++--- lib/spack/spack/test/cmd/buildcache.py | 4 +-- lib/spack/spack/test/cmd/ci.py | 8 +++-- lib/spack/spack/test/cmd/commands.py | 8 ++--- lib/spack/spack/test/cmd/diff.py | 4 +-- lib/spack/spack/test/cmd/edit.py | 4 +-- lib/spack/spack/test/cmd/env.py | 10 +++--- lib/spack/spack/test/cmd/find.py | 4 +-- lib/spack/spack/test/cmd/gpg.py | 10 +++--- lib/spack/spack/test/cmd/license.py | 6 ++-- lib/spack/spack/test/cmd/list.py | 6 ++-- lib/spack/spack/test/cmd/location.py | 12 +++---- lib/spack/spack/test/cmd/pkg.py | 4 +-- lib/spack/spack/test/cmd/style.py | 33 ++++++++++--------- lib/spack/spack/test/cmd/test.py | 4 +-- .../test/concretization/compiler_runtimes.py | 4 +-- lib/spack/spack/test/concretization/core.py | 10 +++--- .../spack/test/concretization/flag_mixing.py | 4 +-- .../spack/test/concretization/requirements.py | 4 +-- lib/spack/spack/test/database.py | 4 +-- lib/spack/spack/test/directory_layout.py | 6 ++-- .../spack/test/environment_modifications.py | 4 +-- lib/spack/spack/test/link_paths.py | 8 ++--- lib/spack/spack/test/llnl/util/file_list.py | 4 +-- lib/spack/spack/test/packages.py | 10 +++--- lib/spack/spack/test/packaging.py | 6 ++-- lib/spack/spack/test/patch.py | 4 +-- lib/spack/spack/test/repo.py | 24 +++++++------- lib/spack/spack/test/spec_semantics.py | 4 +-- lib/spack/spack/test/spec_yaml.py | 6 ++-- lib/spack/spack/test/util/compression.py | 6 ++-- lib/spack/spack/test/util/package_hash.py | 4 +-- lib/spack/spack/test/web.py | 4 +-- lib/spack/spack/util/gpg.py | 4 +-- lib/spack/spack/version/git_ref_lookup.py | 4 +-- 60 files changed, 233 insertions(+), 199 deletions(-) diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index c3af0ac2a7716c..d42f6129a1130a 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -57,6 +57,7 @@ import spack.oci.image import spack.oci.oci import spack.oci.opener +import spack.paths import spack.platforms import spack.relocate as relocate import spack.spec @@ -91,7 +92,6 @@ upload_manifest_with_retry, ) from spack.package_prefs import get_package_dir_permissions, get_package_group -from spack.paths import locations as paths from spack.relocate_text import utf8_paths_to_single_binary_regex from spack.stage import Stage from spack.util.executable import which @@ -564,7 +564,7 @@ def get_buildinfo_dict(spec): return { "sbang_install_path": spack.hooks.sbang.sbang_install_path(), "buildpath": spack.store.STORE.layout.root, - "spackprefix": paths.prefix, + "spackprefix": spack.paths.prefix, "relative_prefix": os.path.relpath(spec.prefix, spack.store.STORE.layout.root), # "relocate_textfiles": [], # "relocate_binaries": [], diff --git a/lib/spack/spack/ci/__init__.py b/lib/spack/spack/ci/__init__.py index 9079705b760203..2e900bc6ed7969 100644 --- a/lib/spack/spack/ci/__init__.py +++ b/lib/spack/spack/ci/__init__.py @@ -26,6 +26,7 @@ import spack.llnl.util.tty as tty import spack.main import spack.mirrors.mirror +import spack.paths import spack.repo import spack.spec import spack.stage @@ -38,7 +39,6 @@ from spack import traverse from spack.error import SpackError from spack.llnl.util.tty.color import cescape, colorize -from spack.paths import locations as paths from spack.reporters.cdash import SPACK_CDASH_TIMEOUT from .common import ( @@ -741,11 +741,11 @@ def download_and_extract_artifacts(url: str, work_dir: str) -> str: def get_spack_info(): """If spack is running from a git repo, return the most recent git log entry, otherwise, return a string containing the spack version.""" - git_path = os.path.join(paths.prefix, ".git") + git_path = os.path.join(spack.paths.prefix, ".git") if os.path.exists(git_path): git = spack.util.git.git() if git: - with fs.working_dir(paths.prefix): + with fs.working_dir(spack.paths.prefix): git_log = git("log", "-1", output=str, error=os.devnull, fail_on_error=False) return git_log @@ -780,12 +780,12 @@ def setup_spack_repro_version( tty.info(f"checkout_commit: {checkout_commit}") tty.info(f"merge_commit: {merge_commit}") - dot_git_path = os.path.join(paths.prefix, ".git") + dot_git_path = os.path.join(spack.paths.prefix, ".git") if not os.path.exists(dot_git_path): tty.error("Unable to find the path to your local spack clone") return False - spack_git_path = paths.prefix + spack_git_path = spack.paths.prefix git = spack.util.git.git() if not git: diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 5a55a60c8af889..8df8c810521dfa 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -20,6 +20,7 @@ import spack.extensions import spack.llnl.string import spack.llnl.util.tty as tty +import spack.paths import spack.repo import spack.spec import spack.spec_parser @@ -32,7 +33,6 @@ from spack.llnl.util.lang import attr_setdefault, index_by from spack.llnl.util.tty.colify import colify from spack.llnl.util.tty.color import colorize -from spack.paths import locations as paths from spack.paths_base import locations as paths_base from ..enums import InstallRecordStatus @@ -581,7 +581,7 @@ def print_how_many_pkgs(specs, pkg_type="", suffix=""): def spack_is_git_repo(): """Ensure that this instance of Spack is a git clone.""" - return is_git_repo(paths.prefix) + return is_git_repo(spack.paths.prefix) def is_git_repo(path): diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 061f738d61b581..965aba9c0bfe70 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -11,11 +11,11 @@ import spack.config import spack.llnl.util.filesystem import spack.llnl.util.tty as tty +import spack.paths import spack.stage import spack.store import spack.util.path from spack.cmd.common import arguments -from spack.paths import locations as paths description = "remove temporary build files and/or downloaded archives" section = "build" @@ -67,7 +67,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def remove_python_cache(): - for directory in [paths.lib_path]: + for directory in [spack.paths.lib_path]: for root, dirs, files in os.walk(directory): for f in files: if f.endswith(".pyc") or f.endswith(".pyo"): diff --git a/lib/spack/spack/cmd/commands.py b/lib/spack/spack/cmd/commands.py index bfb41a8bdb8538..4ebf3fa2691381 100644 --- a/lib/spack/spack/cmd/commands.py +++ b/lib/spack/spack/cmd/commands.py @@ -15,11 +15,11 @@ import spack.config import spack.llnl.util.tty as tty import spack.main +import spack.paths import spack.platforms from spack.llnl.util.argparsewriter import ArgparseRstWriter, ArgparseWriter, Command from spack.llnl.util.tty.colify import colify from spack.main import SpackArgumentParser, section_descriptions -from spack.paths import locations as paths description = "list available spack commands" section = "config" @@ -36,14 +36,14 @@ "bash": { "aliases": True, "format": "bash", - "header": os.path.join(paths.share_path, "bash", "spack-completion.bash"), - "update": os.path.join(paths.share_path, "spack-completion.bash"), + "header": os.path.join(spack.paths.share_path, "bash", "spack-completion.bash"), + "update": os.path.join(spack.paths.share_path, "spack-completion.bash"), }, "fish": { "aliases": True, "format": "fish", - "header": os.path.join(paths.share_path, "fish", "spack-completion.fish"), - "update": os.path.join(paths.share_path, "spack-completion.fish"), + "header": os.path.join(spack.paths.share_path, "fish", "spack-completion.fish"), + "update": os.path.join(spack.paths.share_path, "spack-completion.fish"), }, } diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index 5ae74581129029..67fe9a15e84283 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -10,9 +10,9 @@ import spack.cmd import spack.llnl.util.tty as tty +import spack.paths import spack.repo import spack.util.editor -from spack.paths import locations as paths description = "open package files in ``$EDITOR``" section = "packaging" @@ -38,7 +38,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: "--command", dest="path", action="store_const", - const=paths.command_path, + const=spack.paths.command_path, help="edit the command with the supplied name", ) excl_args.add_argument( @@ -46,7 +46,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: "--docs", dest="path", action="store_const", - const=os.path.join(paths.lib_path, "docs"), + const=os.path.join(spack.paths.lib_path, "docs"), help="edit the docs with the supplied name", ) excl_args.add_argument( @@ -54,7 +54,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: "--test", dest="path", action="store_const", - const=paths.test_path, + const=spack.paths.test_path, help="edit the test with the supplied name", ) excl_args.add_argument( @@ -62,7 +62,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: "--module", dest="path", action="store_const", - const=paths.module_path, + const=spack.paths.module_path, help="edit the main spack module with the supplied name", ) @@ -117,7 +117,7 @@ def locate_build_system(name: str, repo: Optional[spack.repo.Repo]) -> str: def locate_file(name: str, path: str) -> str: # convert command names to python module name - if path == paths.command_path: + if path == spack.paths.command_path: name = spack.cmd.python_name(name) file_path = os.path.join(path, name) diff --git a/lib/spack/spack/cmd/gpg.py b/lib/spack/spack/cmd/gpg.py index c83dfb2b38c92c..3f1f5233f6cde5 100644 --- a/lib/spack/spack/cmd/gpg.py +++ b/lib/spack/spack/cmd/gpg.py @@ -8,11 +8,11 @@ import spack.binary_distribution import spack.mirrors.mirror +import spack.paths import spack.stage import spack.util.gpg import spack.util.url from spack.cmd.common import arguments -from spack.paths import locations as paths description = "handle GPG actions for spack" section = "packaging" @@ -187,7 +187,7 @@ def gpg_init(args): """add the default keys to the keyring""" import_dir = args.import_dir if import_dir is None: - import_dir = paths.gpg_keys_path + import_dir = spack.paths.gpg_keys_path for root, _, filenames in os.walk(import_dir): for filename in filenames: diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 04f91dd7f98aff..d610e090ebe086 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -13,6 +13,7 @@ import spack.environment as ev import spack.installer_dispatch import spack.llnl.util.filesystem as fs +import spack.paths import spack.spec import spack.store from spack.cmd.common import arguments @@ -20,7 +21,6 @@ from spack.installer import InstallPolicy from spack.llnl.string import plural from spack.llnl.util import tty -from spack.paths import locations as paths description = "build and install packages" section = "build" @@ -230,7 +230,7 @@ def default_log_file(spec): the corresponding directory if not present """ basename = spec.format_path("test-{name}-{version}-{hash}.xml") - dirname = fs.os.path.join(paths.reports_path, "junit") + dirname = fs.os.path.join(spack.paths.reports_path, "junit") fs.mkdirp(dirname) return fs.os.path.join(dirname, basename) diff --git a/lib/spack/spack/cmd/license.py b/lib/spack/spack/cmd/license.py index 71a6abe0f94992..dd8a970383fb60 100644 --- a/lib/spack/spack/cmd/license.py +++ b/lib/spack/spack/cmd/license.py @@ -10,7 +10,7 @@ from typing import Dict, Generator import spack.llnl.util.tty as tty -from spack.paths import locations as paths +import spack.paths description = "list and check license headers on files in spack" section = "query" @@ -61,7 +61,7 @@ ] -def _licensed_files(root: str = paths.prefix) -> Generator[str, None, None]: +def _licensed_files(root: str = spack.paths.prefix) -> Generator[str, None, None]: """Generates paths of licensed files.""" licensed_files = re.compile("|".join(f"(?:{pattern})" for pattern in licensed_files_patterns)) dirs = [ @@ -84,7 +84,7 @@ def _licensed_files(root: str = paths.prefix) -> Generator[str, None, None]: def list_files(args): """list files in spack that should have license headers""" for relpath in sorted(_licensed_files(args.root)): - print(os.path.join(paths.spack_root, relpath)) + print(os.path.join(spack.paths.spack_root, relpath)) # Error codes for license verification. All values are chosen such that @@ -189,7 +189,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "--root", action="store", - default=paths.prefix, + default=spack.paths.prefix, help="scan a different prefix for license issues", ) diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 2b13fe60f5c305..f4f8e1f6258410 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -9,11 +9,11 @@ import spack.cmd import spack.environment as ev import spack.llnl.util.tty as tty +import spack.paths import spack.repo import spack.stage import spack.store from spack.cmd.common import arguments -from spack.paths import locations as paths description = "print out locations of packages and spack directories" section = "query" @@ -107,11 +107,11 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def location(parser, args): if args.module_dir: - print(paths.module_path) + print(spack.paths.module_path) return if args.spack_root: - print(paths.prefix) + print(spack.paths.prefix) return if args.install_root: diff --git a/lib/spack/spack/cmd/make_installer.py b/lib/spack/spack/cmd/make_installer.py index b3f526f813d6dd..4b6a9bf81be989 100644 --- a/lib/spack/spack/cmd/make_installer.py +++ b/lib/spack/spack/cmd/make_installer.py @@ -7,9 +7,9 @@ import sys import spack.concretize +import spack.paths import spack.util.executable from spack.llnl.path import convert_to_posix_path -from spack.paths import locations as paths description = "generate Windows installer" section = "admin" @@ -86,7 +86,7 @@ def make_installer(parser, args): here = os.path.dirname(os.path.abspath(__file__)) source_dir = os.path.join(here, "installer") - posix_root = convert_to_posix_path(paths.spack_root) + posix_root = convert_to_posix_path(spack.paths.spack_root) spack_license = posixpath.join(posix_root, "LICENSE-APACHE") rtf_spack_license = txt_to_rtf(spack_license) spack_license = posixpath.join(source_dir, "LICENSE.rtf") diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py index 1d49c884ea61eb..43e8f908ba1f89 100644 --- a/lib/spack/spack/cmd/migrate.py +++ b/lib/spack/spack/cmd/migrate.py @@ -7,7 +7,7 @@ import shutil import spack.llnl.util.tty as tty -from spack.paths import locations as paths +import spack.paths from spack.paths_base import locations as paths_base description = "migrate user config and cache from old to new locations" @@ -56,9 +56,9 @@ def restore(args: argparse.Namespace) -> None: old_location = os.path.expanduser("~/.spack") # Check both the current backup location and the default one - backup_locations = [paths.dotspack_backup] - default_backup = os.path.join(paths.default_data_home, "dotspack_backup") - if default_backup != paths.dotspack_backup: + backup_locations = [spack.paths.dotspack_backup] + default_backup = os.path.join(spack.paths.default_data_home, "dotspack_backup") + if default_backup != spack.paths.dotspack_backup: backup_locations.append(default_backup) # Find which backup location exists @@ -131,7 +131,7 @@ def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: old_location = os.path.expanduser("~/.spack") new_config_location = paths_base.user_config_path new_state_location = os.path.join(os.path.expanduser("~"), ".local", "state", "spack") - backup_location = paths.dotspack_backup + backup_location = spack.paths.dotspack_backup if args.clear_only: if not os.path.exists(old_location): diff --git a/lib/spack/spack/cmd/style.py b/lib/spack/spack/cmd/style.py index e7b828805c5a4b..a9e3e20bf14904 100644 --- a/lib/spack/spack/cmd/style.py +++ b/lib/spack/spack/cmd/style.py @@ -11,6 +11,7 @@ import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color +import spack.paths import spack.repo import spack.util.git from spack.cmd.common.spec_strings import ( @@ -19,7 +20,6 @@ _spec_str_fix_handler, ) from spack.llnl.util.filesystem import working_dir -from spack.paths import locations as paths from spack.util.executable import Executable, which description = "runs source code style checks on spack" @@ -27,7 +27,7 @@ level = "long" #: List of paths to exclude from checks -- relative to spack root -exclude_paths = [os.path.relpath(paths.vendor_path, paths.prefix)] +exclude_paths = [os.path.relpath(spack.paths.vendor_path, spack.paths.prefix)] #: Order in which tools should be run. #: The list maps an executable name to a method to ensure the tool is @@ -80,7 +80,7 @@ def changed_files(base="develop", untracked=True, all_files=False, root=None) -> root (str): use this directory instead of the Spack prefix. """ if root is None: - root = paths.prefix + root = spack.paths.prefix git = spack.util.git.git(required=True) @@ -244,7 +244,7 @@ def print_tool_result(tool, returncode): @tool("ruff-check", cmd="ruff") def ruff_check(file_list, args): """Run the ruff-check command. Handles config and non generic ruff argument logic""" - cmd_args = ["--config", os.path.join(paths.prefix, "pyproject.toml"), "--quiet"] + cmd_args = ["--config", os.path.join(spack.paths.prefix, "pyproject.toml"), "--quiet"] if args.fix: cmd_args += ["--fix", "--no-unsafe-fixes"] else: @@ -257,7 +257,7 @@ def ruff_check(file_list, args): @tool("ruff-format", cmd="ruff") def ruff_format(file_list, args): """Run the ruff format command""" - cmd_args = ["--config", os.path.join(paths.prefix, "pyproject.toml"), "--quiet"] + cmd_args = ["--config", os.path.join(spack.paths.prefix, "pyproject.toml"), "--quiet"] if not args.fix: cmd_args += ["--check", "--diff"] return run_ruff( @@ -303,7 +303,7 @@ def run_mypy(file_list, args): # always run with config from running spack prefix common_mypy_args = [ "--config-file", - os.path.join(paths.prefix, "pyproject.toml"), + os.path.join(spack.paths.prefix, "pyproject.toml"), "--show-error-codes", ] mypy_arg_sets = [common_mypy_args + ["--package", "spack", "--package", "llnl"]] @@ -516,10 +516,10 @@ def style(parser, args): # ensure that the config files we need actually exist in the spack prefix. # assertions b/c users should not ever see these errors -- they're checked in CI. - assert (Path(paths.prefix) / "pyproject.toml").is_file() + assert (Path(spack.paths.prefix) / "pyproject.toml").is_file() # validate spack root if the user provided one - args.root = Path(args.root).resolve() if args.root else Path(paths.prefix) + args.root = Path(args.root).resolve() if args.root else Path(spack.paths.prefix) spack_script = args.root / "bin" / "spack" if not spack_script.exists(): tty.die("This does not look like a valid spack root.", "No such file: '%s'" % spack_script) diff --git a/lib/spack/spack/cmd/tutorial.py b/lib/spack/spack/cmd/tutorial.py index dc93e9e9844b4f..afa2463ad63285 100644 --- a/lib/spack/spack/cmd/tutorial.py +++ b/lib/spack/spack/cmd/tutorial.py @@ -10,11 +10,11 @@ import spack.cmd import spack.config import spack.llnl.util.tty as tty +import spack.paths import spack.util.git import spack.util.gpg from spack.cmd.common import arguments from spack.llnl.util.filesystem import working_dir -from spack.paths import locations as paths from spack.util.spack_yaml import syaml_dict description = "set up spack for our tutorial (WARNING: modifies config!)" @@ -25,7 +25,7 @@ # tutorial configuration parameters tutorial_branch = "releases/v1.1" tutorial_mirror = "file:///mirror" -tutorial_key = os.path.join(paths.share_path, "keys", "tutorial.pub") +tutorial_key = os.path.join(spack.paths.share_path, "keys", "tutorial.pub") # configs to remove rm_configs = [ @@ -83,6 +83,6 @@ def tutorial(parser, args): # that follows (exacerbated by the various lazy singletons we use) tty.msg(f"Ensuring we're on the {tutorial_branch} branch") git = spack.util.git.git(required=True) - with working_dir(paths.prefix): + with working_dir(spack.paths.prefix): git("checkout", tutorial_branch) # NO CODE BEYOND HERE diff --git a/lib/spack/spack/cmd/unit_test.py b/lib/spack/spack/cmd/unit_test.py index 0546d2e5ca9e43..271da82c4918e6 100644 --- a/lib/spack/spack/cmd/unit_test.py +++ b/lib/spack/spack/cmd/unit_test.py @@ -19,8 +19,8 @@ import spack.llnl.util.filesystem import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color +import spack.paths from spack.llnl.util.tty.colify import colify -from spack.paths import locations as paths description = "run spack's unit tests (wrapper around pytest)" section = "developer" @@ -120,9 +120,11 @@ def colorize(c, prefix): # To list the files we just need to inspect the filesystem, # which doesn't need to wait for pytest collection and doesn't # require parsing pytest output - files = spack.llnl.util.filesystem.find(root=paths.test_path, files="*.py", recursive=True) + files = spack.llnl.util.filesystem.find( + root=spack.paths.test_path, files="*.py", recursive=True + ) files = [ - os.path.relpath(f, start=paths.spack_root) + os.path.relpath(f, start=spack.paths.spack_root) for f in files if not f.endswith(("conftest.py", "__init__.py")) ] @@ -152,7 +154,7 @@ def colorize(c, prefix): len_indent = len(indent) if os.path.isabs(name): - name = os.path.relpath(name, start=paths.spack_root) + name = os.path.relpath(name, start=spack.paths.spack_root) item = (len_indent, name, nodetype) @@ -231,7 +233,7 @@ def unit_test(parser, args, unknown_args): # The default is to test the core of Spack. If the option `--extension` # has been used, then test that extension. - pytest_root = paths.spack_root + pytest_root = spack.paths.spack_root if args.extension: pytest_root = spack.extensions.load_extension(args.extension) diff --git a/lib/spack/spack/environment/depfile.py b/lib/spack/spack/environment/depfile.py index 71c2dbf5147a80..46389b0c24ffa0 100644 --- a/lib/spack/spack/environment/depfile.py +++ b/lib/spack/spack/environment/depfile.py @@ -14,9 +14,9 @@ import spack.deptypes as dt import spack.environment.environment as ev +import spack.paths import spack.spec import spack.traverse as traverse -from spack.paths import locations as paths class UseBuildCache(Enum): @@ -229,7 +229,7 @@ def to_dict(self): "install_deps_target": self._target("install-deps"), "any_hash_target": self._target("%"), "jobserver_support": self.jobserver_support, - "spack_script": shlex.quote(paths.spack_script), + "spack_script": shlex.quote(spack.paths.spack_script), "adjacency_list": self.make_adjacency_list, "phony_convenience_targets": " ".join(self.phony_convenience_targets), "pkg_ids_variable": self.pkg_identifier_variable, diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index 326712571360d0..c24947dffbdb24 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -39,6 +39,7 @@ import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as clr import spack.package_base +import spack.paths import spack.repo import spack.schema.env import spack.spec @@ -56,7 +57,6 @@ from spack.llnl.util.filesystem import copy_tree, islink, readlink, symlink from spack.llnl.util.lang import stable_partition from spack.llnl.util.link_tree import ConflictingSpecsError -from spack.paths import locations as paths from spack.schema.env import TOP_LEVEL_KEY from spack.spec import Spec from spack.spec_filter import SpecFilter @@ -102,7 +102,7 @@ def default_env_path(): global _default_env_path if not _default_env_path: - _default_env_path = paths.default_envs_path + _default_env_path = spack.paths.default_envs_path return _default_env_path diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py index eeb47eb12d20fa..9ed331b0b19b2e 100644 --- a/lib/spack/spack/modules/common.py +++ b/lib/spack/spack/modules/common.py @@ -43,6 +43,7 @@ import spack.error import spack.llnl.util.filesystem import spack.llnl.util.tty as tty +import spack.paths import spack.projections as proj import spack.schema import spack.schema.environment @@ -56,7 +57,6 @@ import spack.util.spack_yaml as syaml from spack.context import Context from spack.llnl.util.lang import Singleton, dedupe, memoized -from spack.paths import locations as paths #: config section for this file @@ -213,7 +213,7 @@ def root_path(name, module_set_name): # Root folders where the various module files should be written roots = spack.config.get(f"modules:{module_set_name}:roots", {}) - path = roots.get(name, os.path.join(paths.data_home, name)) + path = roots.get(name, os.path.join(spack.paths.data_home, name)) return spack.util.path.canonicalize_path(path) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 0aca1cb57e2201..557379e39a6897 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -12,6 +12,8 @@ import itertools import os import pathlib +import sys as _sys +import types as _types from contextlib import contextmanager from enum import Enum from functools import partial @@ -448,3 +450,20 @@ def restore(bundled_state): locations._cache_home = bundled_state["cache_home"] locations.old_layout_detected = bundled_state["old_layout_detected"] locations._new_layout_enforced = bundled_state["new_layout_enforced"] + + +# Module shim: lets callers keep using `spack.paths.X` for any attribute on +# `locations` (e.g. `spack.paths.gpg_path`), which itself delegates to +# `paths_base.locations` for static attributes (e.g. `spack.paths.prefix`). +# Uses a sys.modules swap because Spack still supports Python 3.6, which +# predates PEP 562 (module-level `__getattr__`). When 3.6 support is dropped, +# replace this block with a plain +# `def __getattr__(name): return getattr(locations, name)`. +class _PathsModule(_types.ModuleType): + def __getattr__(self, name): + return getattr(locations, name) + + +_shim = _PathsModule(__name__) +_shim.__dict__.update(_sys.modules[__name__].__dict__) +_sys.modules[__name__] = _shim diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py index 8844c75f203157..c62be55213511a 100644 --- a/lib/spack/spack/repo.py +++ b/lib/spack/spack/repo.py @@ -46,6 +46,7 @@ import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.patch +import spack.paths import spack.provider_index import spack.tag import spack.util.executable @@ -58,7 +59,6 @@ import spack.util.spack_yaml as syaml from spack.llnl.util.filesystem import working_dir from spack.llnl.util.lang import Singleton, memoized -from spack.paths import locations as paths if TYPE_CHECKING: import spack.package_base @@ -74,7 +74,9 @@ def package_repository_lock() -> spack.util.lock.Lock: """Lock for process safety when cloning remote package repositories""" - return spack.util.lock.Lock(os.path.join(paths.user_cache_path, "package-repository.lock")) + return spack.util.lock.Lock( + os.path.join(spack.paths.user_cache_path, "package-repository.lock") + ) def is_package_module(fullname: str) -> bool: @@ -2062,7 +2064,7 @@ def parse_config_descriptor( if destination is None: # use a default destination dir_name = spack.util.hash.b32_hash(repository)[-7:] - destination = os.path.join(paths.package_repos_path, dir_name) + destination = os.path.join(spack.paths.package_repos_path, dir_name) else: destination = spack.util.path.canonicalize_path(destination) diff --git a/lib/spack/spack/reporters/cdash.py b/lib/spack/spack/reporters/cdash.py index 08754a6378293b..cd3ac31a3724f2 100644 --- a/lib/spack/spack/reporters/cdash.py +++ b/lib/spack/spack/reporters/cdash.py @@ -18,6 +18,7 @@ import spack import spack.llnl.util.tty as tty +import spack.paths import spack.platforms import spack.spec import spack.tengine @@ -25,7 +26,6 @@ import spack.util.web as web_util from spack.error import SpackError from spack.llnl.util.filesystem import working_dir -from spack.paths import locations as paths from spack.util.crypto import checksum from spack.util.log_parse import parse_log_events @@ -117,7 +117,7 @@ def __init__(self, configuration: CDashConfiguration): self.buildIds: Dict[str, str] = {} self.revision = "" git = spack.util.git.git(required=True) - with working_dir(paths.spack_root): + with working_dir(spack.paths.spack_root): self.revision = git("rev-parse", "HEAD", output=str).strip() self.generator = "spack-{0}".format(spack.get_version()) self.multiple_packages = False diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index ade6927fde41a9..fab65cba4a46fa 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -90,6 +90,7 @@ import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as clr import spack.patch +import spack.paths import spack.platforms import spack.provider_index import spack.repo @@ -103,7 +104,6 @@ import spack.version import spack.version as vn import spack.version.git_ref_lookup -from spack.paths import locations as paths from .enums import InstallRecordStatus, PropagationPolicy @@ -4082,7 +4082,7 @@ def namespace_if_anonymous(self): @property def spack_root(self): """Special field for using ``{spack_root}`` in :meth:`format`.""" - return paths.spack_root + return spack.paths.spack_root @property def spack_install(self): diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 669db2c6d5ee44..56e8c53abcd5a1 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -33,11 +33,11 @@ import spack.error import spack.llnl.util.lang import spack.package_prefs +import spack.paths import spack.spec import spack.util.path from spack.llnl.util import filesystem as fs from spack.llnl.util import tty -from spack.paths import locations as paths def parse_install_tree(config_dict: dict) -> Tuple[str, str, Dict[str, str]]: @@ -84,7 +84,7 @@ def parse_install_tree(config_dict: dict) -> Tuple[str, str, Dict[str, str]]: else: unpadded_root = install_tree.get("root", None) if not unpadded_root: - unpadded_root = paths.default_install_location + unpadded_root = spack.paths.default_install_location unpadded_root = spack.util.path.canonicalize_path(unpadded_root) @@ -215,7 +215,7 @@ def install_sbang(self) -> None: sbang_path = os.path.join(self.unpadded_root, "bin", "sbang") try: - if filecmp.cmp(sbang_path, paths.sbang_script): + if filecmp.cmp(sbang_path, spack.paths.sbang_script): return # installed and up to date except FileNotFoundError: pass @@ -239,7 +239,7 @@ def install_sbang(self) -> None: # to ensure we don't delete a file created by another process in the except block. sbang_tmp_file = open(sbang_tmp_path, "xb") try: - with open(paths.sbang_script, "rb") as src, sbang_tmp_file as dst: + with open(spack.paths.sbang_script, "rb") as src, sbang_tmp_file as dst: shutil.copyfileobj(src, dst) os.fchmod(dst.fileno(), config_mode | 0o111) # ensure executable if group_name: diff --git a/lib/spack/spack/test/builder.py b/lib/spack/spack/test/builder.py index d3aa8614998e2a..ed548278a18881 100644 --- a/lib/spack/spack/test/builder.py +++ b/lib/spack/spack/test/builder.py @@ -8,14 +8,14 @@ import spack.builder import spack.concretize +import spack.paths import spack.repo from spack.llnl.util.filesystem import touch -from spack.paths import locations as paths @pytest.fixture() def builder_test_repository(config): - builder_test_path = os.path.join(paths.test_repos_path, "spack_repo", "builder_test") + builder_test_path = os.path.join(spack.paths.test_repos_path, "spack_repo", "builder_test") with spack.repo.use_repositories(builder_test_path) as mock_repo: yield mock_repo diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index 49f2bc2701fd5d..29a1437c3eada2 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -14,9 +14,9 @@ import spack.environment as ev import spack.error import spack.llnl.util.filesystem as fs +import spack.paths import spack.repo as repo import spack.util.git -from spack.paths import locations as paths from spack.spec import Spec from spack.test.conftest import MockHTTPResponse, RepoBuilder from spack.version import Version @@ -190,7 +190,7 @@ def test_pipeline_dag(config, repo_builder: RepoBuilder): @pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_import_signing_key(mock_gnupghome): - signing_key_dir = paths.mock_gpg_keys_path + signing_key_dir = spack.paths.mock_gpg_keys_path signing_key_path = os.path.join(signing_key_dir, "package-signing-key") with open(signing_key_path, encoding="utf-8") as fd: signing_key = fd.read() @@ -204,7 +204,9 @@ def test_download_and_extract_artifacts(tmp_path: pathlib.Path, monkeypatch): url = "https://www.nosuchurlexists.itsfake/artifacts.zip" working_dir = tmp_path / "repro" - test_artifacts_path = os.path.join(paths.test_path, "data", "ci", "gitlab", "artifacts.zip") + test_artifacts_path = os.path.join( + spack.paths.test_path, "data", "ci", "gitlab", "artifacts.zip" + ) def _urlopen_OK(*args, **kwargs): with open(test_artifacts_path, "rb") as f: @@ -267,7 +269,7 @@ def test_setup_spack_repro_version( spack_dir = repro_dir / "spack" spack_dir.mkdir(parents=True) - prefix_save = paths.prefix + prefix_save = spack.paths.prefix override_path("prefix", "/garbage") ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1) diff --git a/lib/spack/spack/test/cmd/blame.py b/lib/spack/spack/test/cmd/blame.py index df7c24b43ff3f2..b08cd43ca096ab 100644 --- a/lib/spack/spack/test/cmd/blame.py +++ b/lib/spack/spack/test/cmd/blame.py @@ -9,11 +9,11 @@ import pytest import spack.cmd.blame +import spack.paths import spack.util.spack_json as sjson from spack.cmd.blame import ensure_full_history, git_prefix, package_repo_root from spack.llnl.util.filesystem import mkdirp, working_dir from spack.main import SpackCommand, SpackCommandError -from spack.paths import locations as paths from spack.repo import RepoDescriptors from spack.util.executable import ProcessError @@ -40,7 +40,7 @@ def test_blame_by_percent(mock_packages): def test_blame_file(): """Sanity check the blame command to make sure it works.""" - with working_dir(paths.prefix): + with working_dir(spack.paths.prefix): out = blame(os.path.join("bin", "spack")) assert "LAST_COMMIT" in out assert "AUTHOR" in out @@ -72,7 +72,7 @@ def test_blame_file_outside_spack_repo(tmp_path: Path): def test_blame_spack_not_git_clone(override_path): """Ensure attempt to get blame when spack not a git clone fails.""" - non_git_dir = os.path.join(paths.prefix, "..") + non_git_dir = os.path.join(spack.paths.prefix, "..") override_path("prefix", non_git_dir) with pytest.raises(SpackCommandError): @@ -82,7 +82,7 @@ def test_blame_spack_not_git_clone(override_path): def test_blame_json(mock_packages): """Ensure that we can output json as a blame.""" - with working_dir(paths.prefix): + with working_dir(spack.paths.prefix): out = blame("--json", "mpich") # Test loading the json, and top level keys diff --git a/lib/spack/spack/test/cmd/buildcache.py b/lib/spack/spack/test/cmd/buildcache.py index 8870dbd59f7c1c..810aeeb6f301c6 100644 --- a/lib/spack/spack/test/cmd/buildcache.py +++ b/lib/spack/spack/test/cmd/buildcache.py @@ -22,13 +22,13 @@ import spack.error import spack.main import spack.mirrors.mirror +import spack.paths import spack.spec import spack.util.url as url_util import spack.util.web as web_util from spack.installer import PackageInstaller from spack.llnl.util.filesystem import copy_tree, find, getuid from spack.llnl.util.lang import nullcontext -from spack.paths import locations as paths from spack.url_buildcache import ( BuildcacheComponent, URLBuildcacheEntry, @@ -564,7 +564,7 @@ def test_push_without_build_deps( def v2_buildcache_layout(tmp_path: pathlib.Path): def _layout(signedness: str = "signed"): source_path = str( - pathlib.Path(paths.test_path) / "data" / "mirrors" / "v2_layout" / signedness + pathlib.Path(spack.paths.test_path) / "data" / "mirrors" / "v2_layout" / signedness ) test_mirror_path = tmp_path / "mirror" copy_tree(source_path, test_mirror_path) diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index fdd72fe44ca101..a2fabac1b24608 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -19,6 +19,7 @@ import spack.environment as ev import spack.hash_types as ht import spack.main +import spack.paths import spack.repo import spack.spec import spack.stage @@ -31,7 +32,6 @@ from spack.cmd.ci import FAILED_CREATE_BUILDCACHE_CODE from spack.error import SpackError from spack.llnl.util.filesystem import mkdirp, working_dir -from spack.paths import locations as paths from spack.schema.database_index import schema as db_idx_schema from spack.test.conftest import MockHTTPResponse, RepoBuilder @@ -477,7 +477,7 @@ def test_ci_rebuild_missing_config(tmp_path: pathlib.Path, working_env, mutable_ def _signing_key(): - signing_key_path = pathlib.Path(paths.mock_gpg_keys_path) / "package-signing-key" + signing_key_path = pathlib.Path(spack.paths.mock_gpg_keys_path) / "package-signing-key" return signing_key_path.read_text() @@ -1100,7 +1100,9 @@ def test_ci_get_stack_changed(mock_git_repo, monkeypatch, override_path): """Test that we can detect the change to .gitlab-ci.yml in a mock spack git repo.""" override_path("prefix", mock_git_repo) - fake_env_path = os.path.join(paths.prefix, os.path.sep.join(("no", "such", "env", "path"))) + fake_env_path = os.path.join( + spack.paths.prefix, os.path.sep.join(("no", "such", "env", "path")) + ) assert ci.stack_changed(fake_env_path) is True diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index ea7f30790fad86..22e96ee4b5579f 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -15,8 +15,8 @@ import spack.cmd.commands import spack.config import spack.main +import spack.paths from spack.cmd.commands import _dest_to_fish_complete, _positional_to_subroutine -from spack.paths import locations as paths from spack.util.executable import Executable @@ -24,7 +24,7 @@ def commands(*args: str) -> str: """Run `spack commands args...` and return output as a string. It's a separate process so that we run through the main Spack command logic and avoid caching issues.""" python = Executable(sys.executable) - return python(paths.spack_script, "commands", *args, output=str) + return python(spack.paths.spack_script, "commands", *args, output=str) def test_names(): @@ -298,9 +298,9 @@ def test_updated_completion_scripts(shell, tmp_path: pathlib.Path, config): ) msg = "\n".join(lines) - header = os.path.join(paths.share_path, shell, f"spack-completion.{shell}") + header = os.path.join(spack.paths.share_path, shell, f"spack-completion.{shell}") script = f"spack-completion.{shell}" - old_script = os.path.join(paths.share_path, script) + old_script = os.path.join(spack.paths.share_path, script) new_script = str(tmp_path / script) commands("--aliases", "--format", shell, "--header", header, "--update", new_script) diff --git a/lib/spack/spack/test/cmd/diff.py b/lib/spack/spack/test/cmd/diff.py index 4ba0c89d847153..db28be8a2b5fa2 100644 --- a/lib/spack/spack/test/cmd/diff.py +++ b/lib/spack/spack/test/cmd/diff.py @@ -9,10 +9,10 @@ import spack.cmd.diff import spack.concretize import spack.main +import spack.paths import spack.repo import spack.util.spack_json as sjson import spack.version -from spack.paths import locations as paths install_cmd = spack.main.SpackCommand("install") diff_cmd = spack.main.SpackCommand("diff") @@ -33,7 +33,7 @@ @pytest.fixture def test_repo(config): - builder_test_path = os.path.join(paths.test_repos_path, "spack_repo", "diff") + builder_test_path = os.path.join(spack.paths.test_repos_path, "spack_repo", "diff") with spack.repo.use_repositories(builder_test_path) as mock_repo: yield mock_repo diff --git a/lib/spack/spack/test/cmd/edit.py b/lib/spack/spack/test/cmd/edit.py index a5b30f3df0babe..47f52fc6d66f87 100644 --- a/lib/spack/spack/test/cmd/edit.py +++ b/lib/spack/spack/test/cmd/edit.py @@ -5,10 +5,10 @@ import os import pathlib +import spack.paths import spack.repo import spack.util.editor from spack.main import SpackCommand -from spack.paths import locations as paths edit = SpackCommand("edit") @@ -61,7 +61,7 @@ def editor(*args: str, **kwargs): monkeypatch.setattr(spack.util.editor, "editor", editor) # set up an additional repo - extra_repo_dir = pathlib.Path(paths.test_repos_path) / "spack_repo" / "requirements_test" + extra_repo_dir = pathlib.Path(spack.paths.test_repos_path) / "spack_repo" / "requirements_test" with spack.repo.use_repositories(str(extra_repo_dir), override=False): edit("--build-system", "builtin_mock.autotools", "builtin_mock.cmake") assert called diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index ac35e2bda30d47..327f2778703094 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -27,6 +27,7 @@ import spack.modules import spack.modules.tcl import spack.package_base +import spack.paths import spack.repo import spack.schema.env import spack.solver.asp @@ -40,7 +41,6 @@ from spack.llnl.util.filesystem import readlink from spack.llnl.util.lang import dedupe from spack.main import SpackCommand, SpackCommandError -from spack.paths import locations as paths from spack.spec import Spec from spack.stage import stage_prefix from spack.test.conftest import RepoBuilder @@ -2076,7 +2076,7 @@ def test_env_include_concrete_env_yaml(env_name): @pytest.mark.regression("45766") @pytest.mark.parametrize("format", ["v1", "v2", "v3"]) def test_env_include_concrete_old_env(format): - lockfile = os.path.join(paths.test_path, "data", "legacy_env", f"{format}.lock") + lockfile = os.path.join(spack.paths.test_path, "data", "legacy_env", f"{format}.lock") # create an env from old .lock file -- this does not update the format env("create", "old-env", lockfile) env("create", "--include-concrete", "old-env", "test") @@ -3958,7 +3958,7 @@ def test_read_old_lock_and_write_new(tmp_path: pathlib.Path, lockfile): # the environment, anyway. # # This test ensures the behavior described above. - lockfile_path = os.path.join(paths.test_path, "data", "legacy_env", "%s.lock" % lockfile) + lockfile_path = os.path.join(spack.paths.test_path, "data", "legacy_env", "%s.lock" % lockfile) # read in the JSON from a legacy lockfile with open(lockfile_path, encoding="utf-8") as f: @@ -4009,7 +4009,7 @@ def test_read_v1_lock_creates_backup(tmp_path: pathlib.Path): """When reading a version-1 lockfile, make sure that a backup of that file is created. """ - v1_lockfile_path = pathlib.Path(paths.test_path) / "data" / "legacy_env" / "v1.lock" + v1_lockfile_path = pathlib.Path(spack.paths.test_path) / "data" / "legacy_env" / "v1.lock" test_lockfile_path = tmp_path / "init" / ev.lockfile_name test_lockfile_path.parent.mkdir(parents=True, exist_ok=False) shutil.copy(v1_lockfile_path, test_lockfile_path) @@ -4032,7 +4032,7 @@ def test_read_legacy_lockfile_and_reconcretize( # After reconcretization with the *new*, finer-grained DAG hash, there should no # longer be conflicts, and the previously conflicting specs can coexist in the # same environment. - test_path = pathlib.Path(paths.test_path) + test_path = pathlib.Path(spack.paths.test_path) lockfile_content = test_path / "data" / "legacy_env" / f"{lockfile}.lock" legacy_lockfile_path = tmp_path / ev.lockfile_name shutil.copy(lockfile_content, legacy_lockfile_path) diff --git a/lib/spack/spack/test/cmd/find.py b/lib/spack/spack/test/cmd/find.py index 037cf6c3c7c8ed..aaa8adba0c1ff4 100644 --- a/lib/spack/spack/test/cmd/find.py +++ b/lib/spack/spack/test/cmd/find.py @@ -15,6 +15,7 @@ import spack.concretize import spack.environment as ev import spack.package_base +import spack.paths import spack.repo import spack.spec import spack.store @@ -22,7 +23,6 @@ from spack.enums import InstallRecordStatus from spack.llnl.util.filesystem import working_dir from spack.main import SpackCommand -from spack.paths import locations as paths from spack.test.utilities import SpackCommandArgs from spack.util.pattern import Bunch @@ -471,7 +471,7 @@ def test_environment_with_version_range_in_compiler_doesnt_fail( @pytest.fixture def test_repo(mock_stage): with spack.repo.use_repositories( - os.path.join(paths.test_repos_path, "spack_repo", "find") + os.path.join(spack.paths.test_repos_path, "spack_repo", "find") ) as mock_packages_repo: yield mock_packages_repo diff --git a/lib/spack/spack/test/cmd/gpg.py b/lib/spack/spack/test/cmd/gpg.py index e3945ecb0275be..3bf80930bd3eb1 100644 --- a/lib/spack/spack/test/cmd/gpg.py +++ b/lib/spack/spack/test/cmd/gpg.py @@ -9,9 +9,9 @@ import spack.binary_distribution import spack.llnl.util.filesystem as fs +import spack.paths import spack.util.gpg from spack.main import SpackCommand -from spack.paths import locations as paths from spack.util.executable import ProcessError #: spack command used by tests below @@ -62,10 +62,10 @@ def test_no_gpg_in_path(tmp_path: pathlib.Path, mock_gnupghome, monkeypatch, mut def test_gpg(tmp_path: pathlib.Path, mutable_config, mock_gnupghome): # Verify a file with an empty keyring. with pytest.raises(ProcessError): - gpg("verify", os.path.join(paths.mock_gpg_data_path, "content.txt")) + gpg("verify", os.path.join(spack.paths.mock_gpg_data_path, "content.txt")) # Import the default key. - gpg("init", "--from", paths.mock_gpg_keys_path) + gpg("init", "--from", spack.paths.mock_gpg_keys_path) # List the keys. # TODO: Test the output here. @@ -73,14 +73,14 @@ def test_gpg(tmp_path: pathlib.Path, mutable_config, mock_gnupghome): gpg("list", "--signing") # Verify the file now that the key has been trusted. - gpg("verify", os.path.join(paths.mock_gpg_data_path, "content.txt")) + gpg("verify", os.path.join(spack.paths.mock_gpg_data_path, "content.txt")) # Untrust the default key. gpg("untrust", "Spack testing") # Now that the key is untrusted, verification should fail. with pytest.raises(ProcessError): - gpg("verify", os.path.join(paths.mock_gpg_data_path, "content.txt")) + gpg("verify", os.path.join(spack.paths.mock_gpg_data_path, "content.txt")) # Create a file to test signing. test_path = tmp_path / "to-sign.txt" diff --git a/lib/spack/spack/test/cmd/license.py b/lib/spack/spack/test/cmd/license.py index 8d1ee210f4433c..4ab5188e08cf4f 100644 --- a/lib/spack/spack/test/cmd/license.py +++ b/lib/spack/spack/test/cmd/license.py @@ -8,9 +8,9 @@ import pytest +import spack.paths from spack.llnl.util.filesystem import mkdirp, touch from spack.main import SpackCommand -from spack.paths import locations as paths license = SpackCommand("license") @@ -19,8 +19,8 @@ def test_list_files(): files = license("list-files").strip().split("\n") - assert all(f.startswith(paths.prefix) for f in files) - assert os.path.join(paths.bin_path, "spack") in files + assert all(f.startswith(spack.paths.prefix) for f in files) + assert os.path.join(spack.paths.bin_path, "spack") in files assert os.path.abspath(__file__) in files diff --git a/lib/spack/spack/test/cmd/list.py b/lib/spack/spack/test/cmd/list.py index 50f6d1ac41f5a6..8f1d8de5ac5eb3 100644 --- a/lib/spack/spack/test/cmd/list.py +++ b/lib/spack/spack/test/cmd/list.py @@ -8,10 +8,10 @@ import pytest import spack.cmd.list +import spack.paths import spack.repo import spack.util.git from spack.main import SpackCommand -from spack.paths import locations as paths from spack.test.conftest import RepoBuilder pytestmark = [pytest.mark.usefixtures("mock_packages")] @@ -212,8 +212,8 @@ def test_list_count(): def test_list_repos(): with spack.repo.use_repositories( - os.path.join(paths.test_repos_path, "spack_repo", "builtin_mock"), - os.path.join(paths.test_repos_path, "spack_repo", "builder_test"), + os.path.join(spack.paths.test_repos_path, "spack_repo", "builtin_mock"), + os.path.join(spack.paths.test_repos_path, "spack_repo", "builder_test"), ): total_pkgs = len(list().strip().split()) mock_pkgs = len(list("-r", "builtin_mock").strip().split()) diff --git a/lib/spack/spack/test/cmd/location.py b/lib/spack/spack/test/cmd/location.py index 3842e1c730a53f..cef2e7ac37b4d2 100644 --- a/lib/spack/spack/test/cmd/location.py +++ b/lib/spack/spack/test/cmd/location.py @@ -10,11 +10,11 @@ import spack.concretize import spack.environment as ev import spack.main +import spack.paths import spack.repo import spack.stage from spack.llnl.util.filesystem import mkdirp from spack.main import SpackCommand -from spack.paths import locations as paths # Everything here uses (or can use) the mock config and database. pytestmark = [pytest.mark.usefixtures("mutable_config", "mutable_database")] @@ -202,9 +202,9 @@ def test_location_package_dir(mock_spec): @pytest.mark.parametrize( "option,expected", [ - ("--module-dir", paths.module_path), - ("--packages", paths.mock_packages_path), - ("--spack-root", paths.prefix), + ("--module-dir", spack.paths.module_path), + ("--packages", spack.paths.mock_packages_path), + ("--spack-root", spack.paths.prefix), ], ) def test_location_paths_options(option, expected): @@ -239,8 +239,8 @@ def test_location_stages(mock_spec): def test_location_specified_repo(): """Tests spack location --repo .""" with spack.repo.use_repositories( - os.path.join(paths.test_repos_path, "spack_repo", "builtin_mock"), - os.path.join(paths.test_repos_path, "spack_repo", "builder_test"), + os.path.join(spack.paths.test_repos_path, "spack_repo", "builtin_mock"), + os.path.join(spack.paths.test_repos_path, "spack_repo", "builder_test"), ): assert location("--repo").strip() == spack.repo.PATH.get_repo("builtin_mock").root assert ( diff --git a/lib/spack/spack/test/cmd/pkg.py b/lib/spack/spack/test/cmd/pkg.py index f4a39f2b51477c..6a1f6dfd1244c1 100644 --- a/lib/spack/spack/test/cmd/pkg.py +++ b/lib/spack/spack/test/cmd/pkg.py @@ -10,11 +10,11 @@ import spack.cmd import spack.cmd.pkg import spack.main +import spack.paths import spack.repo import spack.util.executable import spack.util.file_cache from spack.llnl.util.filesystem import mkdirp, working_dir -from spack.paths import locations as paths pkg = spack.main.SpackCommand("pkg") @@ -43,7 +43,7 @@ def _builtin_mock_copy( # create spack_repo subdir (root_dir / "spack_repo").mkdir() repo_dir = root_dir / "spack_repo" / "builtin_mock" - shutil.copytree(paths.mock_packages_path, str(repo_dir)) + shutil.copytree(spack.paths.mock_packages_path, str(repo_dir)) repo_cache = spack.util.file_cache.FileCache(root_dir / "cache") mock_repo = spack.repo.Repo(str(repo_dir), cache=repo_cache) diff --git a/lib/spack/spack/test/cmd/style.py b/lib/spack/spack/test/cmd/style.py index 93ea65a56d85d6..8b78e9287773d8 100644 --- a/lib/spack/spack/test/cmd/style.py +++ b/lib/spack/spack/test/cmd/style.py @@ -13,14 +13,14 @@ import spack.cmd.style import spack.main +import spack.paths import spack.repo from spack.cmd.style import _run_import_check, changed_files from spack.llnl.util.filesystem import FileFilter, working_dir -from spack.paths import locations as paths from spack.util.executable import which #: directory with sample style files -style_data = os.path.join(paths.test_path, "data", "style") +style_data = os.path.join(spack.paths.test_path, "data", "style") style = spack.main.SpackCommand("style") @@ -47,9 +47,9 @@ def ruff_package(tmp_path: pathlib.Path): change to the ``ruff`` mock package, yields the filename, then undoes the change on cleanup. """ - repo = spack.repo.from_path(paths.mock_packages_path) + repo = spack.repo.from_path(spack.paths.mock_packages_path) filename = repo.filename_for_package_name("ruff") - rel_path = os.path.dirname(os.path.relpath(filename, paths.prefix)) + rel_path = os.path.dirname(os.path.relpath(filename, spack.paths.prefix)) tmp = tmp_path / rel_path / "ruff-ci-package.py" tmp.parent.mkdir(parents=True, exist_ok=True) tmp.touch() @@ -64,7 +64,7 @@ def ruff_package(tmp_path: pathlib.Path): @pytest.fixture def ruff_package_with_errors(scope="function"): """A ruff package with errors.""" - repo = spack.repo.from_path(paths.mock_packages_path) + repo = spack.repo.from_path(spack.paths.mock_packages_path) filename = repo.filename_for_package_name("ruff") tmp = filename + ".tmp" @@ -122,7 +122,7 @@ def test_changed_no_base(git, tmp_path: pathlib.Path, capfd): def test_changed_files_all_files(mock_packages): # it's hard to guarantee "all files", so do some sanity checks. files = { - os.path.join(paths.prefix, os.path.normpath(path)) + os.path.join(spack.paths.prefix, os.path.normpath(path)) for path in changed_files(all_files=True) } @@ -137,10 +137,10 @@ def test_changed_files_all_files(mock_packages): assert zlib_file in files # a core spack file - assert os.path.join(paths.module_path, "spec.py") in files + assert os.path.join(spack.paths.module_path, "spec.py") in files # a mock package - repo = spack.repo.from_path(paths.mock_packages_path) + repo = spack.repo.from_path(spack.paths.mock_packages_path) filename = repo.filename_for_package_name("ruff") assert filename in files @@ -148,7 +148,7 @@ def test_changed_files_all_files(mock_packages): assert __file__ in files # ensure externals are excluded - assert not any(f.startswith(paths.vendor_path) for f in files) + assert not any(f.startswith(spack.paths.vendor_path) for f in files) def test_bad_root(tmp_path: pathlib.Path): @@ -241,7 +241,7 @@ def test_external_root(external_style_root): @pytest.mark.skipif(not RUFF, reason="ruff is not installed.") def test_style(ruff_package, tmp_path: pathlib.Path): - root_relative = os.path.relpath(ruff_package, paths.prefix) + root_relative = os.path.relpath(ruff_package, spack.paths.prefix) # use a working directory to test cwd-relative paths, as tests run in # the spack prefix by default @@ -267,7 +267,7 @@ def test_style(ruff_package, tmp_path: pathlib.Path): @pytest.mark.skipif(not RUFF, reason="ruff is not installed.") def test_style_with_errors(ruff_package_with_errors): - root_relative = os.path.relpath(ruff_package_with_errors, paths.prefix) + root_relative = os.path.relpath(ruff_package_with_errors, spack.paths.prefix) output = style( "--tool", "ruff-check", "--root-relative", ruff_package_with_errors, fail_on_error=False ) @@ -325,7 +325,7 @@ def something(y: spack.util.url.Url): ... fix=False, out=output_buf, root_relative=False, - root=pathlib.Path(paths.prefix), + root=pathlib.Path(spack.paths.prefix), working_dir=pathlib.Path(root), ) output = output_buf.getvalue() @@ -348,7 +348,7 @@ def something(y: spack.util.url.Url): ... fix=True, out=output_buf, root_relative=False, - root=pathlib.Path(paths.prefix), + root=pathlib.Path(spack.paths.prefix), working_dir=pathlib.Path(root), ) output = output_buf.getvalue() @@ -366,7 +366,7 @@ def something(y: spack.util.url.Url): ... fix=True, out=output_buf, root_relative=False, - root=pathlib.Path(paths.prefix), + root=pathlib.Path(spack.paths.prefix), working_dir=pathlib.Path(root), ) output = output_buf.getvalue() @@ -410,6 +410,7 @@ def test_case_sensitive_imports(tmp_path: pathlib.Path): def test_pkg_imports(): assert ( - spack.cmd.style._module_part(pathlib.Path(paths.prefix), "spack.pkg.builtin.boost") is None + spack.cmd.style._module_part(pathlib.Path(spack.paths.prefix), "spack.pkg.builtin.boost") + is None ) - assert spack.cmd.style._module_part(pathlib.Path(paths.prefix), "spack.pkg") is None + assert spack.cmd.style._module_part(pathlib.Path(spack.paths.prefix), "spack.pkg") is None diff --git a/lib/spack/spack/test/cmd/test.py b/lib/spack/spack/test/cmd/test.py index 61858234c1d886..2389d337d828a7 100644 --- a/lib/spack/spack/test/cmd/test.py +++ b/lib/spack/spack/test/cmd/test.py @@ -13,10 +13,10 @@ import spack.concretize import spack.config import spack.install_test +import spack.paths from spack.install_test import TestStatus from spack.llnl.util.filesystem import copy_tree, working_dir from spack.main import SpackCommand -from spack.paths import locations as paths install = SpackCommand("install") spack_test = SpackCommand("test") @@ -226,7 +226,7 @@ def test_read_old_results(mock_packages, mock_test_stage): # spack install printing-package # spack test run --alias printpkg printing-package - test_data_src = os.path.join(paths.test_path, "data", "test", "test_stage") + test_data_src = os.path.join(spack.paths.test_path, "data", "test", "test_stage") # Copy the old test data into the mock stage directory copy_tree(test_data_src, mock_test_stage) diff --git a/lib/spack/spack/test/concretization/compiler_runtimes.py b/lib/spack/spack/test/concretization/compiler_runtimes.py index 2825f80c8116b3..35f46795ffa344 100644 --- a/lib/spack/spack/test/concretization/compiler_runtimes.py +++ b/lib/spack/spack/test/concretization/compiler_runtimes.py @@ -11,11 +11,11 @@ import spack.concretize import spack.config +import spack.paths import spack.repo import spack.solver.asp import spack.spec from spack.environment.environment import ViewDescriptor -from spack.paths import locations as paths from spack.solver.reuse import create_external_parser, spec_filter_from_packages_yaml from spack.solver.runtimes import external_config_with_implicit_externals from spack.version import Version @@ -42,7 +42,7 @@ def _concretize_with_reuse(*, root_str, reused_str, config): @pytest.fixture def runtime_repo(mutable_config): - repo = os.path.join(paths.test_repos_path, "spack_repo", "compiler_runtime_test") + repo = os.path.join(spack.paths.test_repos_path, "spack_repo", "compiler_runtime_test") with spack.repo.use_repositories(repo) as mock_repo: yield mock_repo diff --git a/lib/spack/spack/test/concretization/core.py b/lib/spack/spack/test/concretization/core.py index df512d0f627956..41f4241b83d2e3 100644 --- a/lib/spack/spack/test/concretization/core.py +++ b/lib/spack/spack/test/concretization/core.py @@ -28,6 +28,7 @@ import spack.hash_types as ht import spack.llnl.util.lang import spack.package_base +import spack.paths import spack.platforms import spack.platforms.test import spack.repo @@ -44,7 +45,6 @@ import spack.variant as vt from spack.externals import ExternalDependencyError from spack.installer import PackageInstaller -from spack.paths import locations as paths from spack.solver.asp import Result from spack.solver.reuse import create_external_parser, spec_filter_from_packages_yaml from spack.solver.runtimes import external_config_with_implicit_externals @@ -2608,7 +2608,9 @@ def test_select_lower_priority_package_from_repository_stack( from cli. """ # 'builtin_mock" and "duplicates_test" share a 'gmake' package - additional_repo = os.path.join(paths.test_repos_path, "spack_repo", "duplicates_test") + additional_repo = os.path.join( + spack.paths.test_repos_path, "spack_repo", "duplicates_test" + ) with spack.repo.use_repositories(additional_repo, override=False): s = spack.concretize.concretize_one(spec_str) @@ -2859,7 +2861,7 @@ def test_git_based_version_must_exist_to_use_ref(self): @pytest.fixture() def duplicates_test_repository(): - repository_path = os.path.join(paths.test_repos_path, "spack_repo", "duplicates_test") + repository_path = os.path.join(spack.paths.test_repos_path, "spack_repo", "duplicates_test") with spack.repo.use_repositories(repository_path) as mock_repo: yield mock_repo @@ -3114,7 +3116,7 @@ def test_adding_specs(self, input_specs, default_mock_concretization): @pytest.fixture() def edges_test_repository(): - repository_path = os.path.join(paths.test_repos_path, "spack_repo", "edges_test") + repository_path = os.path.join(spack.paths.test_repos_path, "spack_repo", "edges_test") with spack.repo.use_repositories(repository_path) as mock_repo: yield mock_repo diff --git a/lib/spack/spack/test/concretization/flag_mixing.py b/lib/spack/spack/test/concretization/flag_mixing.py index e5b2433b98f0e9..59e1ee6e583957 100644 --- a/lib/spack/spack/test/concretization/flag_mixing.py +++ b/lib/spack/spack/test/concretization/flag_mixing.py @@ -38,15 +38,15 @@ import spack.concretize import spack.config import spack.environment as ev +import spack.paths import spack.repo import spack.spec import spack.util.spack_yaml as syaml -from spack.paths import locations as paths @pytest.fixture def test_repo(mutable_config, monkeypatch, mock_stage): - repo_dir = pathlib.Path(paths.test_repos_path) / "spack_repo" / "flags_test" + repo_dir = pathlib.Path(spack.paths.test_repos_path) / "spack_repo" / "flags_test" with spack.repo.use_repositories(str(repo_dir)) as mock_packages_repo: yield mock_packages_repo diff --git a/lib/spack/spack/test/concretization/requirements.py b/lib/spack/spack/test/concretization/requirements.py index c19c9492775d2b..140e77b69b7595 100644 --- a/lib/spack/spack/test/concretization/requirements.py +++ b/lib/spack/spack/test/concretization/requirements.py @@ -10,6 +10,7 @@ import spack.error import spack.installer import spack.package_base +import spack.paths import spack.platforms import spack.repo import spack.solver.asp @@ -18,7 +19,6 @@ import spack.util.spack_yaml as syaml import spack.version from spack.installer import PackageInstaller -from spack.paths import locations as paths from spack.solver.asp import InternalConcretizerError, UnsatisfiableSpecError from spack.solver.reuse import create_external_parser, spec_filter_from_packages_yaml from spack.solver.runtimes import external_config_with_implicit_externals @@ -33,7 +33,7 @@ def update_packages_config(conf_str): @pytest.fixture def test_repo(mutable_config, monkeypatch, mock_stage): - repo_dir = pathlib.Path(paths.test_repos_path) / "spack_repo" / "requirements_test" + repo_dir = pathlib.Path(spack.paths.test_repos_path) / "spack_repo" / "requirements_test" with spack.repo.use_repositories(str(repo_dir)) as mock_packages_repo: yield mock_packages_repo diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 307d261d9eba82..480b538412bed2 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -35,6 +35,7 @@ import spack.llnl.util.filesystem as fs import spack.llnl.util.lock as lk import spack.package_base +import spack.paths import spack.repo import spack.spec import spack.store @@ -43,7 +44,6 @@ from spack.enums import InstallRecordStatus from spack.installer import PackageInstaller from spack.llnl.util.tty.colify import colify -from spack.paths import locations as paths from spack.schema.database_index import schema from spack.test.conftest import RepoBuilder from spack.util.executable import Executable @@ -1350,7 +1350,7 @@ def test_querying_reindexed_database_specfilev5(tmp_path: pathlib.Path): """Tests that we can query a reindexed database from before compilers as dependencies, and get appropriate results for % and similar selections. """ - test_path = pathlib.Path(paths.test_path) + test_path = pathlib.Path(spack.paths.test_path) zipfile = test_path / "data" / "database" / "index.json.v7_v8.json.gz" with gzip.open(str(zipfile), "rt", encoding="utf-8") as f: data = json.load(f) diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 367ff3754fc392..d5f9b9a3bf207b 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -14,11 +14,11 @@ import spack.concretize import spack.hash_types +import spack.paths import spack.repo import spack.util.file_cache from spack.directory_layout import DirectoryLayout, InvalidDirectoryLayoutParametersError from spack.llnl.path import path_to_os_path -from spack.paths import locations as paths from spack.spec import Spec # number of packages to test (to reduce test time) @@ -158,7 +158,7 @@ def test_handle_unknown_package(temporary_store, config, mock_packages, tmp_path layout = temporary_store.layout repo_cache = spack.util.file_cache.FileCache(tmp_path / "cache") - mock_db = spack.repo.Repo(paths.mock_packages_path, cache=repo_cache) + mock_db = spack.repo.Repo(spack.paths.mock_packages_path, cache=repo_cache) not_in_mock = set.difference( set(spack.repo.all_package_names()), set(mock_db.all_package_names()) @@ -178,7 +178,7 @@ def test_handle_unknown_package(temporary_store, config, mock_packages, tmp_path layout.create_install_directory(spec) installed_specs[spec] = layout.path_for_spec(spec) - with spack.repo.use_repositories(paths.mock_packages_path): + with spack.repo.use_repositories(spack.paths.mock_packages_path): # Now check that even without the package files, we know # enough to read a spec from the spec file. for spec, path in installed_specs.items(): diff --git a/lib/spack/spack/test/environment_modifications.py b/lib/spack/spack/test/environment_modifications.py index 06c33d9b3a9750..d3402c06d99711 100644 --- a/lib/spack/spack/test/environment_modifications.py +++ b/lib/spack/spack/test/environment_modifications.py @@ -8,8 +8,8 @@ import pytest +import spack.paths import spack.util.environment as environment -from spack.paths import locations as paths from spack.util.environment import ( AppendPath, EnvironmentModifications, @@ -21,7 +21,7 @@ is_system_path, ) -datadir = os.path.join(paths.spack_root, "lib", "spack", "spack", "test", "data") +datadir = os.path.join(spack.paths.spack_root, "lib", "spack", "spack", "test", "data") shell_extension = ".bat" if sys.platform == "win32" else ".sh" diff --git a/lib/spack/spack/test/link_paths.py b/lib/spack/spack/test/link_paths.py index aff8326abc1a82..eba591388065be 100644 --- a/lib/spack/spack/test/link_paths.py +++ b/lib/spack/spack/test/link_paths.py @@ -8,18 +8,18 @@ import pytest import spack.compilers.libraries +import spack.paths from spack.compilers.libraries import parse_non_system_link_dirs -from spack.paths import locations as paths drive = "" if sys.platform == "win32": - match = re.search(r"[A-Za-z]:", paths.test_path) + match = re.search(r"[A-Za-z]:", spack.paths.test_path) if match: drive = match.group() root = drive + os.sep #: directory with sample compiler data -datadir = os.path.join(paths.test_path, "data", "compiler_verbose_output") +datadir = os.path.join(spack.paths.test_path, "data", "compiler_verbose_output") @pytest.fixture(autouse=True) @@ -199,6 +199,6 @@ def test_obscure_parsing_rules(): # TODO: add a comment explaining why this happens if sys.platform == "win32": - paths.remove(os.path.join(root, "second", "path")) + spack.paths.remove(os.path.join(root, "second", "path")) check_link_paths("obscure-parsing-rules.txt", paths) diff --git a/lib/spack/spack/test/llnl/util/file_list.py b/lib/spack/spack/test/llnl/util/file_list.py index 5e06aa98871c83..6c5ddf4618d83d 100644 --- a/lib/spack/spack/test/llnl/util/file_list.py +++ b/lib/spack/spack/test/llnl/util/file_list.py @@ -8,8 +8,8 @@ import pytest +import spack.paths from spack.llnl.util.filesystem import HeaderList, LibraryList, find_headers, find_libraries -from spack.paths import locations as paths @pytest.fixture() @@ -240,7 +240,7 @@ def test_add(self, header_list): #: Directory where the data for the test below is stored -search_dir = os.path.join(paths.test_path, "data", "directory_search") +search_dir = os.path.join(spack.paths.test_path, "data", "directory_search") @pytest.mark.parametrize( diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index afead09b65cae3..b576fdba96446b 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -15,8 +15,8 @@ import spack.fetch_strategy import spack.package import spack.package_base +import spack.paths import spack.repo -from spack.paths import locations as paths from spack.spec import Spec from spack.util.naming import pkg_name_to_class_name from spack.version import VersionChecksumError @@ -46,17 +46,17 @@ def test_package_name(self): assert pkg_cls.name == "mpich" def test_package_filename(self): - repo = spack.repo.from_path(paths.mock_packages_path) + repo = spack.repo.from_path(spack.paths.mock_packages_path) filename = repo.filename_for_package_name("mpich") assert filename == os.path.join( - paths.mock_packages_path, "packages", "mpich", "package.py" + spack.paths.mock_packages_path, "packages", "mpich", "package.py" ) def test_nonexisting_package_filename(self): - repo = spack.repo.from_path(paths.mock_packages_path) + repo = spack.repo.from_path(spack.paths.mock_packages_path) filename = repo.filename_for_package_name("some-nonexisting-package") assert filename == os.path.join( - paths.mock_packages_path, "packages", "some_nonexisting_package", "package.py" + spack.paths.mock_packages_path, "packages", "some_nonexisting_package", "package.py" ) def test_package_class_names(self): diff --git a/lib/spack/spack/test/packaging.py b/lib/spack/spack/test/packaging.py index dd4b11aaddfa5d..d744ff2f5ffd7e 100644 --- a/lib/spack/spack/test/packaging.py +++ b/lib/spack/spack/test/packaging.py @@ -24,6 +24,7 @@ import spack.error import spack.fetch_strategy import spack.package_base +import spack.paths import spack.stage import spack.util.gpg import spack.util.url as url_util @@ -31,7 +32,6 @@ from spack.installer import PackageInstaller from spack.llnl.util import filesystem as fs from spack.llnl.util.filesystem import readlink, symlink -from spack.paths import locations as paths from spack.relocate import _macho_find_paths, relocate_links, relocate_text pytestmark = pytest.mark.not_on_windows("does not run on windows") @@ -110,7 +110,9 @@ def test_buildcache(mock_archive, tmp_path: pathlib.Path, monkeypatch, mutable_c buildcache.buildcache(parser, args) # Copy a key to the mirror to have something to download - shutil.copyfile(paths.mock_gpg_keys_path + "/external.key", mirror_path + "/external.key") + shutil.copyfile( + spack.paths.mock_gpg_keys_path + "/external.key", mirror_path + "/external.key" + ) args = parser.parse_args(["keys"]) buildcache.buildcache(parser, args) diff --git a/lib/spack/spack/test/patch.py b/lib/spack/spack/test/patch.py index cc0c2f88914c57..28318b0b742fec 100644 --- a/lib/spack/spack/test/patch.py +++ b/lib/spack/spack/test/patch.py @@ -16,12 +16,12 @@ import spack.error import spack.fetch_strategy import spack.patch +import spack.paths import spack.repo import spack.spec import spack.stage import spack.util.url as url_util from spack.llnl.util.filesystem import mkdirp, touch, working_dir -from spack.paths import locations as paths from spack.spec import Spec from spack.stage import Stage from spack.util.executable import Executable @@ -73,7 +73,7 @@ def mock_patch_stage(tmp_path_factory: pytest.TempPathFactory, monkeypatch): return mock_path -data_path = os.path.join(paths.test_path, "data", "patch") +data_path = os.path.join(spack.paths.test_path, "data", "patch") @pytest.mark.not_on_windows("Line ending conflict on Windows") diff --git a/lib/spack/spack/test/repo.py b/lib/spack/spack/test/repo.py index 2fdcea0bbc0b49..a647425bccd1d4 100644 --- a/lib/spack/spack/test/repo.py +++ b/lib/spack/spack/test/repo.py @@ -8,6 +8,7 @@ import spack.environment import spack.package_base +import spack.paths import spack.repo import spack.schema.repos import spack.spec @@ -15,7 +16,6 @@ import spack.util.file_cache import spack.util.lock import spack.util.naming -from spack.paths import locations as paths from spack.test.conftest import RepoBuilder from spack.util.naming import valid_module_name @@ -178,7 +178,7 @@ def _repo_descriptors(repos): for entry in repos: if entry == "mock": descriptors["builtin_mock"] = spack.repo.LocalRepoDescriptor( - "builtin_mock", paths.mock_packages_path + "builtin_mock", spack.paths.mock_packages_path ) if entry == "extra": repo_dir = tmp_path / "extra_mock" @@ -210,7 +210,7 @@ def test_path_computation_with_names(method_name, mock_packages_repo): def test_use_repositories_and_import(): """Tests that use_repositories changes the import search too""" - repo_dir = pathlib.Path(paths.test_repos_path) + repo_dir = pathlib.Path(spack.paths.test_repos_path) with spack.repo.use_repositories(str(repo_dir / "spack_repo" / "compiler_runtime_test")): import spack_repo.compiler_runtime_test.packages.gcc_runtime.package # type: ignore[import] # noqa: E501 @@ -225,7 +225,7 @@ class TestRepo: """ def test_creation(self, mock_test_cache): - repo = spack.repo.Repo(paths.mock_packages_path, cache=mock_test_cache) + repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) assert repo.config_file.endswith("repo.yaml") assert repo.namespace == "builtin_mock" @@ -233,7 +233,7 @@ def test_creation(self, mock_test_cache): "name,expected", [("mpi", True), ("mpich", False), ("mpileaks", False)] ) def test_is_virtual(self, name, expected, mock_test_cache): - repo = spack.repo.Repo(paths.mock_packages_path, cache=mock_test_cache) + repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) assert repo.is_virtual(name) is expected assert repo.is_virtual_safe(name) is expected @@ -266,7 +266,7 @@ def test_real_name(self, module_name, pkg_name, mock_test_cache, tmp_path: pathl @pytest.mark.parametrize("name", ["mpileaks", "7zip", "dla-future"]) def test_get(self, name, mock_test_cache): - repo = spack.repo.Repo(paths.mock_packages_path, cache=mock_test_cache) + repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) mock_spec = spack.spec.Spec(name) mock_spec._mark_concrete() pkg = repo.get(mock_spec) @@ -274,7 +274,7 @@ def test_get(self, name, mock_test_cache): @pytest.mark.parametrize("virtual_name,expected", [("mpi", ["mpich", "zmpi"])]) def test_providers(self, virtual_name, expected, mock_test_cache): - repo = spack.repo.Repo(paths.mock_packages_path, cache=mock_test_cache) + repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) provider_names = {x.name for x in repo.providers_for(virtual_name)} assert provider_names.issuperset(expected) @@ -283,14 +283,14 @@ def test_providers(self, virtual_name, expected, mock_test_cache): [("python", ["py-extension1", "python-venv"]), ("perl", ["perl-extension"])], ) def test_extensions(self, extended, expected, mock_test_cache): - repo = spack.repo.Repo(paths.mock_packages_path, cache=mock_test_cache) + repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) repo_path = spack.repo.RepoPath(repo) for instance in (repo, repo_path): provider_names = {x.name for x in instance.extensions_for(extended)} assert provider_names.issuperset(expected) def test_all_package_names(self, mock_test_cache): - repo = spack.repo.Repo(paths.mock_packages_path, cache=mock_test_cache) + repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) repo_path = spack.repo.RepoPath(repo) for instance in (repo, repo_path): @@ -302,7 +302,7 @@ def test_all_package_names(self, mock_test_cache): assert instance.is_virtual_safe(name) def test_packages_with_tags(self, mock_test_cache): - repo = spack.repo.Repo(paths.mock_packages_path, cache=mock_test_cache) + repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) repo_path = spack.repo.RepoPath(repo) for instance in (repo, repo_path): @@ -320,7 +320,7 @@ def test_creation_from_string(self, mock_test_cache): spack.repo.RepoDescriptors( { "builtin_mock": spack.repo.LocalRepoDescriptor( - "builtin_mock", paths.mock_packages_path + "builtin_mock", spack.paths.mock_packages_path ) } ), @@ -334,7 +334,7 @@ def test_get_repo(self, mock_test_cache): spack.repo.RepoDescriptors( { "builtin_mock": spack.repo.LocalRepoDescriptor( - "builtin_mock", paths.mock_packages_path + "builtin_mock", spack.paths.mock_packages_path ) } ), diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 609a50b8c3e547..813c1ef1365146 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -11,6 +11,7 @@ import spack.directives import spack.llnl.util.lang import spack.package_base +import spack.paths import spack.repo import spack.solver.asp import spack.spec @@ -21,7 +22,6 @@ from spack.enums import PropagationPolicy from spack.error import SpecError, UnsatisfiableSpecError from spack.llnl.util.tty.color import colorize -from spack.paths import locations as paths from spack.spec import ArchSpec, DependencySpec, Spec, SpecFormatSigilError, SpecFormatStringError from spack.variant import ( InvalidVariantValueError, @@ -1006,7 +1006,7 @@ def test_spec_formatting(self, default_mock_concretization): ] other_segments = [ - ("{spack_root}", paths.spack_root), + ("{spack_root}", spack.paths.spack_root), ("{spack_install}", spack.store.STORE.layout.root), ] diff --git a/lib/spack/spack/test/spec_yaml.py b/lib/spack/spack/test/spec_yaml.py index d8c459a363bff3..f6ac104dcab7dd 100644 --- a/lib/spack/spack/test/spec_yaml.py +++ b/lib/spack/spack/test/spec_yaml.py @@ -24,11 +24,11 @@ import spack.concretize import spack.config import spack.hash_types as ht +import spack.paths import spack.repo import spack.spec import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml -from spack.paths import locations as paths from spack.spec import Spec, save_dependency_specfiles from spack.test.conftest import RepoBuilder from spack.util.spack_yaml import SpackYAMLError, syaml_dict @@ -47,7 +47,7 @@ def check_json_round_trip(spec): def test_read_spec_from_signed_json(): - spec_dir = os.path.join(paths.test_path, "data", "mirrors", "signed_json") + spec_dir = os.path.join(spack.paths.test_path, "data", "mirrors", "signed_json") file_name = ( "linux-ubuntu18.04-haswell-gcc-8.4.0-" "zlib-1.2.12-g7otk5dra3hifqxej36m5qzm7uyghqgb.spec.json.sig" @@ -411,7 +411,7 @@ def test_legacy_yaml(install_mockery, mock_packages): ], ) def test_load_json_specfiles(specfile, expected_hash, reader_cls): - fullpath = os.path.join(paths.test_path, "data", specfile) + fullpath = os.path.join(spack.paths.test_path, "data", specfile) with gzip.open(fullpath, "rt", encoding="utf-8") as f: data = json.load(f) diff --git a/lib/spack/spack/test/util/compression.py b/lib/spack/spack/test/util/compression.py index 6f8ed53e5768e3..1b82191b30be52 100644 --- a/lib/spack/spack/test/util/compression.py +++ b/lib/spack/spack/test/util/compression.py @@ -12,12 +12,14 @@ import pytest import spack.llnl.url +import spack.paths from spack.llnl.util.filesystem import working_dir -from spack.paths import locations as paths from spack.util import compression from spack.util.executable import CommandNotFoundError -datadir = os.path.join(paths.spack_root, "lib", "spack", "spack", "test", "data", "compression") +datadir = os.path.join( + spack.paths.spack_root, "lib", "spack", "spack", "test", "data", "compression" +) ext_archive = { ext: f"Foo.{ext}" for ext in spack.llnl.url.ALLOWED_ARCHIVE_TYPES if "TAR" not in ext diff --git a/lib/spack/spack/test/util/package_hash.py b/lib/spack/spack/test/util/package_hash.py index 911136939b3526..58a6a3b0c85a5d 100644 --- a/lib/spack/spack/test/util/package_hash.py +++ b/lib/spack/spack/test/util/package_hash.py @@ -9,13 +9,13 @@ import spack.concretize import spack.directives_meta +import spack.paths import spack.repo import spack.util.package_hash as ph -from spack.paths import locations as paths from spack.spec import Spec from spack.util.unparse import unparse -datadir = os.path.join(paths.test_path, "data", "unparse") +datadir = os.path.join(spack.paths.test_path, "data", "unparse") def compare_sans_name(eq, spec1, spec2): diff --git a/lib/spack/spack/test/web.py b/lib/spack/spack/test/web.py index 1f5f913a813a96..5f6c6d6345e201 100644 --- a/lib/spack/spack/test/web.py +++ b/lib/spack/spack/test/web.py @@ -15,17 +15,17 @@ import spack.config import spack.llnl.util.tty as tty import spack.mirrors.mirror +import spack.paths import spack.url import spack.util.s3 import spack.util.url as url_util import spack.util.web from spack.llnl.util.filesystem import working_dir -from spack.paths import locations as paths from spack.version import Version def _create_url(relative_url): - web_data_path = os.path.join(paths.test_path, "data", "web") + web_data_path = os.path.join(spack.paths.test_path, "data", "web") return url_util.path_to_file_url(os.path.join(web_data_path, relative_url)) diff --git a/lib/spack/spack/util/gpg.py b/lib/spack/spack/util/gpg.py index f94ca296aa09fa..a2232d6c7b2e9b 100644 --- a/lib/spack/spack/util/gpg.py +++ b/lib/spack/spack/util/gpg.py @@ -10,9 +10,9 @@ import spack.error import spack.llnl.util.filesystem +import spack.paths import spack.util.executable import spack.version -from spack.paths import locations as paths #: Executable instance for "gpg", initialized lazily GPG = None @@ -59,7 +59,7 @@ def init(gnupghome=None, force=False): # Set the value of GNUPGHOME to be used in this module GNUPGHOME = gnupghome or os.getenv("SPACK_GNUPGHOME") if not GNUPGHOME: - GNUPGHOME = paths.gpg_path + GNUPGHOME = spack.paths.gpg_path # Set the executable objects for "gpg" and "gpgconf" with spack.bootstrap.ensure_bootstrap_configuration(): diff --git a/lib/spack/spack/version/git_ref_lookup.py b/lib/spack/spack/version/git_ref_lookup.py index 566d7459eac09a..9a7b7653a80eed 100644 --- a/lib/spack/spack/version/git_ref_lookup.py +++ b/lib/spack/spack/version/git_ref_lookup.py @@ -9,12 +9,12 @@ import spack.caches import spack.fetch_strategy +import spack.paths import spack.repo import spack.util.executable import spack.util.hash import spack.util.spack_json as sjson from spack.llnl.util.filesystem import mkdirp, working_dir -from spack.paths import locations as paths from .common import VersionLookupError from .lookup import AbstractRefLookup @@ -122,7 +122,7 @@ def lookup_ref(self, ref) -> Tuple[Optional[str], int]: known version prior to the commit, as well as the distance from that version to the commit in the git repo. Those values are used to compare Version objects. """ - pathlib_dest = Path(paths.user_repos_cache_path) / self.repository_uri + pathlib_dest = Path(spack.paths.user_repos_cache_path) / self.repository_uri dest = str(pathlib_dest) # prepare a cache for the repository From 1b8812c1606a664b83380a41a5fbbf23c67c3c68 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 23 May 2026 21:16:50 -0700 Subject: [PATCH 496/506] config: add old/ and xdg/ layout scheme yamls under defaults/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each scheme yaml sets config:locations:{home,data,state,cache} and (for old) the install_tree:root, environments_root, license_dir, and source_cache that diverge from the XDG layout. The xdg scheme covers the standard case; the old scheme keeps everything under $spack so a pre-1.2 clone that pulls this PR stays put. Also adds $xdg_data_home, $xdg_state_home, $xdg_cache_home path substitutions so the xdg scheme can respect XDG_*_HOME env vars without pushing that resolution back into Python. These files are not yet wired in — the include + layout_detected() helper come in a follow-up commit. Putting them in their own scheme directories now so the rest of the work has something to point at. --- etc/spack/defaults/old/config.yaml | 27 +++++++++++++++++++++++++++ etc/spack/defaults/xdg/config.yaml | 17 +++++++++++++++++ lib/spack/spack/util/path.py | 26 ++++++++++++++++---------- 3 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 etc/spack/defaults/old/config.yaml create mode 100644 etc/spack/defaults/xdg/config.yaml diff --git a/etc/spack/defaults/old/config.yaml b/etc/spack/defaults/old/config.yaml new file mode 100644 index 00000000000000..dcd937750bb0c2 --- /dev/null +++ b/etc/spack/defaults/old/config.yaml @@ -0,0 +1,27 @@ +# ------------------------------------------------------------------------- +# Legacy `$spack`-local layout defaults. +# +# Selected by `etc/spack/defaults/include.yaml` when an existing Spack +# instance has installs, environments, gpg keys, etc. under `$spack`. Keeps +# everything in those legacy locations so pulling a newer Spack into an old +# clone is non-disruptive. +# +# The four `config:locations:*` keys give the rest of the default config +# (which uses `$data_home`, `$state_home`, `$cache_home` substitutions) a +# coherent answer for old layout — but old layout doesn't actually share a +# single root, so individual path keys below override the substitutions +# directly. +# ------------------------------------------------------------------------- +config: + locations: + home: ~ + data: $spack + state: ~/.spack + cache: $spack/var + + install_tree: + root: $spack/opt/spack + + environments_root: $spack/var/spack/environments + license_dir: $spack/etc/spack/licenses + source_cache: $spack/var/cache diff --git a/etc/spack/defaults/xdg/config.yaml b/etc/spack/defaults/xdg/config.yaml new file mode 100644 index 00000000000000..c3afb545716764 --- /dev/null +++ b/etc/spack/defaults/xdg/config.yaml @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# XDG-compliant layout defaults. +# +# Selected by `etc/spack/defaults/include.yaml` when no legacy `$spack`-local +# data is detected. Sets the four `config:locations:*` keys that the rest of +# the default config uses via `$data_home`, `$state_home`, and `$cache_home` +# substitutions. +# +# Spack-specific env vars (SPACK_DATA_HOME etc.) still override these at +# runtime; XDG env vars are picked up here via the `$xdg_*_home` subs. +# ------------------------------------------------------------------------- +config: + locations: + home: ~ + data: $xdg_data_home/spack + state: $xdg_state_home/spack + cache: $xdg_cache_home/spack diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 0e119240b93b91..11b48e6ec82e61 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -64,6 +64,9 @@ def replacements(): paths = spack.paths.locations arch = architecture() + def xdg_or_default(var, default): + return os.environ.get(var) or os.path.expanduser(default) + replace = { "spack": lambda: spack.paths_base.locations.prefix, "user": lambda: get_user(), @@ -80,15 +83,16 @@ def replacements(): "env": lambda: ev.active_environment().path if ev.active_environment() else NOMATCH, "spack_short_version": lambda: spack.get_short_version(), "user_cache_path": lambda: paths.user_cache_path, - "default_install_root": lambda: paths.default_install_location, - "default_envs_root": lambda: paths.default_envs_path, - "default_license_dir": lambda: paths.default_license_dir, - "default_downloads_dir": lambda: paths.default_downloads_dir, - "default_modules_base": lambda: paths.default_modules_base, "data_home": lambda: paths.data_home, "cache_home": lambda: paths.cache_home, "state_home": lambda: paths.state_home, "spack_home": lambda: paths.spack_home, + # XDG base-dir env vars with their spec defaults. Used by the xdg + # scheme yaml so $XDG_*_HOME is respected without forcing all of + # paths.py through Python. See etc/spack/defaults/xdg/config.yaml. + "xdg_data_home": lambda: xdg_or_default("XDG_DATA_HOME", "~/.local/share"), + "xdg_state_home": lambda: xdg_or_default("XDG_STATE_HOME", "~/.local/state"), + "xdg_cache_home": lambda: xdg_or_default("XDG_CACHE_HOME", "~/.cache"), } return replace @@ -176,14 +180,16 @@ def substitute_config_variables(path): - $env The active Spack environment. - $spack The Spack instance's prefix + - $spack_home Base for spack's user-level data; defaults to ``~`` - $tempdir Default temporary directory returned by tempfile.gettempdir() - $user The current user's username - - $default_install_root Where installs go by default - - $default_envs_root Where environments are managed by default - - $data_home SPACK_DATA_HOME, XDG_DATA_HOME, or its default - - $cache_home SPACK_CACHE_HOME, XDG_CACHE_HOME, or its default - - $state_home SPACK_STATE_HOME, XDG_STATE_HOME, or its default + - $data_home SPACK_DATA_HOME, config:locations:data, XDG_DATA_HOME, or default + - $cache_home SPACK_CACHE_HOME, config:locations:cache, XDG_CACHE_HOME, or default + - $state_home SPACK_STATE_HOME, config:locations:state, XDG_STATE_HOME, or default - $user_cache_path The user cache directory (same as state_home) + - $xdg_data_home $XDG_DATA_HOME or ``~/.local/share`` (no ``/spack`` suffix) + - $xdg_state_home $XDG_STATE_HOME or ``~/.local/state`` + - $xdg_cache_home $XDG_CACHE_HOME or ``~/.cache`` - $spack_instance_id Hash that distinguishes Spack instances on the filesystem - $architecture The spack architecture triple for the current system - $arch The spack architecture triple for the current system From baa836420a439f3cf41c0d9a8b1dce66a4913a7d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 23 May 2026 21:19:30 -0700 Subject: [PATCH 497/506] paths: add detect_layout() and expose as layout_detected() in include when: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit paths.detect_layout('old' | 'xdg') reports which scheme is active. spec.eval_conditional adds it to the include `when:` clause namespace as layout_detected, so etc/spack/defaults/include.yaml can do `when: layout_detected('old')`. Honors "unilateral override": any SPACK_*_HOME (or SPACK_HOME) env var forces xdg even when legacy $spack-local data is present, so a user who sets one of those knobs doesn't end up with split-state where data goes new but envs/installs go old. At include-time the user/site/system scopes aren't loaded yet, so config:locations:* set in those scopes does not force the scheme — only env vars do at this layer. Specific values still override per-key once the higher scopes are loaded. --- lib/spack/spack/paths.py | 21 +++++++++++++++++++++ lib/spack/spack/spec.py | 6 +++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 557379e39a6897..c54c74b4ae2405 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -423,6 +423,27 @@ def detect_old_spack_layout(paths: paths_base.SpackPathsBase): return False +def detect_layout(scheme): + """True if ``scheme`` is the active layout (``"old"`` or ``"xdg"``). + + Used by ``etc/spack/defaults/include.yaml`` to choose which scheme + yaml to include. Honors "unilateral override" semantics: if the user + has set any new-style location knob (SPACK_DATA_HOME, SPACK_HOME, + config:locations:*), the xdg scheme is selected even when legacy + $spack-local data is present, so partial overrides don't produce a + split layout. + + Note: at include-time the user/site/system scopes are not yet on the + config stack, so the config:locations:* check here only sees the + _builtin defaults (which don't set them). Env vars are the practical + override mechanism at this layer. + """ + if scheme not in ("old", "xdg"): + raise ValueError(f"unknown layout scheme: {scheme!r} (expected 'old' or 'xdg')") + is_old = detect_old_spack_layout(paths_base.locations) and not new_layout_enforced() + return is_old if scheme == "old" else not is_old + + locations = SpackPaths(paths_base.locations) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index fab65cba4a46fa..aaccb7a146b519 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -5783,8 +5783,12 @@ def get_host_environment() -> Dict[str, Any]: def eval_conditional(string): """Evaluate conditional definitions using restricted variable scope.""" + import spack.paths # local import to avoid cycle at module load + valid_variables = get_host_environment() - valid_variables.update({"re": re, "env": os.environ}) + valid_variables.update( + {"re": re, "env": os.environ, "layout_detected": spack.paths.detect_layout} + ) return eval(string, valid_variables) From 7aad34a383cd9d54322367c308b0d5b57fd6ce52 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 23 May 2026 21:47:45 -0700 Subject: [PATCH 498/506] paths/config: collapse default_* properties; wire scheme includes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the layout decision live in yaml: * etc/spack/defaults/include.yaml now includes old/ or xdg/ via when: layout_detected("old"). Schemes are positioned earlier than base in the include list, so they have higher priority and can override base's $data_home-relative defaults. * etc/spack/defaults/base/config.yaml drops the $default_install_root, $default_envs_root, $default_license_dir, $default_downloads_dir substitutions. Uses $data_home/{installs,environments,licenses,...} directly. Old scheme yaml overrides each one to its $spack-local location. xdg scheme has none of those overrides — the substitutions alone produce the right paths. * config schema gains gpg_path and gpg_keys_path so they too can move out of Python and into the scheme yamls. * SpackPaths shrinks dramatically: no more default_install_location, default_envs_path, default_license_dir, default_modules_base, default_downloads_dir, _decide_old_or_new_location, resolve_a_home, Provenance enum, new_layout_enforced. The four "home" properties stay in Python because they preserve env-var > config priority for SPACK_DATA_HOME and friends. gpg_path/gpg_keys_path read config first and fall back to data_home-relative. * store.py reads install_tree:root directly (no Python fallback — config always provides it via the scheme yaml). * environment.py.env_root_path() reads config:environments_root directly (no Python default_envs_path fallback). * migrate.py recovers from any prior backup location by also probing the XDG default, regardless of active scheme. * main.py replaces the default_state_home_dot_spack flag with an inline env-var check that distinguishes "state_home is ~/.spack via the scheme default" (warn) from "user explicitly pointed an env var there" (don't warn). Note: test/paths.py is broken by these changes (tests against removed properties) — updated in the tests commit. --- etc/spack/defaults/base/config.yaml | 35 +- etc/spack/defaults/include.yaml | 14 +- etc/spack/defaults/old/config.yaml | 6 +- etc/spack/defaults/xdg/config.yaml | 2 +- lib/spack/spack/cmd/migrate.py | 11 +- lib/spack/spack/config.py | 2 - lib/spack/spack/environment/environment.py | 22 +- lib/spack/spack/main.py | 8 +- lib/spack/spack/paths.py | 375 ++++++--------------- lib/spack/spack/schema/config.py | 9 + lib/spack/spack/store.py | 7 +- lib/spack/spack/test/conftest.py | 1 - 12 files changed, 175 insertions(+), 317 deletions(-) diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index b0e5d8f626d97d..8339173ed9d642 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -13,12 +13,12 @@ config: # This is the path to the root of the Spack install tree. # You can use $spack here to refer to the root of the spack instance. install_tree: - # If set to a non-default value, this setting has precedence. - # This default value tells spack to - # * Prefer $data_home/installs - # * The old install location is $spack/opt/spack: Spack will use this old - # location if the new location is empty and the old one isn't - root: $default_install_root + # If layout_detected("old"), this is overridden to $spack/opt/spack by + # etc/spack/defaults/old/config.yaml (the scheme yaml chosen by + # defaults/include.yaml when legacy data is present in $spack). + # Otherwise $data_home resolves to ~/.local/share/spack (or + # $XDG_DATA_HOME/spack, $SPACK_DATA_HOME, etc.). + root: $data_home/installs projections: all: "{architecture.platform}-{architecture.target}/{name}-{version}-{hash}" # install_tree can include an optional padded length (int or boolean) @@ -34,8 +34,9 @@ config: template_dirs: - $spack/share/spack/templates - # Directory where licenses should be located - license_dir: $default_license_dir + # Directory where licenses should be located. + # Overridden to $spack/etc/spack/licenses by the old-layout scheme yaml. + license_dir: $data_home/licenses # Temporary locations Spack can try to use for builds. # @@ -80,16 +81,18 @@ config: # Cache directory for already downloaded source tarballs and archived # repositories. This can be purged with `spack clean --downloads`. - source_cache: $default_downloads_dir + # Overridden to $spack/var/cache by the old-layout scheme yaml. + source_cache: $data_home/downloads + # Directories Spack uses for its gpg keyring and pre-imported public keys. + # Overridden to $spack-local paths by the old-layout scheme yaml. + gpg_path: $data_home/gpg + gpg_keys_path: $data_home/gpg-keys - ## Directory where spack managed environments are created and stored - # If set to a non-default value, this setting has precedence. - # This default value tells spack to: - # * Prefer $data_home/environments - # * The old location for envs is $spack/var/spack/environments: Spack will - # use this old location if the new location is empty and the old one isn't - environments_root: $default_envs_root + + ## Directory where spack managed environments are created and stored. + # Overridden to $spack/var/spack/environments by the old-layout scheme yaml. + environments_root: $data_home/environments # Cache directory for miscellaneous files, like the package index. diff --git a/etc/spack/defaults/include.yaml b/etc/spack/defaults/include.yaml index d03fb044b8be19..beb03ffcb4ba17 100644 --- a/etc/spack/defaults/include.yaml +++ b/etc/spack/defaults/include.yaml @@ -3,5 +3,17 @@ include: - path: "${platform}" optional: true - # base packages.yaml overridable by platform-specific settings + # Layout scheme: chooses old (data lives under $spack) or xdg (data lives + # under ~/.local/share/spack etc.). Earlier entries override later ones, + # so the scheme can supersede `base` for paths that diverge between + # layouts. See lib/spack/docs/where_spack_writes_data.rst. + - path: old + when: 'layout_detected("old")' + optional: true + + - path: xdg + when: 'not layout_detected("old")' + optional: true + + # base config (lowest priority among defaults) - path: base diff --git a/etc/spack/defaults/old/config.yaml b/etc/spack/defaults/old/config.yaml index dcd937750bb0c2..272a523d60f48a 100644 --- a/etc/spack/defaults/old/config.yaml +++ b/etc/spack/defaults/old/config.yaml @@ -14,9 +14,9 @@ # ------------------------------------------------------------------------- config: locations: - home: ~ + home: "~" data: $spack - state: ~/.spack + state: "~/.spack" cache: $spack/var install_tree: @@ -25,3 +25,5 @@ config: environments_root: $spack/var/spack/environments license_dir: $spack/etc/spack/licenses source_cache: $spack/var/cache + gpg_path: $spack/opt/spack/gpg + gpg_keys_path: $spack/var/gpg diff --git a/etc/spack/defaults/xdg/config.yaml b/etc/spack/defaults/xdg/config.yaml index c3afb545716764..187bf6d8ea090a 100644 --- a/etc/spack/defaults/xdg/config.yaml +++ b/etc/spack/defaults/xdg/config.yaml @@ -11,7 +11,7 @@ # ------------------------------------------------------------------------- config: locations: - home: ~ + home: "~" data: $xdg_data_home/spack state: $xdg_state_home/spack cache: $xdg_cache_home/spack diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py index 43e8f908ba1f89..827d72fd15ae12 100644 --- a/lib/spack/spack/cmd/migrate.py +++ b/lib/spack/spack/cmd/migrate.py @@ -55,11 +55,14 @@ def restore(args: argparse.Namespace) -> None: """Restore ~/.spack from backup location.""" old_location = os.path.expanduser("~/.spack") - # Check both the current backup location and the default one + # Check both the current backup location and the XDG default. The XDG + # default is checked even when the active scheme is "old", because the + # backup may have been written by a previous xdg-scheme run before the + # user reverted (e.g. by `git pull`-ing an older Spack). backup_locations = [spack.paths.dotspack_backup] - default_backup = os.path.join(spack.paths.default_data_home, "dotspack_backup") - if default_backup != spack.paths.dotspack_backup: - backup_locations.append(default_backup) + xdg_default_backup = os.path.expanduser("~/.local/share/spack/dotspack_backup") + if xdg_default_backup != spack.paths.dotspack_backup: + backup_locations.append(xdg_default_backup) # Find which backup location exists backup_location = None diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 5b51fe5ab73f7f..067c6be7436c93 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -146,8 +146,6 @@ "cache_home", "data_home", "spack_home", - "default_envs_root", - "default_install_root", "user_cache_path", ) _CVARS_RE = "|".join(CONFIGURABLE_VARS) diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index c24947dffbdb24..7bef6fdd31692e 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -39,7 +39,6 @@ import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as clr import spack.package_base -import spack.paths import spack.repo import spack.schema.env import spack.spec @@ -95,22 +94,13 @@ env_subdir_name = ".spack-env" -#: default path where environments are stored in the spack tree -_default_env_path = None - - -def default_env_path(): - global _default_env_path - if not _default_env_path: - _default_env_path = spack.paths.default_envs_path - return _default_env_path - - def env_root_path() -> str: - """Override default root path if the user specified it""" - return spack.util.path.canonicalize_path( - spack.config.get("config:environments_root", default=default_env_path()) - ) + """Resolve config:environments_root with substitutions and ~ expansion. + + Always set by the active layout scheme yaml + (etc/spack/defaults/{old,xdg}/config.yaml via base). + """ + return spack.util.path.canonicalize_path(spack.config.get("config:environments_root")) def environment_name(path: Union[str, pathlib.Path]) -> str: diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 0c317af4069a06..778bc020bd7e34 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -956,7 +956,13 @@ def uses_old_dotspack(path): explicit_path = False if uses_old_dotspack(spack.paths.locations.state_home): - if spack.paths.locations.default_state_home_dot_spack: + # state_home is ~/.spack: distinguish "default from the old-layout + # scheme yaml" (warn) from "user explicitly pointed an env var + # there" (don't warn). + env_override = any( + v in os.environ for v in ("SPACK_USER_CACHE_PATH", "SPACK_STATE_HOME", "SPACK_HOME") + ) + if not env_override: reasons.append("User cache path fallback") else: tty.debug( diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index c54c74b4ae2405..58eb84b9a7a621 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -16,7 +16,6 @@ import types as _types from contextlib import contextmanager from enum import Enum -from functools import partial import spack.config as config import spack.paths_base as paths_base @@ -33,31 +32,6 @@ def dir_is_occupied(x, except_for=None): return False -class Provenance(Enum): - # Used entirely inside this module, for recording configuration - # or environment options that the user set in order to influence - # the location of data that used to live in $spack and following - # #47615 now lives outside of it - - SPACK_ENV = 1 # SPACK_x_HOME - SPACK_HOME_ENV = 2 # SPACK_HOME - CONFIG_VAR = 3 # config:locations:x - CONFIG_HOME_VAR = 4 # config:locations:home - XDG_VAR = 5 # XDG_x_HOME - NOTHING_SET = 6 # None of the above are set - - def unilateral_override(self): - # The following mechanisms for indicating user preference - # override the existence of data stored in its old location - # in $spack prior to #47615 - return self in { - Provenance.SPACK_ENV, - Provenance.SPACK_HOME_ENV, - Provenance.CONFIG_VAR, - Provenance.CONFIG_HOME_VAR, - } - - class XDG_vars(Enum): state_home = "XDG_STATE_HOME" data_home = "XDG_DATA_HOME" @@ -73,7 +47,8 @@ class Spack_vars(Enum): @classmethod def new_layout(cls): - # Exclude SPACK_USER_CACHE_PATH + # Exclude SPACK_USER_CACHE_PATH (legacy env var, doesn't signal new + # layout preference the way the others do). return [ Spack_vars.state_home, Spack_vars.data_home, @@ -82,18 +57,40 @@ def new_layout(cls): ] -# This is for tests that want to clean the environment of XDG_ variables that -# affect spack behavior. Note that this will not influence install_test.py's -# view of config:test_stage +# Used by tests to scrub the environment of XDG_ variables that would +# affect Spack's path resolution. Does not influence config:test_stage. def _unset_path_vars(env): for env_var in itertools.chain(XDG_vars, Spack_vars): env.pop(env_var.value, None) +# Relative paths under SPACK_HOME used when SPACK_HOME or +# config:locations:home is set without explicit data/state/cache homes. +_RELATIVE_STATE = os.path.join(".local", "state") +_RELATIVE_DATA = os.path.join(".local", "share") +_RELATIVE_CACHE = ".cache" + + class SpackPaths: - relative_state_home = os.path.join(".local", "state") - relative_data_home = os.path.join(".local", "share") - relative_cache_home = ".cache" + """Per-instance Spack path resolution. + + The four "home" properties (``state_home``, ``data_home``, + ``cache_home``, ``spack_home``) resolve to whatever the active layout + scheme yaml provides via ``config:locations:*`` — with two + higher-priority overrides preserved in Python: + + 1. ``SPACK_x_HOME`` env var (highest) + 2. ``SPACK_HOME`` env var with the XDG-style subpath appended + 3. ``config:locations:x`` (from scheme yaml or higher scopes) + 4. ``config:locations:home`` with subpath (rarely needed when + scheme yaml is loaded, but kept for completeness) + + Everything else that used to live in this module as ``default_*`` + properties (default install root, envs root, license dir, gpg paths, + download cache) now comes directly from config, with the active + scheme yaml supplying the defaults. See + ``etc/spack/defaults/{old,xdg}/config.yaml``. + """ def __init__(self, base): self.base = base @@ -102,89 +99,73 @@ def __init__(self, base): self._data_home = None self._cache_home = None - self.default_data_home = os.path.join( - os.path.expanduser("~"), SpackPaths.relative_data_home, "spack" - ) - self.old_layout_detected = detect_old_spack_layout(base) - self._new_layout_enforced = None - @property - def new_layout_enforced(self): - if self._new_layout_enforced is None: - self._new_layout_enforced = new_layout_enforced() - return self._new_layout_enforced + def _resolve_home(self, spack_var, config_var, home_rel): + """Resolve one of state/data/cache home with env-var precedence. + + See SpackPaths docstring for the precedence order. + """ + disable_env = config.get("config:locations:disable_env", False) + + if not disable_env and spack_var in os.environ: + return os.path.expanduser(os.environ[spack_var]) + + if not disable_env and "SPACK_HOME" in os.environ: + return os.path.join(os.path.expanduser(os.environ["SPACK_HOME"]), home_rel, "spack") + + cfg = config.get(f"config:locations:{config_var}", None) + if cfg: + import spack.util.path + + return spack.util.path.canonicalize_path(cfg) + + cfg_home = config.get("config:locations:home", None) + if cfg_home: + import spack.util.path + + return os.path.join(spack.util.path.canonicalize_path(cfg_home), home_rel, "spack") + + # Final fallback. Shouldn't be hit when the scheme yamls are + # loaded — they always set config:locations:*. + return os.path.join(os.path.expanduser("~"), home_rel, "spack") @property def state_home(self): if not self._state_home: - state_home, _ = self.resolve_a_home( - ["SPACK_STATE_HOME", "SPACK_USER_CACHE_PATH"], - "state", - SpackPaths.relative_state_home, - "XDG_STATE_HOME", - ) - - self.default_state_home_dot_spack = False - - def cfg_state_home(): - return config.get("config:locations:home", None) or config.get( - "config:locations:state", None - ) - - def env_state_home(): - disable_env = config.get("config:locations:disable_env", False) - return not disable_env and any( - x in os.environ - for x in ["SPACK_USER_CACHE_PATH", "SPACK_STATE_HOME", "SPACK_HOME"] - ) - - if env_state_home() or cfg_state_home(): - self._state_home = state_home - elif dir_is_occupied(state_home): - self._state_home = state_home - elif dir_is_occupied(self.base.old_default_dot_spack): - self._state_home = self.base.old_default_dot_spack - self.default_state_home_dot_spack = True + # SPACK_USER_CACHE_PATH is a legacy alias for SPACK_STATE_HOME. + disable_env = config.get("config:locations:disable_env", False) + if not disable_env and "SPACK_USER_CACHE_PATH" in os.environ: + self._state_home = os.path.expanduser(os.environ["SPACK_USER_CACHE_PATH"]) else: - self._state_home = state_home - + self._state_home = self._resolve_home("SPACK_STATE_HOME", "state", _RELATIVE_STATE) return self._state_home @property def cache_home(self): if not self._cache_home: - self._cache_home, _ = self.resolve_a_home( - "SPACK_CACHE_HOME", "cache", SpackPaths.relative_cache_home, "XDG_CACHE_HOME" - ) + self._cache_home = self._resolve_home("SPACK_CACHE_HOME", "cache", _RELATIVE_CACHE) return self._cache_home @property def data_home(self): if not self._data_home: - self._data_home, self._data_home_provenance = self.resolve_a_home( - "SPACK_DATA_HOME", "data", SpackPaths.relative_data_home, "XDG_DATA_HOME" - ) + self._data_home = self._resolve_home("SPACK_DATA_HOME", "data", _RELATIVE_DATA) return self._data_home - @property - def data_home_provenance(self): - if not self._data_home_provenance: - self._data_home, self._data_home_provenance = self.resolve_a_home( - "SPACK_DATA_HOME", "data", SpackPaths.relative_data_home, "XDG_DATA_HOME" - ) - return self._data_home_provenance - @property def spack_home(self): disable_env = config.get("config:locations:disable_env", False) - spack_home_env = os.environ.get("SPACK_HOME", None) - if not disable_env and spack_home_env: - return spack_home_env + if not disable_env: + env_home = os.environ.get("SPACK_HOME") + if env_home: + return os.path.expanduser(env_home) - spack_home_cfg = config.get("config:locations:home", None) - if spack_home_cfg: - return spack_home_cfg + cfg_home = config.get("config:locations:home", None) + if cfg_home: + import spack.util.path + + return spack.util.path.canonicalize_path(cfg_home) return os.path.expanduser("~") @@ -192,45 +173,6 @@ def spack_home(self): def user_cache_path(self): return self.state_home - @property - def default_install_location(self): - return self._decide_old_or_new_location( - self.base.old_install_path, - os.path.join(self.data_home, "installs"), - os.path.join(self.default_data_home, "installs"), - self.data_home_provenance, - ) - - @property - def default_envs_path(self): - return self._decide_old_or_new_location( - self.base.old_envs_path, - os.path.join(self.data_home, "environments"), - os.path.join(self.default_data_home, "environments"), - self.data_home_provenance, - ) - - @property - def default_license_dir(self): - if self.old_layout_detected: - return self.base.old_licenses_path - else: - return os.path.join(self.data_home, "licenses") - - @property - def default_modules_base(self): - if self.old_layout_detected: - return self.base.share_path - else: - return os.path.join(self.data_home) - - @property - def default_downloads_dir(self): - if self.old_layout_detected: - return self.base.old_fetch_cache_path - else: - return os.path.join(self.data_home, "downloads") - @property def reports_path(self): #: junit, cdash, etc. reports about builds @@ -270,26 +212,28 @@ def package_repos_path(self): @property def dotspack_backup(self): - #: backup location for old ~/.spack directory during migration + #: backup location for old ~/.spack directory during migration. + # Lives under data_home regardless of scheme — the backup is a + # one-time migration artifact, not legacy state to preserve. return os.path.join(self.data_home, "dotspack_backup") @property def gpg_path(self): - return self._decide_old_or_new_location( - self.base.old_gpg_path, - os.path.join(self.data_home, "gpg"), - os.path.join(self.default_data_home, "gpg"), - self.data_home_provenance, - ) + cfg = config.get("config:gpg_path", None) + if cfg: + import spack.util.path + + return spack.util.path.canonicalize_path(cfg) + return os.path.join(self.data_home, "gpg") @property def gpg_keys_path(self): - return self._decide_old_or_new_location( - self.base.old_gpg_keys_path, - os.path.join(self.data_home, "gpg-keys"), - os.path.join(self.default_data_home, "gpg-keys"), - self.data_home_provenance, - ) + cfg = config.get("config:gpg_keys_path", None) + if cfg: + import spack.util.path + + return spack.util.path.canonicalize_path(cfg) + return os.path.join(self.data_home, "gpg-keys") def __getattr__(self, name): # Things that aren't sensitive to import cycles can import the @@ -300,111 +244,6 @@ def __getattr__(self, name): raise AttributeError(name) return getattr(base, name) - def resolve_a_home(self, spack_vars, config_var, home_rel, xdg_var): - """ - Files stored by spack are split into state, data, and cache components. - Each of these categories has the same fall-through/prioritization path, - established by this function: - - 1. ``SPACK_x_HOME``: for example if the ``SPACK_DATA_HOME`` env var is - set, it has the highest precedence. - 2. If the ``SPACK_HOME`` env variable is set, it can collect all of these - components together - 3. ``config:locations:x`` - 4. ``config:locations:home`` - 5. ``XDG_x_HOME``: e.g. if the ``XDG_DATA_HOME`` env var is set - 6. In the user's home directory, in the XDG default location for that - component. - - Note that configuration settings for specific data (e.g. - ``config:install_tree:root`` for where installs are placed) will take - precedence over any of this. - - Args: - spack_vars: spack-specific environment variables that indicate the - component location. Can be a list or a single variable. If this - is a list, earlier elements have precedence. - config_var: the spack config variable that indicates the component - location. - home_rel: for $SPACK_HOME and config:locations:home, this relative - path is appended to the result to get the component location. - xdg_var: the XDG-based environment variable that indicates the - component location. - """ - disable_env = config.get("config:locations:disable_env", False) - - append_rel = lambda base, rel: str(pathlib.Path(base) / (rel or "")) - - def cfg_check(path, provenance, rel=None): - found = config.get(path, None) - if found: - import spack.util.path - - found = spack.util.path.canonicalize_path(found) - return append_rel(found, rel), provenance - - spack_cfg_check = partial( - cfg_check, f"config:locations:{config_var}", Provenance.CONFIG_VAR - ) - spack_home_cfg_check = partial( - cfg_check, - "config:locations:home", - Provenance.CONFIG_HOME_VAR, - rel=os.path.join(home_rel, "spack"), - ) - - def env_check(env_vars, provenance, rel=None): - if disable_env: - return - - for v in env_vars: - if v in os.environ: - return append_rel(os.environ[v], rel), provenance - - spack_vars = [spack_vars] if isinstance(spack_vars, str) else spack_vars - spack_env_check = partial(env_check, spack_vars, Provenance.SPACK_ENV) - spack_home_env_check = partial( - env_check, - ["SPACK_HOME"], - Provenance.SPACK_HOME_ENV, - rel=os.path.join(home_rel, "spack"), - ) - xdg_env_check = partial(env_check, [xdg_var], Provenance.XDG_VAR, rel="spack") - - for check in [ - spack_env_check, - spack_home_env_check, - spack_cfg_check, - spack_home_cfg_check, - xdg_env_check, - ]: - possible_resolution = check() - if possible_resolution: - path, provenance = possible_resolution - return os.path.expanduser(path), provenance - - return os.path.join(os.path.expanduser("~"), home_rel, "spack"), Provenance.NOTHING_SET - - def _decide_old_or_new_location( - self, old_location, new_location, default_new_location, provenance - ): - if self.new_layout_enforced: - return new_location - if self.old_layout_detected: - return old_location - else: - return new_location - - -def new_layout_enforced(): - cfg = config.get("config:locations", {}) - for x in ["home", "data", "state", "cache"]: - if cfg.get(x, None): - return True - - use_env = not bool(cfg.get("disable_env", False)) - return use_env and any(x.value in os.environ for x in Spack_vars.new_layout()) - def detect_old_spack_layout(paths: paths_base.SpackPathsBase): checks = [ @@ -427,20 +266,22 @@ def detect_layout(scheme): """True if ``scheme`` is the active layout (``"old"`` or ``"xdg"``). Used by ``etc/spack/defaults/include.yaml`` to choose which scheme - yaml to include. Honors "unilateral override" semantics: if the user - has set any new-style location knob (SPACK_DATA_HOME, SPACK_HOME, - config:locations:*), the xdg scheme is selected even when legacy - $spack-local data is present, so partial overrides don't produce a - split layout. - - Note: at include-time the user/site/system scopes are not yet on the - config stack, so the config:locations:* check here only sees the - _builtin defaults (which don't set them). Env vars are the practical - override mechanism at this layer. + yaml to include. Honors "unilateral override": if the user has set + any new-style location env var (SPACK_DATA_HOME, SPACK_STATE_HOME, + SPACK_CACHE_HOME, SPACK_HOME), xdg is selected even when legacy + $spack-local data is present. + + Cannot call ``config.get(...)`` here: this runs during config + initialization (an include's ``when:`` is evaluated before the scope + is pushed), so reading config would recurse into the singleton init. + Env-var + filesystem probes only. config:locations:* set in + user/site/system scopes still overrides specific values once those + scopes are loaded. """ if scheme not in ("old", "xdg"): raise ValueError(f"unknown layout scheme: {scheme!r} (expected 'old' or 'xdg')") - is_old = detect_old_spack_layout(paths_base.locations) and not new_layout_enforced() + env_forces_new = any(v.value in os.environ for v in Spack_vars.new_layout()) + is_old = detect_old_spack_layout(paths_base.locations) and not env_forces_new return is_old if scheme == "old" else not is_old @@ -448,20 +289,17 @@ def detect_layout(scheme): def freeze(): - """ - Run this first-thing when starting a spack child process that is - used for builds. Resolve all variables that are XDG-dependent, in - case the build sets those XDG variables. + """Snapshot resolved homes for a child build process. - Note that for this reason, the private variables like _data_home - are important for more than just caching. + Builds may set their own XDG_* env vars, which would otherwise change + Spack's path resolution mid-build. Call this in the parent, ship the + dict to the child, and call ``restore()`` there. """ return { "state_home": locations.state_home, "data_home": locations.data_home, "cache_home": locations.cache_home, "old_layout_detected": locations.old_layout_detected, - "new_layout_enforced": new_layout_enforced(), } @@ -470,7 +308,6 @@ def restore(bundled_state): locations._data_home = bundled_state["data_home"] locations._cache_home = bundled_state["cache_home"] locations.old_layout_detected = bundled_state["old_layout_detected"] - locations._new_layout_enforced = bundled_state["new_layout_enforced"] # Module shim: lets callers keep using `spack.paths.X` for any attribute on diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py index 929b92eeb72808..21853b408576df 100644 --- a/lib/spack/spack/schema/config.py +++ b/lib/spack/spack/schema/config.py @@ -77,6 +77,10 @@ }, "locations": { "type": "object", + "description": "Roots for Spack's user-level data. Used by " + "$data_home/$state_home/$cache_home/$spack_home substitutions. " + "Defaults come from the active layout scheme yaml " + "(etc/spack/defaults/{old,xdg}/config.yaml).", "properties": { "home": {"type": "string"}, "data": {"type": "string"}, @@ -86,6 +90,11 @@ }, "additionalProperties": False, }, + "gpg_path": {"type": "string", "description": "Directory holding Spack's gpg keyring"}, + "gpg_keys_path": { + "type": "string", + "description": "Directory of pre-imported public keys", + }, "install_hash_length": { "type": "integer", "minimum": 1, diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 56e8c53abcd5a1..046ad70e9bc38e 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -82,10 +82,9 @@ def parse_install_tree(config_dict: dict) -> Tuple[str, str, Dict[str, str]]: projections = {"all": all_projection} else: - unpadded_root = install_tree.get("root", None) - if not unpadded_root: - unpadded_root = spack.paths.default_install_location - + # config:install_tree:root is always set by the active layout + # scheme yaml (etc/spack/defaults/{old,xdg}/config.yaml via base). + unpadded_root = install_tree["root"] unpadded_root = spack.util.path.canonicalize_path(unpadded_root) padded_length = install_tree.get("padded_length", False) diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index bf0a22c98fbe92..8403e34946cf97 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -1979,7 +1979,6 @@ def mutable_mock_env_path(tmp_path: Path, mutable_config, monkeypatch): """Fixture for mocking the internal spack environments directory.""" mock_path = tmp_path / "mock-env-path" mutable_config.set("config:environments_root", str(mock_path)) - monkeypatch.setattr(ev.environment, "default_env_path", LazyValue(str(mock_path))) return mock_path From 384719e0ff824290176768c7290636d824017ed5 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 23 May 2026 21:54:28 -0700 Subject: [PATCH 499/506] debug: add `spack debug paths` to show how each Spack path was resolved MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For each Spack data location, prints the resolved value and the source that produced it: which env var (SPACK_DATA_HOME, etc.), which config key (config:locations:data, config:install_tree:root, ...), and which config scope set it (defaults:old, defaults:xdg, defaults:base, user, site, etc.). Also surfaces the active layout scheme up-front and explains why it was chosen — including the "old-layout markers exist but an SPACK_*_HOME env var forced xdg" case that used to live in the Provenance enum. Intended as the breadcrumb users follow when paths land somewhere unexpected, in place of the per-command warnings the previous PR emitted. Regenerates shell completion to include the new subcommand. --- lib/spack/spack/cmd/debug.py | 103 ++++++++++++++++++++++++++++++ share/spack/spack-completion.bash | 6 +- share/spack/spack-completion.fish | 66 ++++++++++--------- 3 files changed, 144 insertions(+), 31 deletions(-) diff --git a/lib/spack/spack/cmd/debug.py b/lib/spack/spack/cmd/debug.py index d7e711631d202c..72a8f410731923 100644 --- a/lib/spack/spack/cmd/debug.py +++ b/lib/spack/spack/cmd/debug.py @@ -10,10 +10,12 @@ import spack import spack.config +import spack.paths import spack.platforms import spack.repo import spack.spec import spack.util.git +import spack.util.path description = "debugging commands for troubleshooting Spack" section = "developer" @@ -23,6 +25,11 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="debug_command") sp.add_parser("report", help="print information useful for bug reports") + sp.add_parser( + "paths", + help="show how each Spack data path was resolved (active layout scheme, " + "config keys, env-var overrides)", + ) def _format_repo_info(source, commit): @@ -88,6 +95,102 @@ def report(args): print("* **Platform:**", architecture) +def _row(label, value, source): + return f" {label:<22} {value}\n {source}" + + +def _resolved_home(spack_var, config_var): + """Return (value, source) for state/data/cache home.""" + disable_env = spack.config.get("config:locations:disable_env", False) + if not disable_env and spack_var in os.environ: + return os.path.expanduser(os.environ[spack_var]), f"env: {spack_var}" + if not disable_env and "SPACK_HOME" in os.environ: + # The actual value is computed by SpackPaths._resolve_home; + # we just report SPACK_HOME as the source. + return getattr(spack.paths.locations, f"{config_var}_home"), "env: SPACK_HOME" + cfg = spack.config.get(f"config:locations:{config_var}", None) + if cfg: + scope = _scope_that_set(f"config:locations:{config_var}") + return spack.util.path.canonicalize_path(cfg), f"config:locations:{config_var} ({scope})" + return getattr(spack.paths.locations, f"{config_var}_home"), "default (no source set)" + + +def _scope_that_set(config_path): + """Walk scopes high-to-low and return the name of the first one with this key.""" + section, _, key = config_path.partition(":") + for scope in reversed(list(spack.config.CONFIG.scopes.values())): + data = scope.get_section(section) or {} + # Walk nested keys. + node = data.get(section, {}) + for part in key.split(":") if key else (): + if not isinstance(node, dict) or part not in node: + node = None + break + node = node[part] + if node not in (None, {}): + return f"scope: {scope.name}" + return "scope: (not set)" + + +def paths(args): + p = spack.paths.locations + + # Active layout scheme + if p.old_layout_detected: + env_forces_new = any( + v in os.environ + for v in ("SPACK_DATA_HOME", "SPACK_STATE_HOME", "SPACK_CACHE_HOME", "SPACK_HOME") + ) + if env_forces_new: + scheme = "xdg (forced by SPACK_*_HOME env var; old-layout markers also present)" + else: + scheme = "old ($spack-local layout markers detected)" + else: + scheme = "xdg (no legacy $spack-local data)" + print(f"layout scheme: {scheme}\n") + + print("homes:") + for spack_var, cfg_var in ( + ("SPACK_DATA_HOME", "data"), + ("SPACK_STATE_HOME", "state"), + ("SPACK_CACHE_HOME", "cache"), + ): + value, source = _resolved_home(spack_var, cfg_var) + print(_row(f"${cfg_var}_home", value, source)) + print(_row("$spack_home", p.spack_home, _spack_home_source())) + print() + + # Paths that are read out of config (with substitutions resolved). + print("config-driven paths:") + for key in ( + "config:install_tree:root", + "config:environments_root", + "config:license_dir", + "config:source_cache", + "config:misc_cache", + "config:test_stage", + "config:gpg_path", + "config:gpg_keys_path", + ): + raw = spack.config.get(key, None) + if raw is None: + print(_row(key, "(unset)", "")) + continue + resolved = spack.util.path.canonicalize_path(raw) + scope = _scope_that_set(key) + print(_row(key, resolved, f"{raw} [{scope}]")) + + +def _spack_home_source(): + if "SPACK_HOME" in os.environ and not spack.config.get("config:locations:disable_env", False): + return "env: SPACK_HOME" + if spack.config.get("config:locations:home", None): + return f"config:locations:home ({_scope_that_set('config:locations:home')})" + return "default (~)" + + def debug(parser, args): if args.debug_command == "report": report(args) + elif args.debug_command == "paths": + paths(args) diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 0b004f71bf841c..392650a9cb6d69 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -961,7 +961,7 @@ _spack_debug() { then SPACK_COMPREPLY="-h --help" else - SPACK_COMPREPLY="report" + SPACK_COMPREPLY="report paths" fi } @@ -969,6 +969,10 @@ _spack_debug_report() { SPACK_COMPREPLY="-h --help" } +_spack_debug_paths() { + SPACK_COMPREPLY="-h --help" +} + _spack_deconcretize() { if $list_options then diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 4042869d71a9f9..90f14d28a1dc71 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -601,7 +601,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_enable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap enable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap disable @@ -609,7 +609,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_disable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap disable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap reset @@ -624,14 +624,14 @@ set -g __fish_spack_optspecs_spack_bootstrap_root h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap root' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap list set -g __fish_spack_optspecs_spack_bootstrap_list h/help scope= complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap add @@ -640,7 +640,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap add' -f -a '(__ complete -c spack -n '__fish_spack_using_command_pos 1 bootstrap add' -f -a '(__fish_spack_environments)' complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -f -a trust complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -d 'enable the source immediately upon addition' @@ -826,7 +826,7 @@ complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirro complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -d 'override any configured mirrors with this mirror URL' complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -f -a output_file complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -d 'file where rebuild info should be written' -complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -d 'configuration scope containing mirrors to check' # spack buildcache download @@ -1136,7 +1136,7 @@ set -g __fish_spack_optspecs_spack_compiler_find h/help scope= j/jobs= complete -c spack -n '__fish_spack_using_command compiler find' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler find' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1146,7 +1146,7 @@ set -g __fish_spack_optspecs_spack_compiler_add h/help scope= j/jobs= complete -c spack -n '__fish_spack_using_command compiler add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1158,7 +1158,7 @@ complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help - complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -d 'configuration scope to modify' # spack compiler rm @@ -1168,14 +1168,14 @@ complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -d 'configuration scope to modify' # spack compiler list set -g __fish_spack_optspecs_spack_compiler_list h/help scope= remote complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler list' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler list' -l remote -d 'list also compilers from registered buildcaches' @@ -1184,7 +1184,7 @@ complete -c spack -n '__fish_spack_using_command compiler list' -l remote -d 'li set -g __fish_spack_optspecs_spack_compiler_ls h/help scope= remote complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler ls' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler ls' -l remote -d 'list also compilers from registered buildcaches' @@ -1194,7 +1194,7 @@ set -g __fish_spack_optspecs_spack_compiler_info h/help scope= remote complete -c spack -n '__fish_spack_using_command_pos 0 compiler info' -f -a '(__fish_spack_installed_compilers)' complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler info' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler info' -l remote -d 'list also compilers from registered buildcaches' @@ -1203,7 +1203,7 @@ complete -c spack -n '__fish_spack_using_command compiler info' -l remote -d 'li set -g __fish_spack_optspecs_spack_compilers h/help scope= remote complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command compilers' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compilers' -l remote -d 'list also compilers from registered buildcaches' @@ -1270,7 +1270,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a update -d ' complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a revert -d 'revert configuration files to their state before update' complete -c spack -n '__fish_spack_using_command config' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify' # spack config get @@ -1410,6 +1410,7 @@ complete -c spack -n '__fish_spack_using_command create' -s b -l batch -d 'don'" # spack debug set -g __fish_spack_optspecs_spack_debug h/help complete -c spack -n '__fish_spack_using_command_pos 0 debug' -f -a report -d 'print information useful for bug reports' +complete -c spack -n '__fish_spack_using_command_pos 0 debug' -f -a paths -d 'show how each Spack data path was resolved (active layout scheme, config keys, env-var overrides)' complete -c spack -n '__fish_spack_using_command debug' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command debug' -s h -l help -d 'show this help message and exit' @@ -1418,6 +1419,11 @@ set -g __fish_spack_optspecs_spack_debug_report h/help complete -c spack -n '__fish_spack_using_command debug report' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command debug report' -s h -l help -d 'show this help message and exit' +# spack debug paths +set -g __fish_spack_optspecs_spack_debug_paths h/help +complete -c spack -n '__fish_spack_using_command debug paths' -s h -l help -f -a help +complete -c spack -n '__fish_spack_using_command debug paths' -s h -l help -d 'show this help message and exit' + # spack deconcretize set -g __fish_spack_optspecs_spack_deconcretize h/help root y/yes-to-all a/all complete -c spack -n '__fish_spack_using_command_pos_remainder 0 deconcretize' -f -k -a '(__fish_spack_specs)' @@ -1830,7 +1836,7 @@ complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -f complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -d 'packages to exclude from search' complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -f -a path complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -d 'one or more alternative search paths for finding externals' -complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command external find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command external find' -l all -f -a all complete -c spack -n '__fish_spack_using_command external find' -l all -d 'search for all packages that Spack knows about' @@ -2441,7 +2447,7 @@ set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsig complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use ``--type binary --type source`` (default)' @@ -2477,7 +2483,7 @@ set -g __fish_spack_optspecs_spack_mirror_remove h/help scope= all-scopes complete -c spack -n '__fish_spack_using_command_pos 0 mirror remove' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror remove' -l all-scopes -f -a all_scopes complete -c spack -n '__fish_spack_using_command mirror remove' -l all-scopes -d 'remove from all config scopes (default: highest scope with matching mirror)' @@ -2487,7 +2493,7 @@ set -g __fish_spack_optspecs_spack_mirror_rm h/help scope= all-scopes complete -c spack -n '__fish_spack_using_command_pos 0 mirror rm' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror rm' -l all-scopes -f -a all_scopes complete -c spack -n '__fish_spack_using_command mirror rm' -l all-scopes -d 'remove from all config scopes (default: highest scope with matching mirror)' @@ -2501,7 +2507,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -f -a p complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -d 'set only the URL used for uploading' complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -f -a fetch complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -d 'set only the URL used for downloading' -complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2543,7 +2549,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a s complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache' complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed complete -c spack -n '__fish_spack_using_command mirror set' -l signed -d 'require signing and signature verification when pushing and installing from this build cache' -complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2568,14 +2574,14 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password-var set -g __fish_spack_optspecs_spack_mirror_list h/help scope= complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'configuration scope to read from' # spack mirror ls set -g __fish_spack_optspecs_spack_mirror_ls h/help scope= complete -c spack -n '__fish_spack_using_command mirror ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -d 'configuration scope to read from' # spack module @@ -2889,7 +2895,7 @@ complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirecto set -g __fish_spack_optspecs_spack_repo_list h/help scope= names namespaces json complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command repo list' -l names -f -a names complete -c spack -n '__fish_spack_using_command repo list' -l names -d 'show configuration names only' @@ -2902,7 +2908,7 @@ complete -c spack -n '__fish_spack_using_command repo list' -l json -d 'output r set -g __fish_spack_optspecs_spack_repo_ls h/help scope= names namespaces json complete -c spack -n '__fish_spack_using_command repo ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command repo ls' -l names -f -a names complete -c spack -n '__fish_spack_using_command repo ls' -l names -d 'show configuration names only' @@ -2920,7 +2926,7 @@ complete -c spack -n '__fish_spack_using_command repo add' -l name -r -f -a name complete -c spack -n '__fish_spack_using_command repo add' -l name -r -d 'config name for the package repository, defaults to the namespace of the repository' complete -c spack -n '__fish_spack_using_command repo add' -l path -r -f -a path complete -c spack -n '__fish_spack_using_command repo add' -l path -r -d 'relative path to the Spack package repository inside a git repository. Can be repeated to add multiple package repositories in case of a monorepo' -complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -d 'configuration scope to modify' # spack repo set @@ -2932,7 +2938,7 @@ complete -c spack -n '__fish_spack_using_command repo set' -l destination -r -f complete -c spack -n '__fish_spack_using_command repo set' -l destination -r -d 'destination to clone git repository into' complete -c spack -n '__fish_spack_using_command repo set' -l path -r -f -a path complete -c spack -n '__fish_spack_using_command repo set' -l path -r -d 'relative path to the Spack package repository inside a git repository. Can be repeated to add multiple package repositories in case of a monorepo' -complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -d 'configuration scope to modify' # spack repo remove @@ -2940,7 +2946,7 @@ set -g __fish_spack_optspecs_spack_repo_remove h/help scope= all-scopes complete -c spack -n '__fish_spack_using_command_pos 0 repo remove' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command repo remove' -l all-scopes -f -a all_scopes complete -c spack -n '__fish_spack_using_command repo remove' -l all-scopes -d 'remove from all config scopes (default: highest scope with matching repo)' @@ -2950,7 +2956,7 @@ set -g __fish_spack_optspecs_spack_repo_rm h/help scope= all-scopes complete -c spack -n '__fish_spack_using_command_pos 0 repo rm' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command repo rm' -l all-scopes -f -a all_scopes complete -c spack -n '__fish_spack_using_command repo rm' -l all-scopes -d 'remove from all config scopes (default: highest scope with matching repo)' @@ -2972,7 +2978,7 @@ complete -c spack -n '__fish_spack_using_command repo update' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command repo update' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command repo update' -l remote -s r -r -f -a remote complete -c spack -n '__fish_spack_using_command repo update' -l remote -s r -r -d 'name of remote to check for branches, tags, or commits' -complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command repo update' -l branch -s b -r -f -a branch complete -c spack -n '__fish_spack_using_command repo update' -l branch -s b -r -d 'name of a branch to change to' From 01b5429376afc1b93d7652408de71be248ec3b64 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 23 May 2026 21:57:08 -0700 Subject: [PATCH 500/506] config: note layout scheme scopes in module docstring Make it discoverable from spack/config.py that defaults/include.yaml conditionally pulls in defaults/old/ or defaults/xdg/, and point at spack.paths.detect_layout for the decision logic. --- lib/spack/spack/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 067c6be7436c93..1e4bcc3d1aa9b1 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -11,7 +11,10 @@ #. ``spack`` in ``$spack/etc/spack`` - controls all built-in spack scopes, except default #. ``defaults`` in ``$spack/etc/spack/defaults`` - defaults that Spack - needs to function + needs to function. ``defaults/include.yaml`` conditionally pulls in + one of ``defaults/old/`` or ``defaults/xdg/`` (the layout *scheme*) + based on ``layout_detected("old")``; see + :func:`spack.paths.detect_layout`. Important functions in this module are: From 393debcac34497a0205bc2b4e62f395f73f581ad Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 23 May 2026 22:37:12 -0700 Subject: [PATCH 501/506] tests: rewrite test/paths.py for the new scheme-based model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old test/paths.py exercised the SpackPaths.default_install_location ladder, the resolve_a_home() six-way precedence chain, the Provenance enum, and the "gpg falls back to old if new is empty" filesystem-probe logic — all removed when path policy moved into yaml. New tests cover what's left: * Home properties (state/data/cache/spack_home) read config:locations:* with SPACK_*_HOME env var overrides and the disable_env flag. * SPACK_USER_CACHE_PATH is honored as a legacy alias for SPACK_STATE_HOME (older wins when both set). * SPACK_USER_CONFIG_PATH / SPACK_SYSTEM_CONFIG_PATH still work (these live in paths_base since they bootstrap config). * detect_layout("old" | "xdg") and the layout_detected eval_conditional helper: empty root, populated old data, unilateral env-var override. * $xdg_data_home/$xdg_state_home/$xdg_cache_home path substitutions respect $XDG_*_HOME env vars and fall back to spec defaults. * Derived paths (reports, package_repos, etc.) follow state_home. * dotspack_backup is now pinned to the XDG default — not data_home — so the migration artifact lives in a stable spot. paths.py and cmd/migrate.py updated to match; migrate test fixture simplified. * Module shim: spack.paths.X works for both static (prefix) and dynamic (state_home) attributes. cmd/migrate test fixture no longer needs to monkeypatch the now-removed local `paths` alias. --- lib/spack/spack/cmd/migrate.py | 23 +- lib/spack/spack/paths.py | 8 +- lib/spack/spack/test/cmd/migrate.py | 3 +- lib/spack/spack/test/paths.py | 504 +++++++++++++--------------- 4 files changed, 244 insertions(+), 294 deletions(-) diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py index 827d72fd15ae12..92aa7713ed7645 100644 --- a/lib/spack/spack/cmd/migrate.py +++ b/lib/spack/spack/cmd/migrate.py @@ -54,27 +54,10 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def restore(args: argparse.Namespace) -> None: """Restore ~/.spack from backup location.""" old_location = os.path.expanduser("~/.spack") + backup_location = spack.paths.dotspack_backup - # Check both the current backup location and the XDG default. The XDG - # default is checked even when the active scheme is "old", because the - # backup may have been written by a previous xdg-scheme run before the - # user reverted (e.g. by `git pull`-ing an older Spack). - backup_locations = [spack.paths.dotspack_backup] - xdg_default_backup = os.path.expanduser("~/.local/share/spack/dotspack_backup") - if xdg_default_backup != spack.paths.dotspack_backup: - backup_locations.append(xdg_default_backup) - - # Find which backup location exists - backup_location = None - for loc in backup_locations: - if os.path.exists(loc): - backup_location = loc - break - - if not backup_location: - tty.die( - "No backup found. Checked:\n" + "\n".join(f" - {loc}" for loc in backup_locations) - ) + if not os.path.exists(backup_location): + tty.die(f"No backup found at {backup_location}") # Check if ~/.spack already exists if os.path.exists(old_location): diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 58eb84b9a7a621..96f4891d783c16 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -213,9 +213,11 @@ def package_repos_path(self): @property def dotspack_backup(self): #: backup location for old ~/.spack directory during migration. - # Lives under data_home regardless of scheme — the backup is a - # one-time migration artifact, not legacy state to preserve. - return os.path.join(self.data_home, "dotspack_backup") + # Pinned to the XDG default (~/.local/share/spack/dotspack_backup) + # rather than the active scheme's data_home, because the backup is + # a one-time migration artifact and shouldn't move around when the + # user sets SPACK_DATA_HOME or config:locations:data. + return os.path.join(os.path.expanduser("~"), _RELATIVE_DATA, "spack", "dotspack_backup") @property def gpg_path(self): diff --git a/lib/spack/spack/test/cmd/migrate.py b/lib/spack/spack/test/cmd/migrate.py index a4bed1941f35cd..c380a10ac05120 100644 --- a/lib/spack/spack/test/cmd/migrate.py +++ b/lib/spack/spack/test/cmd/migrate.py @@ -51,8 +51,7 @@ def migrate_setup(tmp_path, set_home, monkeypatch, mutable_config): monkeypatch.setattr(spack.paths, "locations", paths) monkeypatch.setattr(spack.paths_base, "locations", base_paths) - # Also need to patch the cmd.migrate module which has already imported these - monkeypatch.setattr(spack.cmd.migrate, "paths", paths) + # cmd.migrate imports `paths_base` directly; patch that too. monkeypatch.setattr(spack.cmd.migrate, "paths_base", base_paths) yield (dotspack, created, new_config, new_state, paths) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py index da0becf1f56a4e..46359cd48eb9ce 100644 --- a/lib/spack/spack/test/paths.py +++ b/lib/spack/spack/test/paths.py @@ -7,10 +7,8 @@ import pytest -import spack.config import spack.paths import spack.paths_base -import spack.subprocess_context from spack.paths import SpackPaths from spack.paths_base import SpackPathsBase @@ -22,332 +20,300 @@ def _ensure_dir(pathlike): @pytest.fixture(autouse=True) def clear_env_vars(working_env): + """Scrub XDG_*/SPACK_* env vars so each test starts from a known state.""" spack.paths._unset_path_vars(os.environ) -def test_install_location(working_env, tmp_path, mutable_config, set_home): - # If prior default install dir inside spack prefix does not - # exist, place installs in $HOME - base_prefix = _ensure_dir(tmp_path / "spack-root") - home_prefix = _ensure_dir(tmp_path / "home-prefix") +def _set_locations(cfg, **kwargs): + """Set config:locations:* keys (avoiding the leftover state pitfall of + overwriting the whole locations block).""" + for k, v in kwargs.items(): + cfg.set(f"config:locations:{k}", v) - empty_dir = _ensure_dir(tmp_path / "empty") - def paths_base_empty_old_install(): - pb = SpackPathsBase(base_prefix) - pb.old_install_path = empty_dir - return pb +# --------------------------------------------------------------------------- +# Home property resolution +# --------------------------------------------------------------------------- - set_home(home_prefix) - _install_path_checks(tmp_path, paths_base_empty_old_install, home_prefix, False) +def test_data_home_from_config(working_env, tmp_path, mutable_config, set_home): + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, data=str(tmp_path / "datadir")) + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.data_home == str(tmp_path / "datadir") -def test_install_location_old_installs_exist(working_env, tmp_path, mutable_config, set_home): - # If prior default install dir inside spack prefix does not - # exist, place installs in $HOME - base_prefix = _ensure_dir(tmp_path / "spack-root") - home_prefix = _ensure_dir(tmp_path / "home-prefix") +def test_state_home_from_config(working_env, tmp_path, mutable_config, set_home): + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, state=str(tmp_path / "statedir")) + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.state_home == str(tmp_path / "statedir") - nonempty_dir = _ensure_dir(tmp_path / "not-empty") - (pathlib.Path(nonempty_dir) / "afile").touch() - def paths_base_nonempty_old_install(): - pb = SpackPathsBase(base_prefix) - pb.old_install_path = nonempty_dir - return pb +def test_cache_home_from_config(working_env, tmp_path, mutable_config, set_home): + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, cache=str(tmp_path / "cachedir")) + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.cache_home == str(tmp_path / "cachedir") - set_home(home_prefix) - _install_path_checks(tmp_path, paths_base_nonempty_old_install, home_prefix, True) +def test_spack_home_from_config(working_env, tmp_path, mutable_config, set_home): + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, home=str(tmp_path / "spackhome")) + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.spack_home == str(tmp_path / "spackhome") -def _install_path_checks(tmp_path, base_paths_generator, home_prefix, force_old_layout): - def checka(paths, new_path, msg): - if force_old_layout: - assert paths.default_install_location == paths.base.old_install_path, msg - else: - assert paths.default_install_location == str(new_path), msg +def test_data_home_env_overrides_config(working_env, tmp_path, mutable_config, set_home): + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, data=str(tmp_path / "fromconfig")) + os.environ["SPACK_DATA_HOME"] = str(tmp_path / "fromenv") + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.data_home == str(tmp_path / "fromenv") - def checkb(paths, new_path, msg): - assert paths.default_install_location == str(new_path), msg - new_default_installs_dir = _ensure_dir( - pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs" - ) - (pathlib.Path(new_default_installs_dir) / "afile").touch() - p0 = SpackPaths(base_paths_generator()) - checka( - p0, - pathlib.Path(home_prefix) / ".local" / "share" / "spack" / "installs", - "p0: default location", - ) +def test_spack_home_env_overrides_subhomes(working_env, tmp_path, mutable_config, set_home): + """SPACK_HOME (no _DATA_/_STATE_/_CACHE_) provides a root and the homes + derive from it via their XDG-style subpaths.""" + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, data=str(tmp_path / "fromconfig")) + os.environ["SPACK_HOME"] = str(tmp_path / "alt-home") + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.data_home == str(tmp_path / "alt-home" / ".local" / "share" / "spack") + assert p.state_home == str(tmp_path / "alt-home" / ".local" / "state" / "spack") + assert p.cache_home == str(tmp_path / "alt-home" / ".cache" / "spack") - # $XDG_DATA_HOME overrides the default - xdg_data_home = _ensure_dir(tmp_path / "xdg_data_home") - os.environ["XDG_DATA_HOME"] = xdg_data_home - p7 = SpackPaths(base_paths_generator()) - checka(p7, pathlib.Path(xdg_data_home) / "spack" / "installs", "p7: XDG_DATA_HOME override") - - spack.config.set("config:locations", {}) - - # "config:locations:home" variable overrides the above - spack_home_cfg_prefix = _ensure_dir(tmp_path / "spack-home2") - spack.config.set("config:locations:home", spack_home_cfg_prefix) - p2 = SpackPaths(base_paths_generator()) - checkb( - p2, - pathlib.Path(spack_home_cfg_prefix) / ".local" / "share" / "spack" / "installs", - "p2: config:locations:home override", - ) - # "config:locations:data" overrides the above - spack_data_prefix = _ensure_dir(tmp_path / "spack-data") - spack.config.set("config:locations:data", spack_data_prefix) - p3 = SpackPaths(base_paths_generator()) - checkb(p3, pathlib.Path(spack_data_prefix) / "installs", "p3: config:locations:data override") - - # SPACK_HOME env variable overrides the above (even if there - # are no installs there and there are installs in the old location) - spack_home_env_prefix = _ensure_dir(tmp_path / "spack-home1") - os.environ["SPACK_HOME"] = spack_home_env_prefix - p1 = SpackPaths(base_paths_generator()) - checkb( - p1, - pathlib.Path(spack_home_env_prefix) / ".local" / "share" / "spack" / "installs", - "p1: SPACK_HOME override", - ) +def test_specific_env_overrides_spack_home(working_env, tmp_path, mutable_config, set_home): + """SPACK_DATA_HOME beats SPACK_HOME for data_home.""" + set_home(_ensure_dir(tmp_path / "home")) + os.environ["SPACK_HOME"] = str(tmp_path / "alt-home") + os.environ["SPACK_DATA_HOME"] = str(tmp_path / "specific-data") + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.data_home == str(tmp_path / "specific-data") - # Check that $SPACK_DATA_HOME overrides all the above - spack_data_home = _ensure_dir(tmp_path / "spack_data_home") - os.environ["SPACK_DATA_HOME"] = spack_data_home - p5 = SpackPaths(base_paths_generator()) - checkb(p5, pathlib.Path(spack_data_home) / "installs", "p5: SPACK_DATA_HOME override") - - # Disable all location-based env vars: this will then defer - # to using "config:locations:data" - spack.config.set("config:locations:disable_env", True) - p6 = SpackPaths(base_paths_generator()) - checkb( - p6, - pathlib.Path(spack_data_prefix) / "installs", - "p6: disable_env defers to config:locations:data", - ) +def test_disable_env_ignores_env_vars(working_env, tmp_path, mutable_config, set_home): + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, data=str(tmp_path / "fromconfig")) + mutable_config.set("config:locations:disable_env", True) + os.environ["SPACK_DATA_HOME"] = str(tmp_path / "fromenv") + os.environ["SPACK_HOME"] = str(tmp_path / "alt-home") + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.data_home == str(tmp_path / "fromconfig") -def test_state_home(working_env, tmp_path, mutable_config, set_home): - base_prefix = _ensure_dir(tmp_path / "spack-root") - home_prefix = _ensure_dir(tmp_path / "home-prefix") - set_home(home_prefix) - pb = SpackPathsBase(base_prefix) +# --------------------------------------------------------------------------- +# SPACK_USER_CACHE_PATH is a legacy alias for SPACK_STATE_HOME +# --------------------------------------------------------------------------- - new_user_cache_path = str(pathlib.Path(home_prefix) / ".local" / "state" / "spack") - p0 = SpackPaths(pb) - # if neither old nor new dir is occupied, choose new - assert p0.state_home == new_user_cache_path +def test_user_cache_path_env_sets_state_home(working_env, tmp_path): + target = str(tmp_path / "cache") + os.environ["SPACK_USER_CACHE_PATH"] = target + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "base"))) + assert p.user_cache_path == target + assert p.state_home == target + # Things rooted under state_home follow. + assert p.package_repos_path == os.path.join(target, "package_repos") - old_user_cache_path = pathlib.Path(home_prefix) / ".spack" - old_user_cache_path.mkdir(parents=True) - (old_user_cache_path / "afile").touch() - p1 = SpackPaths(pb) - # if new dir does not exist, and old dir does, choose old - assert p1.state_home == str(old_user_cache_path) - os.environ["SPACK_DATA_HOME"] = base_prefix - p2 = SpackPaths(pb) - # While setting any new-style location env var or config:locations value - # relocates installs, gpg keys, downloads, etc. It does not make spack - # choose ~/.local/state/spack over ~/.spack - assert p2.state_home == str(old_user_cache_path) +def test_state_home_env_overrides_user_cache_path(working_env, tmp_path): + """When both SPACK_USER_CACHE_PATH and SPACK_STATE_HOME are set, the + older one (SPACK_USER_CACHE_PATH) wins, matching pre-1.2 behavior.""" + legacy = str(tmp_path / "legacy") + new = str(tmp_path / "new") + os.environ["SPACK_USER_CACHE_PATH"] = legacy + os.environ["SPACK_STATE_HOME"] = new + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "base"))) + assert p.state_home == legacy - os.environ["SPACK_HOME"] = home_prefix - p3 = SpackPaths(pb) - # ... SPACK_HOME is special amongst these variables though, because by - # definition it is supposed to relocate everything except for config - # (caches included) - assert p3.state_home == new_user_cache_path + +# --------------------------------------------------------------------------- +# Config path overrides (SPACK_USER_CONFIG_PATH, SPACK_SYSTEM_CONFIG_PATH) +# These still live in paths_base because they bootstrap config itself. +# --------------------------------------------------------------------------- + + +def test_user_config_path_is_overridable(working_env, tmp_path): + target = str(tmp_path / "redirected_usrcfg") + os.environ["SPACK_USER_CONFIG_PATH"] = target + pb = SpackPathsBase(_ensure_dir(tmp_path / "base-prefix")) + assert pb.user_config_path == target + + +def test_user_config_path_default(working_env, tmp_path): + os.environ["SPACK_USER_CONFIG_PATH"] = "" + pb = SpackPathsBase(str(tmp_path)) + assert pb.user_config_path == os.path.expanduser(os.path.join("~", ".config", "spack")) def test_system_config_path_is_overridable(working_env, tmp_path): - redirect_syscfg_path = str(pathlib.Path(tmp_path) / "redirected_syscfg") - os.environ["SPACK_SYSTEM_CONFIG_PATH"] = redirect_syscfg_path - p1 = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "base-prefix"))) - assert p1.system_config_path == redirect_syscfg_path + target = str(tmp_path / "redirected_syscfg") + os.environ["SPACK_SYSTEM_CONFIG_PATH"] = target + pb = SpackPathsBase(_ensure_dir(tmp_path / "base-prefix")) + assert pb.system_config_path == target -def test_system_config_path_is_default_when_env_var_is_empty(working_env, tmp_path): +def test_system_config_path_default(working_env, tmp_path): os.environ["SPACK_SYSTEM_CONFIG_PATH"] = "" - p1 = SpackPaths(SpackPathsBase(str(tmp_path))) - assert os.sep + os.path.join("etc", "spack") == p1.system_config_path + pb = SpackPathsBase(str(tmp_path)) + assert pb.system_config_path == os.sep + os.path.join("etc", "spack") -def test_user_config_path_is_overridable(working_env, tmp_path): - redirect_usrcfg_path = str(pathlib.Path(tmp_path) / "redirected_usrcfg") - os.environ["SPACK_USER_CONFIG_PATH"] = redirect_usrcfg_path - p1 = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "base-prefix"))) - assert p1.user_config_path == redirect_usrcfg_path +# --------------------------------------------------------------------------- +# Layout detection (detect_layout / layout_detected) — the helper exposed +# to include `when:` clauses. +# --------------------------------------------------------------------------- -def test_user_config_path_is_default_when_env_var_is_empty(working_env, tmp_path): - os.environ["SPACK_USER_CONFIG_PATH"] = "" - p1 = SpackPaths(SpackPathsBase(str(tmp_path))) - assert os.path.expanduser(os.path.join("~", ".config", "spack")) == p1.user_config_path - - -def test_user_cache_path_is_overridable(working_env, tmp_path): - redirect1 = str(pathlib.Path(tmp_path) / "redirected_usr_cache") - os.environ["SPACK_USER_CACHE_PATH"] = redirect1 - p1 = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "base-prefix"))) - assert p1.user_cache_path == redirect1 - # Check that things that are supposed to be bundled inside of - # $user_cache_path are also relocated - assert p1.package_repos_path == str(pathlib.Path(redirect1) / "package_repos") - - # Now check that $SPACK_STATE_HOME takes precedence when both are set - redirect2 = str(pathlib.Path(tmp_path) / "redirected_usr_cache2") - os.environ["SPACK_STATE_HOME"] = redirect2 - p2 = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "base-prefix"))) - assert p2.user_cache_path == redirect2 - assert p2.package_repos_path == str(pathlib.Path(redirect2) / "package_repos") - - -def test_gpg_only_use_new_path_if_old_is_empty(working_env, tmp_path, set_home): - base_prefix = _ensure_dir(tmp_path / "base-prefix") - set_home(base_prefix) - - new_default_gpg_base = pathlib.Path(base_prefix) / ".local" / "share" / "spack" - - # Nothing in any of the old locations: we should use the new one - p1 = SpackPaths(SpackPathsBase(base_prefix)) - assert p1.gpg_path == str(new_default_gpg_base / "gpg") - assert p1.gpg_keys_path == str(new_default_gpg_base / "gpg-keys") - - old_gpg_dir = pathlib.Path(base_prefix) / "opt" / "spack" / "gpg" - (old_gpg_dir).mkdir(parents=True) - p1 = SpackPaths(SpackPathsBase(base_prefix)) - # Old dir exists, but is empty, so it should still not be used - assert p1.gpg_path == str(new_default_gpg_base / "gpg") - - # Put something in the old dir: it should now redirect - (old_gpg_dir / "something").touch() - p1 = SpackPaths(SpackPathsBase(base_prefix)) - assert p1.gpg_path == str(old_gpg_dir) - - # Old keys path is used if old gpg path is used: all data is - # relocated together - old_gpg_keys_dir = pathlib.Path(base_prefix) / "var" / "spack" / "gpg" - assert p1.gpg_keys_path == str(old_gpg_keys_dir) - - # When something is in both the new and old locations, prefer the old - new_gpg_dir = new_default_gpg_base / "gpg" - new_gpg_dir.mkdir(parents=True) - (new_gpg_dir / "something").touch() - p1 = SpackPaths(SpackPathsBase(base_prefix)) - assert p1.gpg_keys_path == str(old_gpg_keys_dir) - - # And the gpg dir itself remains the old dir: reaffirm that - assert p1.gpg_path == str(old_gpg_dir) - - -def test_user_cache_path_is_default_when_env_var_is_empty(tmp_path, set_home): - homedir = _ensure_dir(tmp_path / "base-prefix") - set_home(homedir) - p1 = SpackPaths(SpackPathsBase(str(tmp_path))) - assert ( - str(pathlib.Path(homedir) / os.path.join(".local", "state", "spack")) == p1.user_cache_path - ) +@pytest.fixture +def empty_spack_root(tmp_path): + base = SpackPathsBase(_ensure_dir(tmp_path / "spack-root")) + yield base + + +@pytest.fixture +def occupied_spack_root(tmp_path): + base = SpackPathsBase(_ensure_dir(tmp_path / "spack-root")) + # Put something in old install location to trigger old-layout detection. + inst = pathlib.Path(base.old_install_path) + inst.mkdir(parents=True, exist_ok=True) + (inst / "marker").touch() + yield base -def test_location_vars_that_use_other_location_vars( - tmp_path, set_home, mutable_config, monkeypatch -): - homedir = _ensure_dir(tmp_path / "test-home") - set_home(homedir) +def test_detect_layout_empty_root_is_xdg(working_env, empty_spack_root, monkeypatch): + monkeypatch.setattr(spack.paths_base, "locations", empty_spack_root) + assert spack.paths.detect_layout("old") is False + assert spack.paths.detect_layout("xdg") is True - basedir = _ensure_dir(tmp_path / "spack-root") - mutable_config.set("config", {"locations": {"home": "$spack/home"}}) +def test_detect_layout_old_data_present(working_env, occupied_spack_root, monkeypatch): + monkeypatch.setattr(spack.paths_base, "locations", occupied_spack_root) + assert spack.paths.detect_layout("old") is True + assert spack.paths.detect_layout("xdg") is False - p1 = SpackPaths(SpackPathsBase(str(basedir))) - # This is a bit strange but resolution of the config variable involves accessing - # the module, so we need to monkeypatch that - monkeypatch.setattr(spack.paths_base, "locations", p1.base) - install_rel = pathlib.Path(".local") / "share" / "spack" / "installs" - assert p1.default_install_location == str(pathlib.Path(basedir) / "home" / install_rel) - # Now try defining a $data_home -> $spack_home -> $spack - p2 = SpackPaths(SpackPathsBase(str(basedir))) - monkeypatch.setattr(spack.paths_base, "locations", p2.base) - mutable_config.set("config", {"locations": {"home": "$spack/home", "data": "$spack_home"}}) - assert p2.default_install_location == str(pathlib.Path(basedir) / "home" / "installs") +def test_detect_layout_env_var_forces_xdg(working_env, occupied_spack_root, monkeypatch): + """Even with old-layout markers, any SPACK_*_HOME env var forces xdg.""" + monkeypatch.setattr(spack.paths_base, "locations", occupied_spack_root) + for var in ("SPACK_DATA_HOME", "SPACK_STATE_HOME", "SPACK_CACHE_HOME", "SPACK_HOME"): + os.environ.pop(var, None) + os.environ[var] = "/tmp/somewhere" + try: + assert spack.paths.detect_layout("old") is False, f"{var} should force xdg" + finally: + del os.environ[var] -def test_license_dir_config(mutable_config, mock_packages, tmp_path, monkeypatch, set_home): - """Ensure license directory is customizable""" - import spack.package_base - import spack.repo +def test_detect_layout_rejects_unknown_scheme(): + with pytest.raises(ValueError, match="unknown layout scheme"): + spack.paths.detect_layout("eggplant") - basedir = _ensure_dir(tmp_path / "spack-root") - homedir = _ensure_dir(tmp_path / "base-prefix") - set_home(homedir) - p1 = SpackPaths(SpackPathsBase(str(basedir))) - monkeypatch.setattr(spack.paths, "locations", p1) +def test_layout_detected_in_eval_conditional(occupied_spack_root, monkeypatch): + """The helper is reachable from include `when:` clauses via eval_conditional.""" + import spack.spec - default_cfg_val = os.path.join("$data_home", "licenses") - resolved_dir = str(pathlib.Path(homedir) / ".local" / "share" / "spack" / "licenses") - assert spack.config.get("config:license_dir") == default_cfg_val - assert spack.package_base.PackageBase.global_license_dir == resolved_dir - assert spack.repo.PATH.get_pkg_class("pkg-a").global_license_dir == resolved_dir + monkeypatch.setattr(spack.paths_base, "locations", occupied_spack_root) + assert spack.spec.eval_conditional("layout_detected('old')") is True + assert spack.spec.eval_conditional("not layout_detected('old')") is False + assert spack.spec.eval_conditional("layout_detected('xdg')") is False - abs_path = str(tmp_path / "foo" / "bar" / "baz") - spack.config.set("config:license_dir", abs_path) - assert spack.config.get("config:license_dir") == abs_path - assert spack.package_base.PackageBase.global_license_dir == abs_path - assert spack.repo.PATH.get_pkg_class("pkg-a").global_license_dir == abs_path +# --------------------------------------------------------------------------- +# $xdg_data_home / $xdg_state_home / $xdg_cache_home substitutions +# --------------------------------------------------------------------------- -class SetAnXdgVarAndReadDataHome: - """Access an XDG-dependent variable from spack.paths as quickly as - possible. - """ - def __init__(self, home_prefix): - self.home_prefix = home_prefix +def test_xdg_substitutions_respect_env_vars(working_env, tmp_path): + from spack.util.path import canonicalize_path - def __call__(self): - import spack.paths + os.environ["XDG_DATA_HOME"] = str(tmp_path / "xdg-data") + os.environ["XDG_STATE_HOME"] = str(tmp_path / "xdg-state") + os.environ["XDG_CACHE_HOME"] = str(tmp_path / "xdg-cache") - os.environ["XDG_DATA_HOME"] = "/made-up-value-that-shouldnt-matter" + assert canonicalize_path("$xdg_data_home/spack") == str(tmp_path / "xdg-data" / "spack") + assert canonicalize_path("$xdg_state_home/spack") == str(tmp_path / "xdg-state" / "spack") + assert canonicalize_path("$xdg_cache_home/spack") == str(tmp_path / "xdg-cache" / "spack") - expected = str(pathlib.Path(self.home_prefix) / ".local" / "share" / "spack" / "installs") - assert spack.paths.locations.default_install_location == expected, ( - f"Expected {expected}\nGot {spack.paths.locations.default_install_location}" - ) +def test_xdg_substitutions_fall_back_to_defaults(working_env, set_home, tmp_path): + from spack.util.path import canonicalize_path -def test_child_proc_sanity_xdg_based_paths(tmp_path, set_home, monkeypatch): - # Unlike the other tests in this module, this is specifically testing - # the behavior of the spack.paths module vs. (the more targeted testing - # of) classes defined within it. - base_prefix = _ensure_dir(tmp_path / "spack-root") - home_prefix = _ensure_dir(tmp_path / "home-prefix") + home = _ensure_dir(tmp_path / "home") + set_home(home) + assert canonicalize_path("$xdg_data_home/spack") == os.path.join( + home, ".local", "share", "spack" + ) + assert canonicalize_path("$xdg_state_home/spack") == os.path.join( + home, ".local", "state", "spack" + ) + assert canonicalize_path("$xdg_cache_home/spack") == os.path.join(home, ".cache", "spack") - empty_dir = _ensure_dir(tmp_path / "empty") - set_home(home_prefix) +# --------------------------------------------------------------------------- +# Derived paths (reports, package_repos, etc.) follow state_home +# --------------------------------------------------------------------------- - pbtest = SpackPathsBase(base_prefix) - pbtest.old_install_path = empty_dir - ptest = SpackPaths(pbtest) - monkeypatch.setattr(spack.paths, "locations", ptest) - spack_process = spack.subprocess_context.SpackTestProcess( - SetAnXdgVarAndReadDataHome(home_prefix) - ) - p = spack_process.create() - p.start() - p.join() - assert p.exitcode == 0 +def test_derived_paths_follow_state_home(working_env, tmp_path, mutable_config, set_home): + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, state=str(tmp_path / "statedir")) + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.reports_path == str(tmp_path / "statedir" / "reports") + assert p.default_monitor_path == str(tmp_path / "statedir" / "reports" / "monitor") + assert p.user_repos_cache_path == str(tmp_path / "statedir" / "git_repos") + assert p.package_repos_path == str(tmp_path / "statedir" / "package_repos") + + +def test_dotspack_backup_pinned_to_xdg_default(working_env, tmp_path, mutable_config, set_home): + """dotspack_backup is pinned to ~/.local/share/spack/dotspack_backup and + does NOT follow data_home, so the migration backup lives in a stable + location regardless of SPACK_DATA_HOME / config:locations:data.""" + home = _ensure_dir(tmp_path / "home") + set_home(home) + _set_locations(mutable_config, data=str(tmp_path / "datadir")) # ignored by dotspack_backup + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.dotspack_backup == os.path.join(home, ".local", "share", "spack", "dotspack_backup") + + +# --------------------------------------------------------------------------- +# gpg_path / gpg_keys_path read config first, fall back to data_home +# --------------------------------------------------------------------------- + + +def test_gpg_path_from_config(working_env, tmp_path, mutable_config, set_home): + set_home(_ensure_dir(tmp_path / "home")) + mutable_config.set("config:gpg_path", str(tmp_path / "my-gpg")) + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.gpg_path == str(tmp_path / "my-gpg") + + +def test_gpg_path_default_from_scheme(working_env, tmp_path, mutable_config, set_home): + """The xdg scheme yaml sets config:gpg_path to $data_home/gpg by default; + the property simply reads that.""" + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, data=str(tmp_path / "datadir")) + # base/config.yaml sets gpg_path to $data_home/gpg + mutable_config.set("config:gpg_path", "$data_home/gpg") + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.gpg_path == str(tmp_path / "datadir" / "gpg") + + +# --------------------------------------------------------------------------- +# Module shim: spack.paths.X works for both static and dynamic attributes +# --------------------------------------------------------------------------- + + +def test_module_shim_static_attribute(): + # `prefix` lives on paths_base.SpackPathsBase; spack.paths.prefix + # should reach it via the SpackPaths.__getattr__ delegation. + assert spack.paths.prefix == spack.paths.locations.prefix + + +def test_module_shim_dynamic_attribute(): + # `state_home` is computed by SpackPaths; access via the module + # should also work. + assert spack.paths.state_home == spack.paths.locations.state_home From 220f753548143fcf3d97714d37aeba594d5bdeca Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 23 May 2026 22:45:05 -0700 Subject: [PATCH 502/506] env: keep env_root_path fallback for stripped-down test configs env_root_path now defaults config:environments_root to "$data_home/environments" when unset, matching the base default scope expression. Test fixtures that build a minimal mock config (without the scheme yamls) no longer trip on a None config value. --- lib/spack/spack/environment/environment.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index 7bef6fdd31692e..53f3c4d45b41ab 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -97,10 +97,12 @@ def env_root_path() -> str: """Resolve config:environments_root with substitutions and ~ expansion. - Always set by the active layout scheme yaml - (etc/spack/defaults/{old,xdg}/config.yaml via base). + The base default scope sets this to ``$data_home/environments``; tests + using a stripped-down config may leave it unset, so fall back to the + same expression here. """ - return spack.util.path.canonicalize_path(spack.config.get("config:environments_root")) + root = spack.config.get("config:environments_root") or "$data_home/environments" + return spack.util.path.canonicalize_path(root) def environment_name(path: Union[str, pathlib.Path]) -> str: From 16609dd75faac1bae70e3624875fa1cf54637089 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 24 May 2026 08:27:19 -0700 Subject: [PATCH 503/506] docs: update where_spack_writes_data.rst and shared-spack.txt for schemes Rewrite both docs to describe the layout-scheme include model: * etc/spack/defaults/include.yaml picks old/ or xdg/ based on layout_detected("old"). * base/config.yaml's defaults use $data_home substitutions; old scheme overrides install_tree:root, environments_root, etc. directly. * User-visible override priorities: SPACK_*_HOME env > specific config key > config:locations:* > scheme default. * spack debug paths is the recommended diagnostic. * dotspack_backup is pinned to the XDG default. --- lib/spack/docs/where_spack_writes_data.rst | 144 +++++------ shared-spack.txt | 287 ++++++++++----------- 2 files changed, 202 insertions(+), 229 deletions(-) diff --git a/lib/spack/docs/where_spack_writes_data.rst b/lib/spack/docs/where_spack_writes_data.rst index 24c9d83bd433b7..6dd1dda0986d7b 100644 --- a/lib/spack/docs/where_spack_writes_data.rst +++ b/lib/spack/docs/where_spack_writes_data.rst @@ -9,59 +9,85 @@ .. _where_spack_writes_data: -Controlling where spack writes data +Controlling where Spack writes data =================================== -A fresh checkout of spack will not write anything into the ``$spack`` prefix; instead, all data is placed under the user's home directory. -You can control this in the following ways: +A fresh checkout of Spack writes nothing into the ``$spack`` prefix; all data goes under the user's home directory in XDG-compliant locations. +A Spack instance that was installed before this layout — where data lived under ``$spack/opt``, ``$spack/var``, etc. — keeps using those legacy locations, so existing installs and environments are not disrupted. -* Redirect everything with environment variables: set ``SPACK_HOME`` and one of ``SPACK_USER_CONFIG_PATH`` or ``SPACK_DISABLE_LOCAL_CONFIG=1`` -* Or redirect everything with config: +Spack picks one of two *layout schemes* at startup: - * set ``config:locations:home`` - * Update the ``user`` config scope with ``spack config --scope=spack edit include`` -* Or redirect installs, environments, and cached downloads (everything that takes up significant space) by setting ``SPACK_DATA_HOME`` -* Or use finer-grained configuration settings, for example: +* **xdg**: data under ``~/.local/share/spack``, state under ``~/.local/state/spack``, cache under ``~/.cache/spack``. +* **old**: installs in ``$spack/opt/spack``, environments in ``$spack/var/spack/environments``, license files in ``$spack/etc/spack/licenses``, etc. — i.e. the pre-1.2 layout. - * ``config:install_tree:root`` to control where installs go - * ``config:build_stage`` to control where builds are staged +The scheme is chosen by ``etc/spack/defaults/include.yaml`` using a ``when:`` clause that calls :func:`spack.paths.detect_layout`. +The included yaml — ``etc/spack/defaults/old/config.yaml`` or ``etc/spack/defaults/xdg/config.yaml`` — sets ``config:locations:*`` (and, for old, the install_tree/environments/etc. paths that don't share a single root). +Everything else flows through normal config: ``config:install_tree:root`` is ``$data_home/installs``, environments root is ``$data_home/environments``, gpg lives at ``$data_home/gpg``, and so on. -In the absence of any Spack-specific settings, Spack will respect [XDG](https://specifications.freedesktop.org/basedir/latest/) environment variables controlling the home directory for specific types of data. +You can see the active scheme and where each path came from with:: -Spack's older layout, and pulling newer versions of Spack -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + spack debug paths -Spack previously stored many important pieces of data in the Spack prefix: +Sample output:: -* Installs in ``$spack/opt/spack`` -* Environments in ``$spack/var/spack/environments`` -* GPG keys in ``$spack/opt/spack/gpg`` + layout scheme: xdg (no legacy $spack-local data) -If Spack detects this old layout in use, it will continue to use it. -Targeted config settings like ``config:install_tree:root`` will override this, but not other environment variables or general configuration (i.e. anything described below this section). + homes: + $data_home /home/alice/.local/share/spack + config:locations:data (scope: defaults:xdg) + $state_home /home/alice/.local/state/spack + config:locations:state (scope: defaults:xdg) + ... -.. _config-file-data-variables: + config-driven paths: + config:install_tree:root /home/alice/.local/share/spack/installs + $data_home/installs [scope: defaults:base] + ... -Spack-specific variables controlling data location -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +How to override +--------------- -Files generated and used by spack are organized roughly into four categories: +In order of priority (highest first): -* Persistent, large quantities of data (e.g. installs and environments) -* Temporary (or assumed temporary) large quantities of data (e.g. stages for installs) -* Persistent caches/indices used by spack to speed up its commands (small quantities of data) -* The configuration files themselves +1. **Env vars** for individual homes — ``SPACK_DATA_HOME``, ``SPACK_STATE_HOME``, ``SPACK_CACHE_HOME``, or ``SPACK_HOME`` (which sets all three via XDG-style subpaths). Any of these also *forces the xdg scheme* even when legacy ``$spack`` data is present, so a partial override never produces a split layout. -The corresponding variables that describe where this data is placed are: +2. **Specific config keys** for individual paths — set ``config:install_tree:root``, ``config:environments_root``, ``config:license_dir``, ``config:source_cache``, ``config:gpg_path``, or ``config:gpg_keys_path`` in any user/site/system scope. -* ``$data_home`` -* ``$cache_home`` -* ``$state_home`` (also known as ``$user_cache_path``) -* Config file locations are an exception: they can only be controlled with :ref:`environment variables ` or with :ref:`include.yaml ` +3. **Layout roots** — set ``config:locations:{home,data,state,cache}`` to redirect everything that uses the corresponding substitution. -You can refer to these variables when configuring locations for stages, misc cache, etc. +Config locations themselves — ``user_config_path``, ``system_config_path``, the entry-point ``include.yaml`` — are NOT in config (they bootstrap config). Override them with ``SPACK_USER_CONFIG_PATH``, ``SPACK_SYSTEM_CONFIG_PATH``, or ``SPACK_DISABLE_LOCAL_CONFIG``. -The following table may help visualize where spack puts all the files it may generate: +Path substitutions +------------------ + +Config values can reference these in any string field: + +* ``$data_home`` — typically ``~/.local/share/spack`` (xdg) or ``$spack`` (old) +* ``$state_home`` — typically ``~/.local/state/spack`` (xdg) or ``~/.spack`` (old) +* ``$cache_home`` — typically ``~/.cache/spack`` +* ``$spack_home`` — base for spack's user-level data; defaults to ``~`` +* ``$xdg_data_home`` — ``$XDG_DATA_HOME`` if set, else ``~/.local/share`` (no ``/spack`` suffix) +* ``$xdg_state_home`` — ``$XDG_STATE_HOME`` if set, else ``~/.local/state`` +* ``$xdg_cache_home`` — ``$XDG_CACHE_HOME`` if set, else ``~/.cache`` +* ``$user_cache_path`` — alias for ``$state_home`` (legacy) +* ``$spack`` — the Spack instance's prefix +* ``$spack_instance_id`` — hash distinguishing co-installed Spack instances + +The ``$xdg_*_home`` substitutions are used by the xdg scheme yaml so the layout respects XDG env vars without baking that resolution into Python. + +Migrating from the old layout +----------------------------- + +If you have a ``~/.spack`` directory from before 1.2, you'll see a one-time warning. Run:: + + spack migrate --clear + +to copy your config into ``~/.config/spack`` and move ``~/.spack`` to a backup at ``~/.local/share/spack/dotspack_backup``. (The backup location is fixed; it does not move when you set ``SPACK_DATA_HOME``.) Use ``spack migrate --restore`` to undo. + +If you have older Spack instances that can't be upgraded and need ``~/.spack`` to stick around, see ``spack migrate --i-need-old-spack``. + +The location table +------------------ +----------------+-----------+--------------------+------------+--------------------+ | | data_home | state_home | cache_home | somewhere_else | @@ -87,52 +113,12 @@ The following table may help visualize where spack puts all the files it may gen | config files | | | | x [#wheretable-3]_ | +----------------+-----------+--------------------+------------+--------------------+ -.. [#wheretable-1] ``cache_home`` is used as a backup, but Spack prefers to write into the user's temp dir if it's available -.. [#wheretable-2] ``cache_home`` is modeled after `$XDG_CACHE_HOME `_. - Spack assumes that ``$XDG_CACHE_HOME`` can be removed on user log-out. - Spack caches are intended to be longer-lived, so they live in ``state_home`` instead. -.. [#wheretable-3] as discussed elsewhere in this section, user-scope config is controlled with :ref:`environment variables ` or with :ref:`include.yaml ` to avoid recursion issues with configurable locations. - For the locations of other config scopes and how to write to them instead, see :ref:`configuration scopes `. - -Each of these variables are the *default* (fallback) for data in their category: more-specific data in that category may have config that overrides these defaults. -For example while build stages would reasonably be placed in ``$cache_home``, Spack's default configuration sets ``config:build_stage`` to the user's tempdir. -Any configuration controlling location that is more-specific than the above variables will always take precedence (e.g. ``config:install_tree:root``). - -Each of these variables can be set with config or with environment variables. -For example ``$data_home`` evaluates to one of the following (highest-priority first): - -#. ``SPACK_DATA_HOME`` env var if that is set -#. Under ``SPACK_HOME`` env var; for ``$data_home``, it is ``$SPACK_HOME/.local/share/spack`` -#. ``config:locations:data`` -#. Under ``config:locations:home``; for ``$data_home`` it is ``$spack_home/.local/share/spack`` -#. ``XDG_DATA_HOME/spack`` if XDG_DATA_HOME is set -#. Under the default for ``XDG_DATA_HOME``: ``~/.local/share/spack`` - -``config:locations:home`` / ``SPACK_HOME`` can be used to control all 3 of ``data_home``, ``cache_home``, and ``state_home``. -They are placed relative to this directory (``$spack_home``): - -* ``data_home`` is placed in ``$spack_home/.local/share/spack`` (as described above). -* ``state_home`` is placed in ``$spack_home/.local/state/spack``. -* ``cache_home`` is placed in ``$spack_home/.cache/spack``. - -Location of installs and environments -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Of particular interest is where the environments and installs are placed by Spack, because these can take up a lot of space. -These are controlled by ``$data_home``. -Older installs of spack placed these within ``$spack``, and fallback scheme in these cases is augmented to prefer these old locations if no data is detected in the corresponding new locations: - -* ``$default_install_root``: the location where installs go by default. - Overridden by ``config:install_tree:root``. - Prefers ``$data_home/installs``, but if there are no installs there and there are installs in the old location ``$spack/opt/spack``, then the old location will be used. -* ``$default_envs_root``: the location where environments are managed by default. - Overridden by ``config:environments_root``. - Prefers ``$data_home/envs`` but if there are no envs there and there are envs in the old location ``$spack/var/spack/environments``, then the old location will be used. +.. [#wheretable-1] ``cache_home`` is used as a backup, but Spack prefers to write into the user's temp dir if it's available. +.. [#wheretable-2] ``cache_home`` is modeled after ``$XDG_CACHE_HOME``. Spack assumes that ``$XDG_CACHE_HOME`` can be removed on user log-out; misc cache is intended to be longer-lived, so it lives in ``state_home`` instead. +.. [#wheretable-3] User-scope config is controlled with :ref:`environment variables ` or with :ref:`include.yaml ` to avoid recursion issues with configurable locations. References -^^^^^^^^^^ - -For more on this, see: +---------- * :ref:`include.yaml ` * :ref:`config.yaml ` diff --git a/shared-spack.txt b/shared-spack.txt index c429d9d2afceff..8b8d7e7a41be24 100644 --- a/shared-spack.txt +++ b/shared-spack.txt @@ -1,159 +1,146 @@ Shared spack +============ -This document is accurate up to commit e48b8c104 +This document captures the design decisions behind moving Spack's mutable +state out of $spack so the prefix can be treated as immutable (apt/pip +installable, multi-user readable). The implementation is described in +lib/spack/docs/where_spack_writes_data.rst. -# Moving Spack-generated artifacts outside of $spack +# What's moving out of $spack -The biggest blocker to creating a shared spack instance is that Spack -will default to writing data into its own prefix. For new clones of -Spack, this PR moves all generated files out of the cloned directory -and into ~. This includes: +For new clones, the following are written under the user's home directory +in XDG-compliant locations instead of inside the Spack prefix: -1. installs -2. where environments are stored -3. cached downloads -4. gpg keys -5. where modules are generated -6. licenses +1. installs -> $data_home/installs (default ~/.local/share/spack/installs) +2. environments -> $data_home/environments +3. cached downloads -> $data_home/downloads +4. gpg keys -> $data_home/gpg, gpg-keys +5. modules -> $data_home +6. licenses -> $data_home/licenses -# XDG compliance +Existing clones with data already in $spack continue to use those legacy +locations. The decision is made once, at startup, by the layout *scheme*. -Spack by default stores user_cache_path and the user config scope in +# Layout schemes + +`etc/spack/defaults/include.yaml` includes one of two scheme yamls based +on `layout_detected("old")` (a helper exposed to include `when:` clauses +by spack.spec.eval_conditional): + +- `etc/spack/defaults/old/config.yaml` — selected when legacy data exists + in $spack/opt, $spack/var/spack/environments, $spack/var/cache, etc. + Sets install_tree:root, environments_root, license_dir, source_cache, + gpg_path, gpg_keys_path to their $spack-local paths, plus + config:locations:* for general substitutions. + +- `etc/spack/defaults/xdg/config.yaml` — selected otherwise. Just sets + config:locations:{home,data,state,cache} using $xdg_*_home + substitutions; base/config.yaml's $data_home-relative defaults then + resolve to the right XDG paths. + +The schemes are pulled in EARLIER in defaults/include.yaml than +defaults/base/config.yaml, giving them higher priority — they override +base for paths that diverge between layouts. + +# Unilateral override + +`layout_detected("old")` returns False (forces xdg) if any of the +following env vars is set: SPACK_DATA_HOME, SPACK_STATE_HOME, +SPACK_CACHE_HOME, SPACK_HOME. Reason: setting one of these is an +intentional opt-in to the new layout; defaulting back to old anyway +would produce a split state where data goes new but envs/installs stay +in $spack. + +config:locations:* set in user/site/system scopes does NOT force the +scheme (those scopes aren't loaded at include-evaluation time). They +still override individual values once loaded. + +# Override priority + +In order from highest to lowest: + +1. SPACK_DATA_HOME / SPACK_STATE_HOME / SPACK_CACHE_HOME / SPACK_HOME + env vars (env-var, also forces scheme). +2. Specific config keys: config:install_tree:root, + config:environments_root, config:license_dir, config:source_cache, + config:gpg_path, config:gpg_keys_path — set in any scope. +3. config:locations:{home,data,state,cache} — redirects everything that + uses $data_home etc. substitutions. +4. The active scheme yaml's defaults. + +User/system config paths bootstrap the config system and are not +themselves in config. They are controlled by SPACK_USER_CONFIG_PATH, +SPACK_SYSTEM_CONFIG_PATH, and SPACK_DISABLE_LOCAL_CONFIG, exactly as +before. + +# Path substitutions + +Available in any string-valued config field: + +- $data_home / $state_home / $cache_home / $spack_home — read + config:locations:* with the env-var overrides applied +- $xdg_data_home / $xdg_state_home / $xdg_cache_home — $XDG_*_HOME or + the freedesktop default (with no /spack suffix). Used in the xdg + scheme yaml. +- $user_cache_path — legacy alias for $state_home +- $spack, $spack_instance_id — instance-specific; unchanged + +# Diagnostic: spack debug paths + +Prints active scheme, each resolved home with its source (env var? +config? which scope?), and every config-driven path with raw template +and resolved value. Replaces the per-command warnings the previous +attempt emitted; users hit this once when something looks wrong, not on +every invocation. + +# Migration: spack migrate + +`spack migrate` copies config from ~/.spack to ~/.config/spack and moves +package repos from ~/.spack/package_repos to +~/.local/state/spack/package_repos. + +`spack migrate --clear` additionally moves the entire ~/.spack into a +backup at ~/.local/share/spack/dotspack_backup. The backup location is +pinned to the XDG default and does NOT follow SPACK_DATA_HOME / +config:locations:data — the backup is a one-time migration artifact, +not legacy state to preserve, and shouldn't move around if the user +later redirects $data_home. + +`spack migrate --restore` reverses --clear from that fixed location. + +`spack migrate --i-need-old-spack` prints guidance for users who have +older Spack instances that can't upgrade and need to keep using ~/.spack. -- We want new spack instances to start using XDG-compliant locations - for these -- For the data moving out of $spack described in the previous section, - we want to move it into XDG-compliant locations - -# Old/new spack instance behavior: smoothing transition - -- If no new spack clones are generated and only old spack clones - exist, then each of them will continue using their old install - location inside of their respective $spack directories - (including after they pull this PR). -- If a new clone starts placing installed packages in ~, and an - old spack instance git pulls this logic, the old instance will - still keep using its own $spack directory for installs, - environments, etc. -- If an old spack instance pulls this PR and then sets any new- - style location variable (an env var like SPACK_HOME, or a config - variable like config:locations:data), spack assumes that it - should use new-style defaults for *all* types of data. - - e.g. if an older spack instance git pulls this logic and - sees that SPACK_STATE_HOME is set, then it will use installs - in ~/.local/share/spack/installs rather than in $spack/opt/spack - (even though STATE_HOME does not control where installs are - placed) - - A user can avoid env vars messing with their older spack - install by adding config:locations:disable_env:true -- Both old and new spack instances will use ~/.spack for config - and as a user_cache_path if they detect it and if the new - locations (~/.config/spack and ~/.local/state/spack) are not - detected -- A warning will be generated when ~/.spack is detected, telling - the user to run `spack migrate` (a new command provided with - this PR). - - However, this PR also contains logic to detect when the user - is explicitly using ~/.spack (e.g. if a config scope - explicitly targets ~/.spack as a backup, or if a user - sets SPACK_USER_CACHE_PATH to ~/.spack) and does not warn - in that case either. - -## General all-or-nothing behavior - -For items that used to live in $spack, they are generally all relocated -together or not at all: - -- e.g. if the old default install tree in $spack/opt/spack contains - installs, then spack also expects to manage environments, gpg keys, - license files, and downloads in their old locations inside of - $spack -- The user config scope and the user_cache_path are also excluded from - this (they didn't live in $spack by default though) - -## `spack migrate` command - -Simplest usage is if you are updating all spack instances to >= 1.2, -in which case you can do: - -`spack migrate --clear` - -- Copies config files from ~/.spack into ~/.config -- Copies package repositories from ~/.spack/package_repos into - ~/.local/state/spack/package_repos -- Moves the entire ~/.spack directory into a backup location into - `$data_home/dotspack_backup` - -If it's descovered that this operation removed something from -~/.spack that an older spack instance needs (i.e. you realized you can't -update all your spack instances), then you can restore it with - -`spack migrate --restore-old-configs` - -# New mechanisms for controlling data location - -This is described in detail in where_spack_writes_data.rst (that -section is focused on how Spack behaves going forward - questions -about how existing spack instances can keep using their old data -is described here). - -# avoiding ~ - -Before this PR, you could do - -``` -export SPACK_USER_CACHE_PATH=x -export SPACK_DISABLE_LOCAL_CONFIG=1 -``` - -and nothing would be read from or written to ~. - -For an older instance of spack that pulls this PR, this will also be -sufficient to prevent writes into ~ (this depends on e48b8c104). - -For a fresh clone of spack using this PR, additional steps are -required to avoid writing into ~: - -- If you set `SPACK_HOME=x SPACK_USER_CONFIG_PATH=x/config`, everything - will be written to x -- Or you can do `SPACK_HOME=x SPACK_DISABLE_LOCAL_CONFIG=1` -- If you set SPACK_DATA_HOME or `config:locations:data` outside of ~, - this will relocate everything that takes up significant space (envs, - installs, cached downloads) - -To completely move everything, in an env you can do: - -``` -include:: [] -config: - locations: - home: x -``` - -Alternatively, users who have write access to $spack can do - -``` -spack config --scope=spack edit include -spack config --scope=spack add config:locations:home:x -``` - -# Crosstalk between spack instances - -* The misc. cache now includes a per-spack component (based on a hash - of the Spack root): `spack_instance_id` - * `config:misc_cache` still exists, so users can share a single - misc cache among multiple instances of spack -* Ideally each individual spack would have a distinct highest-level - write scope, but for now they all share `~/.config/spack` - -# Admin management, pip-installed spack - -- If admins want to provide an upstream to users but avoid using it - themselves, they can use existing `include:` config in the `spack scope`: - - Include a scope that adds `upstreams:x` config - `when: SPACK_ADMIN not in env` - - Include a scope that adds `install_tree:root:x` config - `when: SPACK_ADMIN in env` - - As long as all admins make sure to define this env var, this will - satisfy [5a/b] +# Avoiding ~ + +To prevent any writes into $HOME from a new clone: + +- SPACK_HOME=x SPACK_USER_CONFIG_PATH=x/config (everything to x) +- or SPACK_HOME=x SPACK_DISABLE_LOCAL_CONFIG=1 +- or `config:locations:home: x` in an env's `spack.yaml` (with + `include: []` to avoid pulling in user config) + +The pre-1.2 isolation incantation still works for old instances that +pull this code: + +- SPACK_USER_CACHE_PATH=x SPACK_DISABLE_LOCAL_CONFIG=1 + +# Crosstalk between Spack instances + +* The misc. cache key (config:misc_cache) includes $spack_instance_id + (a hash of the Spack root), so co-installed Spack instances have + separate misc caches by default. Override config:misc_cache to share + one. +* Multiple Spack instances all writing to ~/.config/spack is still + possible — admins who want isolation between instances should set + config:locations:* in a site/system scope. + +# Admin management, pip-installed Spack + +- Admins can ship users an upstream + a curated set of config via + `include:` in the `spack` scope, using `when: SPACK_ADMIN not in env` + / `when: SPACK_ADMIN in env` to distinguish admin from user runs. +- A pip-installed Spack works as long as $spack (the install prefix) is + not written to. Read-only prefixes are now the default; users land in + the xdg scheme automatically. From 7b3f23788d3a5a98b65d0ee644badd7f2d02c02d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 24 May 2026 08:52:19 -0700 Subject: [PATCH 504/506] defaults: simplify "users can override" comment in 9 yaml headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace two slightly different phrasings with one shorter form that doesn't lean on "instance of this config" — which becomes confusing now that "instance" also means "Spack instance". --- etc/spack/defaults/base/concretizer.yaml | 4 ++-- etc/spack/defaults/base/config.yaml | 4 ++-- etc/spack/defaults/base/modules.yaml | 4 ++-- etc/spack/defaults/base/packages.yaml | 4 ++-- etc/spack/defaults/base/repos.yaml | 4 ++-- etc/spack/defaults/darwin/modules.yaml | 4 ++-- etc/spack/defaults/darwin/packages.yaml | 4 ++-- etc/spack/defaults/linux/modules.yaml | 4 ++-- etc/spack/defaults/windows/packages.yaml | 4 ++-- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/etc/spack/defaults/base/concretizer.yaml b/etc/spack/defaults/base/concretizer.yaml index afb70e615c4ec4..aff364c1c42f87 100644 --- a/etc/spack/defaults/base/concretizer.yaml +++ b/etc/spack/defaults/base/concretizer.yaml @@ -5,8 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override part or all of these settings by defining their -# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # ------------------------------------------------------------------------- concretizer: # Whether to consider installed packages or packages from buildcaches when diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index 8339173ed9d642..4e2c0f0249df67 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -5,8 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override part or all of these settings by defining their -# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # # ------------------------------------------------------------------------- config: diff --git a/etc/spack/defaults/base/modules.yaml b/etc/spack/defaults/base/modules.yaml index c2a23626b5120c..e78264073fedd5 100644 --- a/etc/spack/defaults/base/modules.yaml +++ b/etc/spack/defaults/base/modules.yaml @@ -5,8 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by defining their -# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # ------------------------------------------------------------------------- modules: # This maps paths in the package install prefix to environment variables diff --git a/etc/spack/defaults/base/packages.yaml b/etc/spack/defaults/base/packages.yaml index 9e1e8e72ef8d79..5b8375f6b016e1 100644 --- a/etc/spack/defaults/base/packages.yaml +++ b/etc/spack/defaults/base/packages.yaml @@ -5,8 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by defining their -# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # ------------------------------------------------------------------------- packages: all: diff --git a/etc/spack/defaults/base/repos.yaml b/etc/spack/defaults/base/repos.yaml index 2ac9c78c4c5d51..9e32fa2ce5c084 100644 --- a/etc/spack/defaults/base/repos.yaml +++ b/etc/spack/defaults/base/repos.yaml @@ -2,8 +2,8 @@ # This is the default spack repository configuration. It includes the # builtin spack package repository. # -# Users can override these settings by defining their -# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # ------------------------------------------------------------------------- repos: builtin: diff --git a/etc/spack/defaults/darwin/modules.yaml b/etc/spack/defaults/darwin/modules.yaml index 876feed8998191..7a910d874f0f6b 100644 --- a/etc/spack/defaults/darwin/modules.yaml +++ b/etc/spack/defaults/darwin/modules.yaml @@ -5,8 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by defining their -# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # ------------------------------------------------------------------------- modules: prefix_inspections: diff --git a/etc/spack/defaults/darwin/packages.yaml b/etc/spack/defaults/darwin/packages.yaml index 5c829934590e1b..5e2df9a6957777 100644 --- a/etc/spack/defaults/darwin/packages.yaml +++ b/etc/spack/defaults/darwin/packages.yaml @@ -5,8 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by defining their -# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # ------------------------------------------------------------------------- packages: all: diff --git a/etc/spack/defaults/linux/modules.yaml b/etc/spack/defaults/linux/modules.yaml index cb5511f76940ba..2da7ef5e404af0 100644 --- a/etc/spack/defaults/linux/modules.yaml +++ b/etc/spack/defaults/linux/modules.yaml @@ -5,7 +5,7 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by defining their -# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # ------------------------------------------------------------------------- modules: {} diff --git a/etc/spack/defaults/windows/packages.yaml b/etc/spack/defaults/windows/packages.yaml index 67d38e01480a25..77e5c606b73cd2 100644 --- a/etc/spack/defaults/windows/packages.yaml +++ b/etc/spack/defaults/windows/packages.yaml @@ -5,8 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by defining their -# own instance of this config (see https://spack.readthedocs.io/en/latest/configuration.html). +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # ------------------------------------------------------------------------- packages: all: From f97388658bd35902320aaa571c3bc7ad969522da Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 24 May 2026 08:55:27 -0700 Subject: [PATCH 505/506] env: restore default_env_path() as a monkeypatchable default env_root_path now calls config.get with default=default_env_path() (the runtime equivalent of the base config scope's $data_home/environments default). mutable_mock_env_path monkeypatches default_env_path so the mock survives test fixture composition where another config fixture resets the active scope after mutable_mock_env_path runs. Fixes test_config_remove_from_env in the full suite. --- lib/spack/spack/environment/environment.py | 21 ++++++++++++++------- lib/spack/spack/test/conftest.py | 3 +++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index 53f3c4d45b41ab..56ed3342c97dc6 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -94,15 +94,22 @@ env_subdir_name = ".spack-env" -def env_root_path() -> str: - """Resolve config:environments_root with substitutions and ~ expansion. +def default_env_path() -> str: + """Default for ``config:environments_root`` when no scope sets it. - The base default scope sets this to ``$data_home/environments``; tests - using a stripped-down config may leave it unset, so fall back to the - same expression here. + The base default scope sets the config key to ``$data_home/environments``; + this function is the runtime equivalent. Tests that mock the + environments directory monkeypatch this function (see the + ``mutable_mock_env_path`` fixture). """ - root = spack.config.get("config:environments_root") or "$data_home/environments" - return spack.util.path.canonicalize_path(root) + return spack.util.path.canonicalize_path("$data_home/environments") + + +def env_root_path() -> str: + """Resolve config:environments_root with substitutions and ~ expansion.""" + return spack.util.path.canonicalize_path( + spack.config.get("config:environments_root", default=default_env_path()) + ) def environment_name(path: Union[str, pathlib.Path]) -> str: diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 8403e34946cf97..e244fd89695246 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -1979,6 +1979,9 @@ def mutable_mock_env_path(tmp_path: Path, mutable_config, monkeypatch): """Fixture for mocking the internal spack environments directory.""" mock_path = tmp_path / "mock-env-path" mutable_config.set("config:environments_root", str(mock_path)) + # Also pin the runtime default so tests that compose with other config + # fixtures (which may reset the active config) still see the mock. + monkeypatch.setattr(ev.environment, "default_env_path", lambda: str(mock_path)) return mock_path From 4a788c9094ec95dec089758371bf85f9a379848b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 24 May 2026 18:22:00 -0700 Subject: [PATCH 506/506] modules: drop $default_modules_base; move root override into old scheme MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit base/modules.yaml previously referenced $default_modules_base — an opaque substitution that the earlier refactor removed without updating this file, so module root config silently resolved to the literal string. Switch to $data_home/{modules,lmod} (matching the pattern used by install_tree, environments_root, gpg_path, etc.) and add a new defaults/old/modules.yaml that overrides those roots to $spack/share/spack/{modules,lmod} for legacy layouts. Also strip the last $default_install_root / $default_envs_root / $default_download_root references from the user-facing docs (configuration_basics.rst, config_yaml.rst) so `spack config blame` output is readable end-to-end: every substitution a user sees now names what it actually refers to ($spack, $data_home, $state_home, etc.) rather than an opaque "default" placeholder. --- etc/spack/defaults/base/modules.yaml | 7 ++++--- etc/spack/defaults/old/modules.yaml | 14 ++++++++++++++ lib/spack/docs/config_yaml.rst | 2 +- lib/spack/docs/configuration_basics.rst | 12 +++++------- 4 files changed, 24 insertions(+), 11 deletions(-) create mode 100644 etc/spack/defaults/old/modules.yaml diff --git a/etc/spack/defaults/base/modules.yaml b/etc/spack/defaults/base/modules.yaml index e78264073fedd5..047dee3d17ec37 100644 --- a/etc/spack/defaults/base/modules.yaml +++ b/etc/spack/defaults/base/modules.yaml @@ -31,10 +31,11 @@ modules: # These are configurations for the module set named "default" default: - # Where to install modules + # Where to install modules. Overridden to $spack/share/spack/{modules,lmod} + # by the old-layout scheme yaml. roots: - tcl: $default_modules_base/modules - lmod: $default_modules_base/lmod + tcl: $data_home/modules + lmod: $data_home/lmod # What type of modules to use ("tcl" and/or "lmod") enable: [] diff --git a/etc/spack/defaults/old/modules.yaml b/etc/spack/defaults/old/modules.yaml new file mode 100644 index 00000000000000..6111eb76a91d93 --- /dev/null +++ b/etc/spack/defaults/old/modules.yaml @@ -0,0 +1,14 @@ +# ------------------------------------------------------------------------- +# Module root overrides for the legacy $spack-local layout. +# +# Selected by etc/spack/defaults/include.yaml when an existing Spack +# instance has installs etc. under $spack. base/modules.yaml puts module +# files under $data_home/{modules,lmod} (the xdg default); old layout +# instead puts them in $spack/share/spack so a pre-1.2 clone that pulls +# this PR doesn't lose track of its existing modules. +# ------------------------------------------------------------------------- +modules: + default: + roots: + tcl: $spack/share/spack/modules + lmod: $spack/share/spack/lmod diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index e3600f07533cb6..498600c24a3893 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -59,7 +59,7 @@ For example: config: install_tree: - root: $default_install_root + root: $data_home/installs projections: all: "{name}/{version}/{hash:16}" diff --git a/lib/spack/docs/configuration_basics.rst b/lib/spack/docs/configuration_basics.rst index 9ac64b962ef730..92102c257693b3 100644 --- a/lib/spack/docs/configuration_basics.rst +++ b/lib/spack/docs/configuration_basics.rst @@ -45,7 +45,7 @@ Here is an example ``config.yaml`` file: config: install_tree: - root: $default_install_root + root: $data_home/installs build_stage: - $tempdir/$user/spack-stage - $cache_home/stage @@ -319,7 +319,7 @@ If your configurations look like this: config: install_tree: - root: $default_install_root + root: $data_home/installs build_stage: - $tempdir/$user/spack-stage - $cache_home/stage @@ -541,8 +541,6 @@ The following are additional special variables that describe locations where spa * ``$state_home`` * ``$cache_home`` * ``$spack_home`` -* ``$default_install_root`` -* ``$default_envs_root`` These are described in more detail in :ref:`config-file-data-variables`. @@ -608,14 +606,14 @@ For example, to see the fully merged ``config.yaml``, you can type: dirty: false build_jobs: 8 install_tree: - root: $default_install_root + root: $data_home/installs template_dirs: - $spack/templates directory_layout: {architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash} build_stage: - $tempdir/$user/spack-stage - $cache_home/stage - source_cache: $default_download_root + source_cache: $data_home/downloads misc_cache: $state_home/$spack_instance_id/cache locks: true @@ -657,7 +655,7 @@ If you do not know why Spack is behaving a certain way, this command can help yo /home/myuser/spack/etc/spack/defaults/config.yaml:49 build_stage: /home/myuser/spack/etc/spack/defaults/config.yaml:50 - $tempdir/$user/spack-stage /home/myuser/spack/etc/spack/defaults/config.yaml:51 - $cache_home/stage - /home/myuser/spack/etc/spack/defaults/config.yaml:57 source_cache: $default_download_root + /home/myuser/spack/etc/spack/defaults/config.yaml:57 source_cache: $data_home/downloads /home/myuser/spack/etc/spack/defaults/config.yaml:62 misc_cache: $state_home/$spack_instance_id/cache /home/myuser/spack/etc/spack/defaults/config.yaml:86 locks: True