diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index e76a3dec4771ca..859137f7b36dc3 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -48,7 +48,7 @@ jobs: spack bootstrap disable github-actions-v2 spack bootstrap disable github-actions-v0.6 spack solve zlib - tree ~/.spack/bootstrap/store/ + tree $(spack bootstrap root) clingo-sources: if: github.repository == 'spack/spack' @@ -75,7 +75,7 @@ jobs: spack bootstrap disable github-actions-v0.6 export PATH="$(brew --prefix bison)/bin:$(brew --prefix cmake)/bin:$PATH" spack solve zlib - tree ~/.spack/bootstrap/store/ + tree $(spack bootstrap root) gnupg-sources: if: github.repository == 'spack/spack' @@ -104,7 +104,7 @@ jobs: spack bootstrap disable github-actions-v2 spack bootstrap disable github-actions-v0.6 spack gpg list - tree ~/.spack/bootstrap/store/ + tree $(spack bootstrap root) from-binaries: if: github.repository == 'spack/spack' @@ -152,13 +152,13 @@ jobs: fi spack solve zlib done - tree ~/.spack/bootstrap/store + tree $(spack bootstrap root) - name: Bootstrap GnuPG run: | . share/spack/setup-env.sh spack config add config:installer:new spack gpg list - tree ~/.spack/bootstrap/store/ + tree $(spack bootstrap root) windows: if: github.repository == 'spack/spack' @@ -182,13 +182,13 @@ jobs: spack bootstrap disable github-actions-v0.6 spack -d solve zlib ./share/spack/qa/validate_last_exit.ps1 - tree $env:userprofile/.spack/bootstrap/store/ + tree $(spack bootstrap root) - name: Bootstrap GnuPG run: | ./share/spack/setup-env.ps1 spack -d gpg list ./share/spack/qa/validate_last_exit.ps1 - tree $env:userprofile/.spack/bootstrap/store/ + tree $(spack bootstrap root) dev-bootstrap: runs-on: ubuntu-latest diff --git a/etc/spack/defaults/base/concretizer.yaml b/etc/spack/defaults/base/concretizer.yaml index 51c8a20013af9f..aff364c1c42f87 100644 --- a/etc/spack/defaults/base/concretizer.yaml +++ b/etc/spack/defaults/base/concretizer.yaml @@ -5,9 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by editing -# `$SPACK_ROOT/etc/spack/concretizer.yaml`, `~/.spack/concretizer.yaml`, -# or by adding a `concretizer:` section to an environment. +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # ------------------------------------------------------------------------- concretizer: # Whether to consider installed packages or packages from buildcaches when diff --git a/etc/spack/defaults/base/config.yaml b/etc/spack/defaults/base/config.yaml index 80d8deb88e70e4..4e2c0f0249df67 100644 --- a/etc/spack/defaults/base/config.yaml +++ b/etc/spack/defaults/base/config.yaml @@ -5,19 +5,20 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by editing files in: -# -# * $SPACK_ROOT/etc/spack/ - Spack instance settings -# * ~/.spack/ - user settings -# * $SPACK_ROOT/etc/spack/site - Spack "site" settings. Like instance settings -# but lower priority then user settings. +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # # ------------------------------------------------------------------------- config: # This is the path to the root of the Spack install tree. # You can use $spack here to refer to the root of the spack instance. install_tree: - root: $spack/opt/spack + # If layout_detected("old"), this is overridden to $spack/opt/spack by + # etc/spack/defaults/old/config.yaml (the scheme yaml chosen by + # defaults/include.yaml when legacy data is present in $spack). + # Otherwise $data_home resolves to ~/.local/share/spack (or + # $XDG_DATA_HOME/spack, $SPACK_DATA_HOME, etc.). + root: $data_home/installs projections: all: "{architecture.platform}-{architecture.target}/{name}-{version}-{hash}" # install_tree can include an optional padded length (int or boolean) @@ -33,8 +34,9 @@ config: template_dirs: - $spack/share/spack/templates - # Directory where licenses should be located - license_dir: $spack/etc/spack/licenses + # Directory where licenses should be located. + # Overridden to $spack/etc/spack/licenses by the old-layout scheme yaml. + license_dir: $data_home/licenses # Temporary locations Spack can try to use for builds. # @@ -48,11 +50,9 @@ config: # to specify `$user_cache_path/stage`, which ensures each user builds in their # home directory. # - # A more traditional path uses the value of `$spack/var/spack/stage`, which - # builds directly inside Spack's instance without staging them in a - # temporary space. Problems with specifying a path inside a Spack instance - # are that it precludes its use as a system package and its ability to be - # pip installable. + # If you do Spack builds inside of allocations and want stages to persist + # after those allocations end, you will want to override this and avoid using + # a temporary directory. # # In Spack environment files, chaining onto existing system Spack # installations, the $env variable can be used to download, cache and build @@ -68,8 +68,7 @@ config: # identifies Spack staging to avoid accidentally wiping out non-Spack work. build_stage: - $tempdir/$user/spack-stage - - $user_cache_path/stage - # - $spack/var/spack/stage + - $cache_home/stage # Naming format for individual stage directories stage_name: "spack-stage-{name}-{version}-{hash}" @@ -77,20 +76,28 @@ config: # Directory in which to run tests and store test results. # Tests will be stored in directories named by date/time and package # name/hash. - test_stage: $user_cache_path/test + test_stage: $state_home/test + # Cache directory for already downloaded source tarballs and archived # repositories. This can be purged with `spack clean --downloads`. - source_cache: $spack/var/spack/cache + # Overridden to $spack/var/cache by the old-layout scheme yaml. + source_cache: $data_home/downloads + + # Directories Spack uses for its gpg keyring and pre-imported public keys. + # Overridden to $spack-local paths by the old-layout scheme yaml. + gpg_path: $data_home/gpg + gpg_keys_path: $data_home/gpg-keys - ## Directory where spack managed environments are created and stored - # environments_root: $spack/var/spack/environments + ## Directory where spack managed environments are created and stored. + # Overridden to $spack/var/spack/environments by the old-layout scheme yaml. + environments_root: $data_home/environments # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` - misc_cache: $user_cache_path/cache + misc_cache: $state_home/$spack_instance_id/cache # Abort downloads after this many seconds if not data is received. # Setting this to 0 will disable the timeout. diff --git a/etc/spack/defaults/base/modules.yaml b/etc/spack/defaults/base/modules.yaml index 75ec3661174378..047dee3d17ec37 100644 --- a/etc/spack/defaults/base/modules.yaml +++ b/etc/spack/defaults/base/modules.yaml @@ -5,13 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/modules.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.spack/modules.yaml +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # ------------------------------------------------------------------------- modules: # This maps paths in the package install prefix to environment variables @@ -36,10 +31,11 @@ modules: # These are configurations for the module set named "default" default: - # Where to install modules + # Where to install modules. Overridden to $spack/share/spack/{modules,lmod} + # by the old-layout scheme yaml. roots: - tcl: $spack/share/spack/modules - lmod: $spack/share/spack/lmod + tcl: $data_home/modules + lmod: $data_home/lmod # What type of modules to use ("tcl" and/or "lmod") enable: [] diff --git a/etc/spack/defaults/base/packages.yaml b/etc/spack/defaults/base/packages.yaml index ad86568c4e894b..5b8375f6b016e1 100644 --- a/etc/spack/defaults/base/packages.yaml +++ b/etc/spack/defaults/base/packages.yaml @@ -5,13 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/packages.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.spack/packages.yaml +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # ------------------------------------------------------------------------- packages: all: diff --git a/etc/spack/defaults/base/repos.yaml b/etc/spack/defaults/base/repos.yaml index 48a9469d6df3b9..9e32fa2ce5c084 100644 --- a/etc/spack/defaults/base/repos.yaml +++ b/etc/spack/defaults/base/repos.yaml @@ -2,13 +2,8 @@ # This is the default spack repository configuration. It includes the # builtin spack package repository. # -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/repos.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.spack/repos.yaml +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # ------------------------------------------------------------------------- repos: builtin: diff --git a/etc/spack/defaults/darwin/modules.yaml b/etc/spack/defaults/darwin/modules.yaml index b913d1029ea75b..7a910d874f0f6b 100644 --- a/etc/spack/defaults/darwin/modules.yaml +++ b/etc/spack/defaults/darwin/modules.yaml @@ -5,13 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/modules.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.spack/modules.yaml +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # ------------------------------------------------------------------------- modules: prefix_inspections: diff --git a/etc/spack/defaults/darwin/packages.yaml b/etc/spack/defaults/darwin/packages.yaml index 32a3b940409014..5e2df9a6957777 100644 --- a/etc/spack/defaults/darwin/packages.yaml +++ b/etc/spack/defaults/darwin/packages.yaml @@ -5,13 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/packages.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.spack/packages.yaml +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # ------------------------------------------------------------------------- packages: all: diff --git a/etc/spack/defaults/include.yaml b/etc/spack/defaults/include.yaml index d03fb044b8be19..beb03ffcb4ba17 100644 --- a/etc/spack/defaults/include.yaml +++ b/etc/spack/defaults/include.yaml @@ -3,5 +3,17 @@ include: - path: "${platform}" optional: true - # base packages.yaml overridable by platform-specific settings + # Layout scheme: chooses old (data lives under $spack) or xdg (data lives + # under ~/.local/share/spack etc.). Earlier entries override later ones, + # so the scheme can supersede `base` for paths that diverge between + # layouts. See lib/spack/docs/where_spack_writes_data.rst. + - path: old + when: 'layout_detected("old")' + optional: true + + - path: xdg + when: 'not layout_detected("old")' + optional: true + + # base config (lowest priority among defaults) - path: base diff --git a/etc/spack/defaults/linux/modules.yaml b/etc/spack/defaults/linux/modules.yaml index a80f87b16ab400..2da7ef5e404af0 100644 --- a/etc/spack/defaults/linux/modules.yaml +++ b/etc/spack/defaults/linux/modules.yaml @@ -5,12 +5,7 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/modules.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.spack/modules.yaml +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # ------------------------------------------------------------------------- modules: {} diff --git a/etc/spack/defaults/old/config.yaml b/etc/spack/defaults/old/config.yaml new file mode 100644 index 00000000000000..272a523d60f48a --- /dev/null +++ b/etc/spack/defaults/old/config.yaml @@ -0,0 +1,29 @@ +# ------------------------------------------------------------------------- +# Legacy `$spack`-local layout defaults. +# +# Selected by `etc/spack/defaults/include.yaml` when an existing Spack +# instance has installs, environments, gpg keys, etc. under `$spack`. Keeps +# everything in those legacy locations so pulling a newer Spack into an old +# clone is non-disruptive. +# +# The four `config:locations:*` keys give the rest of the default config +# (which uses `$data_home`, `$state_home`, `$cache_home` substitutions) a +# coherent answer for old layout — but old layout doesn't actually share a +# single root, so individual path keys below override the substitutions +# directly. +# ------------------------------------------------------------------------- +config: + locations: + home: "~" + data: $spack + state: "~/.spack" + cache: $spack/var + + install_tree: + root: $spack/opt/spack + + environments_root: $spack/var/spack/environments + license_dir: $spack/etc/spack/licenses + source_cache: $spack/var/cache + gpg_path: $spack/opt/spack/gpg + gpg_keys_path: $spack/var/gpg diff --git a/etc/spack/defaults/old/modules.yaml b/etc/spack/defaults/old/modules.yaml new file mode 100644 index 00000000000000..6111eb76a91d93 --- /dev/null +++ b/etc/spack/defaults/old/modules.yaml @@ -0,0 +1,14 @@ +# ------------------------------------------------------------------------- +# Module root overrides for the legacy $spack-local layout. +# +# Selected by etc/spack/defaults/include.yaml when an existing Spack +# instance has installs etc. under $spack. base/modules.yaml puts module +# files under $data_home/{modules,lmod} (the xdg default); old layout +# instead puts them in $spack/share/spack so a pre-1.2 clone that pulls +# this PR doesn't lose track of its existing modules. +# ------------------------------------------------------------------------- +modules: + default: + roots: + tcl: $spack/share/spack/modules + lmod: $spack/share/spack/lmod diff --git a/etc/spack/defaults/windows/packages.yaml b/etc/spack/defaults/windows/packages.yaml index f18ea28023f586..77e5c606b73cd2 100644 --- a/etc/spack/defaults/windows/packages.yaml +++ b/etc/spack/defaults/windows/packages.yaml @@ -5,13 +5,8 @@ # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/packages.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.spack/packages.yaml +# Users can override these settings in their own configuration files. +# See https://spack.readthedocs.io/en/latest/configuration.html. # ------------------------------------------------------------------------- packages: all: diff --git a/etc/spack/defaults/xdg/config.yaml b/etc/spack/defaults/xdg/config.yaml new file mode 100644 index 00000000000000..187bf6d8ea090a --- /dev/null +++ b/etc/spack/defaults/xdg/config.yaml @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# XDG-compliant layout defaults. +# +# Selected by `etc/spack/defaults/include.yaml` when no legacy `$spack`-local +# data is detected. Sets the four `config:locations:*` keys that the rest of +# the default config uses via `$data_home`, `$state_home`, and `$cache_home` +# substitutions. +# +# Spack-specific env vars (SPACK_DATA_HOME etc.) still override these at +# runtime; XDG env vars are picked up here via the `$xdg_*_home` subs. +# ------------------------------------------------------------------------- +config: + locations: + home: "~" + data: $xdg_data_home/spack + state: $xdg_state_home/spack + cache: $xdg_cache_home/spack diff --git a/etc/spack/include.yaml b/etc/spack/include.yaml index 4bdc37ccb41895..066b227ad8567f 100644 --- a/etc/spack/include.yaml +++ b/etc/spack/include.yaml @@ -2,7 +2,8 @@ include: # user configuration scope - name: "user" path_override_env_var: SPACK_USER_CONFIG_PATH - path: "~/.spack" + path: "~/.config/spack/" + backwards_compat: "~/.spack/" optional: true prefer_modify: true when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' diff --git a/lib/spack/docs/build_systems/inteloneapipackage.rst b/lib/spack/docs/build_systems/inteloneapipackage.rst index 29722973f32490..440d3c05560be2 100644 --- a/lib/spack/docs/build_systems/inteloneapipackage.rst +++ b/lib/spack/docs/build_systems/inteloneapipackage.rst @@ -142,7 +142,7 @@ Before 2024, the directory structure was different: Libraries ^^^^^^^^^ -If you want Spack to use oneMKL that you have installed without Spack in the default location, then add the following to ``~/.spack/packages.yaml``, adjusting the version as appropriate: +If you want Spack to use oneMKL that you have installed without Spack in the default location, then add the following to ``~/.config/spack/packages.yaml``, adjusting the version as appropriate: .. code-block:: yaml diff --git a/lib/spack/docs/chain.rst b/lib/spack/docs/chain.rst index f4b6a74d9eb634..ec572e7968047f 100644 --- a/lib/spack/docs/chain.rst +++ b/lib/spack/docs/chain.rst @@ -16,12 +16,18 @@ To register the other Spack instance, you can add it as an entry to ``upstreams. .. code-block:: yaml upstreams: - spack-instance-1: - install_tree: /path/to/other/spack/opt/spack - spack-instance-2: - install_tree: /path/to/another/spack/opt/spack + install-tree-1: + install_tree: /path/to/another/spack/install/tree + install-tree-2: + install_tree: /path/to/yet/another/spack/install/tree + +The ``install_tree`` must point to a directory containing a Spack install tree, as defined in :ref:`config.yaml `. +For a given instance of Spack, you can determine the directory to add here by running ``spack location --install-root`` in that instance, or with ``spack config get config``. + +.. note: -The ``install_tree`` must point to the ``opt/spack`` directory inside of the Spack base directory, or the location of the ``install_tree`` defined in :ref:`config.yaml `. + The ``install_tree`` directory is the directory containing the ``.spack-db`` hidden directory. + This can be used to confirm you have the correct directory level listed for your upstream. Once the upstream Spack instance has been added, ``spack find`` will automatically check the upstream instance when querying installed packages, and new package installations for the local Spack installation will use any dependencies that are installed in the upstream instance. diff --git a/lib/spack/docs/config_yaml.rst b/lib/spack/docs/config_yaml.rst index e2d8daf158c026..498600c24a3893 100644 --- a/lib/spack/docs/config_yaml.rst +++ b/lib/spack/docs/config_yaml.rst @@ -18,14 +18,14 @@ You can see the default settings by looking at ``etc/spack/defaults/config.yaml` .. literalinclude:: _spack_root/etc/spack/defaults/base/config.yaml :language: yaml -These settings can be overridden in ``etc/spack/config.yaml`` or ``~/.spack/config.yaml``. +These settings can be overridden in ``etc/spack/config.yaml``, or ``~/.config/spack/config.yaml``, or in an active environment. See :ref:`configuration-scopes` for details. ``install_tree:root`` --------------------- The location where Spack will install packages and their dependencies. -The default is ``$spack/opt/spack``. +The default is ``$data_home/installs``. ``projections`` --------------- @@ -59,7 +59,7 @@ For example: config: install_tree: - root: $spack/opt/spack + root: $data_home/installs projections: all: "{name}/{version}/{hash:16}" @@ -86,14 +86,12 @@ By default, Spack's ``build_stage`` is configured like this: build_stage: - $tempdir/$user/spack-stage - - ~/.spack/stage + - $cache_home/stage This can be an ordered list of paths that Spack should search when trying to find a temporary directory for the build stage. The list is searched in order, and Spack will use the first directory to which it has write access. -Specifying `~/.spack/stage` first will ensure each user builds in their home directory. -The historic Spack stage path `$spack/var/spack/stage` will build directly inside the Spack instance. -See :ref:`config-file-variables` for more on ``$tempdir`` and ``$spack``. +Specifying `$cache_home` first will ensure each user builds in their home directory, or whatever the user overrides ``XDG_CACHE_HOME`` to be - see :ref:`config-file-variables` and :ref:`config-file-data-variables` for more on ``$tempdir``, XDG variables, and ``$spack``. When Spack builds a package, it creates a temporary directory within the ``build_stage``. After the package is successfully installed, Spack deletes the temporary directory it used to build. @@ -107,7 +105,7 @@ Unsuccessful builds are not deleted, but you can manually purge them with ``spac -------------------- Location to cache downloaded tarballs and repositories. -By default, these are stored in ``$spack/var/spack/cache``. +By default, these are stored in ``$data_home/downloads``. These are stored indefinitely by default and can be purged with ``spack clean --downloads``. .. _Misc Cache: @@ -116,14 +114,22 @@ These are stored indefinitely by default and can be purged with ``spack clean -- -------------------- Temporary directory to store long-lived cache files, such as indices of packages available in repositories. -Defaults to ``~/.spack/cache``. +Defaults to ``$state_home/$spack_instance_id/spack``. Can be purged with ``spack clean --misc-cache``. -In some cases, e.g., if you work with many Spack instances or many different versions of Spack, it makes sense to have a cache per instance or per version. -You can do that by changing the value to either: +If you have several Spack instances with the same version and want them to share this cache, you can use ``~/.spack/$spack_short_version/cache``. -* ``~/.spack/$spack_instance_id/cache`` for per-instance caches, or -* ``~/.spack/$spack_short_version/cache`` for per-spack-version caches. +``locations`` +-------------------- + +The ``locations`` section includes variables that control where spack stores data it generates, as described in :ref:`config-file-data-variables`: + +* ``locations:data`` +* ``locations:state`` +* ``locations:cache`` +* ``locations:home`` + +The additional flag setting ``locations:disable_env`` will prevent the environment variables described in :ref:`config-file-data-variables` from influencing these locations. ``verify_ssl`` -------------------- diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration_basics.rst similarity index 92% rename from lib/spack/docs/configuration.rst rename to lib/spack/docs/configuration_basics.rst index 8fe50cd9ded113..92102c257693b3 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration_basics.rst @@ -9,8 +9,15 @@ .. _configuration: +Configuration Basics +==================== + +Spack's behavior can be customized with YAML files that it searches for in specific locations, and also by setting environment variables. +You can customize Spack's behavior by modifying these files, and you can also tell it to look for files in custom locations. +This section covers how the configuration system generally works (e.g. how precedence is established when two files configure different values for the same setting), and later sections cover specific configuration settings (e.g. how to control where installs are placed). + Configuration Files -=================== +------------------- Spack has many configuration files. Here is a quick list of them, in case you want to skip directly to specific docs: @@ -38,10 +45,10 @@ Here is an example ``config.yaml`` file: config: install_tree: - root: $spack/opt/spack + root: $data_home/installs build_stage: - $tempdir/$user/spack-stage - - ~/.spack/stage + - $cache_home/stage Each Spack configuration file is nested under a top-level section corresponding to its name. So, ``config.yaml`` starts with ``config:``, ``mirrors.yaml`` starts with ``mirrors:``, etc. @@ -79,7 +86,7 @@ From lowest to highest precedence: Settings here affect all instances of Spack running with the same Python installation. This scope takes higher precedence than site, system, and default scopes. -#. **user**: Stored in the home directory: ``~/.spack/``. +#. **user**: Stored in the home directory: ``~/.local/config/spack/``. These settings affect all instances of Spack and take higher precedence than site, system, plugin, or defaults scopes. #. **spack**: Stored in ``$(prefix)/etc/spack/``. @@ -140,6 +147,8 @@ If multiple scopes are provided: #. Each must be preceded with the ``--config-scope`` or ``-C`` flag. #. They must be ordered from lowest to highest precedence. + +""""""""""""""""""""""""""""""""""""""""""" Example: scopes for release and development """"""""""""""""""""""""""""""""""""""""""" @@ -310,14 +319,14 @@ If your configurations look like this: config: install_tree: - root: $spack/opt/spack + root: $data_home/installs build_stage: - $tempdir/$user/spack-stage - - ~/.spack/stage + - $cache_home/stage .. code-block:: yaml - :caption: ``~/.spack/config.yaml`` + :caption: ``~/.config/spack/config.yaml`` :name: code-example-user-config-yaml config: @@ -337,7 +346,7 @@ You can see the final, combined configuration with the ``spack config get `_ variable. * ``$user``: name of the current user -* ``$user_cache_path``: user cache directory (``~/.spack`` unless :ref:`overridden `) * ``$architecture``: the architecture triple of the current host, as detected by Spack. * ``$arch``: alias for ``$architecture``. * ``$platform``: the platform of the current host, as detected by Spack. @@ -523,11 +531,19 @@ These are: * ``$date``: the current date in the format YYYY-MM-DD * ``$spack_short_version``: the Spack version truncated to the first components. - Note that, as with shell variables, you can write these as ``$varname`` or with braces to distinguish the variable from surrounding characters: ``${varname}``. Their names are also case insensitive, meaning that ``$SPACK`` works just as well as ``$spack``. These special variables are substituted first, so any environment variables with the same name will not be used. +The following are additional special variables that describe locations where spack writes data: + +* ``$data_home`` +* ``$state_home`` +* ``$cache_home`` +* ``$spack_home`` + +These are described in more detail in :ref:`config-file-data-variables`. + Environment variables ^^^^^^^^^^^^^^^^^^^^^ @@ -590,16 +606,15 @@ For example, to see the fully merged ``config.yaml``, you can type: dirty: false build_jobs: 8 install_tree: - root: $spack/opt/spack + root: $data_home/installs template_dirs: - $spack/templates directory_layout: {architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash} build_stage: - $tempdir/$user/spack-stage - - ~/.spack/stage - - $spack/var/spack/stage - source_cache: $spack/var/spack/cache - misc_cache: ~/.spack/cache + - $cache_home/stage + source_cache: $data_home/downloads + misc_cache: $state_home/$spack_instance_id/cache locks: true Likewise, this will show the fully merged ``packages.yaml``: @@ -639,10 +654,9 @@ If you do not know why Spack is behaving a certain way, this command can help yo /home/myuser/spack/etc/spack/defaults/config.yaml:28 directory_layout: {architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash} /home/myuser/spack/etc/spack/defaults/config.yaml:49 build_stage: /home/myuser/spack/etc/spack/defaults/config.yaml:50 - $tempdir/$user/spack-stage - /home/myuser/spack/etc/spack/defaults/config.yaml:51 - ~/.spack/stage - /home/myuser/spack/etc/spack/defaults/config.yaml:52 - $spack/var/spack/stage - /home/myuser/spack/etc/spack/defaults/config.yaml:57 source_cache: $spack/var/spack/cache - /home/myuser/spack/etc/spack/defaults/config.yaml:62 misc_cache: ~/.spack/cache + /home/myuser/spack/etc/spack/defaults/config.yaml:51 - $cache_home/stage + /home/myuser/spack/etc/spack/defaults/config.yaml:57 source_cache: $data_home/downloads + /home/myuser/spack/etc/spack/defaults/config.yaml:62 misc_cache: $state_home/$spack_instance_id/cache /home/myuser/spack/etc/spack/defaults/config.yaml:86 locks: True You can see above that the ``build_jobs`` and ``debug`` settings are built-in and are not overridden by a configuration file. @@ -657,22 +671,11 @@ Overriding Local Configuration Spack's ``system`` and ``user`` scopes provide ways for administrators and users to set global defaults for all Spack instances, but for use cases where one wants a clean Spack installation, these scopes can be undesirable. For example, users may want to opt out of global system configuration, or they may want to ignore their own home directory settings when running in a continuous integration environment. -Spack also, by default, keeps various caches and user data in ``~/.spack``, but users may want to override these locations. +Spack also, by default, keeps various caches and user data in ``~/.config/spack``, but users may want to override these locations. Spack provides three environment variables that allow you to override or opt out of configuration locations: -* ``SPACK_USER_CONFIG_PATH``: Override the path to use for the ``user`` scope (``~/.spack`` by default). +* ``SPACK_USER_CONFIG_PATH``: Override the path to use for the ``user`` scope (``~/.config/spack`` by default). * ``SPACK_SYSTEM_CONFIG_PATH``: Override the path to use for the ``system`` scope (``/etc/spack`` by default). -* ``SPACK_DISABLE_LOCAL_CONFIG``: Set this environment variable to completely disable **both** the system and user configuration directories. +* ``SPACK_DISABLE_LOCAL_CONFIG``: Set this environment variable to completely disable **all** configurations from the system and user directories. Spack will then only consider its own defaults and ``site`` configuration locations. - -And one that allows you to move the default cache location: - -* ``SPACK_USER_CACHE_PATH``: Override the default path to use for user data (misc_cache, tests, reports, etc.) - -With these settings, if you want to isolate Spack in a CI environment, you can do this: - -.. code-block:: console - - $ export SPACK_DISABLE_LOCAL_CONFIG=true - $ export SPACK_USER_CACHE_PATH=/tmp/spack diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index a4f0ce070f68e1..7d721df3f7755c 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -66,7 +66,6 @@ So that you can familiarize yourself with the project, we will start with a high etc/ spack/ <- Spack config files. - Can be overridden by files in ~/.spack. var/ spack/ diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index aaf3e0c69cce81..84708ba45964cd 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -63,14 +63,14 @@ An environment is created by: $ spack env create myenv -The directory ``$SPACK_ROOT/var/spack/environments/myenv`` is created to manage the environment. +The directory ``$data_home/environments/myenv`` is created to manage the environment. .. note:: - By default, all managed environments are stored in the ``$SPACK_ROOT/var/spack/environments`` folder. + By default, all managed environments are stored in the ``$data_home/environments`` folder. This location can be changed by setting the ``environments_root`` variable in ``config.yaml``. -Spack creates the file ``spack.yaml``, hidden directory ``.spack-env``, and ``spack.lock`` file under ``$SPACK_ROOT/var/spack/environments/myenv``. +Spack creates the file ``spack.yaml``, hidden directory ``.spack-env``, and ``spack.lock`` file under ``$data_home/environments/myenv``. User interaction occurs through the ``spack.yaml`` file and the Spack commands that affect it. Metadata and, by default, the view are stored in the ``.spack-env`` directory. When the environment is concretized, Spack creates the ``spack.lock`` file with the fully configured specs and dependencies for the environment. @@ -124,7 +124,7 @@ It will also include any other files included in the environment directory, such Environment creation also accepts a full path to the file. - If the path is not under the ``$SPACK_ROOT/var/spack/environments`` directory then the source is referred to as an :ref:`independent environment `. + If the path is not under the ``config:environments_root`` directory then the source is referred to as an :ref:`independent environment `. The name of an environment can be a nested path to help organize environments via subdirectories. @@ -240,7 +240,7 @@ The same rule applies to the ``install`` and ``uninstall`` commands. $ spack install zlib@1.2.8 [+] yfc7epf zlib@1.2.8 ~/spack/opt/spack/linux-rhel7-broadwell/gcc-8.1.0/zlib-1.2.8-yfc7epf57nsfn2gn4notccaiyxha6z7x (12s) - ==> Updating view at ~/spack/var/spack/environments/myenv/.spack-env/view + ==> Updating view at ~/.local/share/spack/environments/myenv/.spack-env/view $ spack find ==> In environment myenv diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 97c8a0200d61af..b52f593b054300 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -72,7 +72,8 @@ If you're new to Spack and want to start using it, see :doc:`getting_started`, o :maxdepth: 2 :caption: Configuration - configuration + configuration_basics + where_spack_writes_data config_yaml packages_yaml toolchains_yaml diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index 436c3aff56eb08..ee4f7e9e699733 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -222,7 +222,7 @@ To remove a mirror by name, run: Mirror precedence ----------------- -Adding a mirror really adds a line in ``~/.spack/mirrors.yaml``: +Adding a mirror really adds an entry in the mirrors config, by default stored in the user environment scope at ``~/.config/spack/mirrors.yaml``: .. code-block:: yaml @@ -240,7 +240,7 @@ Local Default Cache Spack caches resources that are downloaded as part of installations. The cache is a valid Spack mirror: it uses the same directory structure and naming scheme as other Spack mirrors (so it can be copied anywhere and referenced with a URL like other mirrors). -The mirror is maintained locally (within the Spack installation directory) at :file:`var/spack/cache/`. +The mirror is maintained locally at :file:`$data_home/downloads`. It is always enabled (and is always searched first when attempting to retrieve files for an installation) but can be cleared with ``spack clean --misc-cache``; the cache directory can also be deleted manually without issue. Caching includes retrieved tarball archives and source control repositories, but only resources with an associated digest or commit ID (e.g. a revision number for SVN) will be cached. diff --git a/lib/spack/docs/module_file_support.rst b/lib/spack/docs/module_file_support.rst index dbae34821e2ddf..4ff78d1184c43f 100644 --- a/lib/spack/docs/module_file_support.rst +++ b/lib/spack/docs/module_file_support.rst @@ -52,7 +52,7 @@ Assuming you have a module system installed, you should now be able to use the ` $ module avail - --------------------------------------------------------------- ~/spack/share/spack/modules/linux-ubuntu14-x86_64 --------------------------------------------------------------- + --------------------------------------------------------------- ~/.local/share/spack/modules/linux-ubuntu14-x86_64 --------------------------------------------------------------- autoconf/2.69-gcc-4.8-qextxkq hwloc/1.11.6-gcc-6.3.0-akcisez m4/1.4.18-gcc-4.8-ev2znoc openblas/0.2.19-gcc-6.3.0-dhkmed6 py-setuptools/34.2.0-gcc-6.3.0-fadur4s automake/1.15-gcc-4.8-maqvukj isl/0.18-gcc-4.8-afi6taq m4/1.4.18-gcc-6.3.0-uppywnz openmpi/2.1.0-gcc-6.3.0-go2s4z5 py-six/1.10.0-gcc-6.3.0-p4dhkaw binutils/2.28-gcc-4.8-5s7c6rs libiconv/1.15-gcc-4.8-at46wg3 mawk/1.3.4-gcc-4.8-acjez57 openssl/1.0.2k-gcc-4.8-dkls5tk python/2.7.13-gcc-6.3.0-tyehea7 @@ -87,9 +87,9 @@ The table below summarizes the essential information associated with the differe +-----------+--------------+------------------------------+----------------------------------------------+----------------------+ | | Hierarchical | **Default root directory** | **Default template file** | **Compatible tools** | +===========+==============+==============================+==============================================+======================+ -| ``tcl`` | No | share/spack/modules | share/spack/templates/modules/modulefile.tcl | Env. Modules/Lmod | +| ``tcl`` | No | ~/.local/share/spack/modules | share/spack/templates/modules/modulefile.tcl | Env. Modules/Lmod | +-----------+--------------+------------------------------+----------------------------------------------+----------------------+ -| ``lmod`` | Yes | share/spack/lmod | share/spack/templates/modules/modulefile.lua | Lmod | +| ``lmod`` | Yes | ~/.local/share/spack/lmod | share/spack/templates/modules/modulefile.lua | Lmod | +-----------+--------------+------------------------------+----------------------------------------------+----------------------+ @@ -153,7 +153,7 @@ All :ref:`Spack commands that operate on modules ` app Changing the modules root ^^^^^^^^^^^^^^^^^^^^^^^^^ -As shown in the table above, the default module root for ``lmod`` is ``$spack/share/spack/lmod`` and the default root for ``tcl`` is ``$spack/share/spack/modules``. +As shown in the table above, the default module root for ``lmod`` is ``~/.local/share/spack/lmod`` and the default root for ``tcl`` is ``~/.local/share/spack/modules``. This can be overridden for any module set by changing the ``roots`` key of the configuration. .. code-block:: yaml diff --git a/lib/spack/docs/packages_yaml.rst b/lib/spack/docs/packages_yaml.rst index d8bce707d971f3..ed9474798ef7b0 100644 --- a/lib/spack/docs/packages_yaml.rst +++ b/lib/spack/docs/packages_yaml.rst @@ -32,8 +32,8 @@ At a high level, the ``packages.yaml`` file is structured like this: You can either set build preferences specifically for *one* package, or you can specify that certain settings should apply to *all* packages. The types of settings you can customize are described in detail below. -Spack's build defaults are in the default ``etc/spack/defaults/packages.yaml`` file. -You can override them in ``~/.spack/packages.yaml`` or ``etc/spack/packages.yaml``. +Spack's build defaults are in the default ``$spack/etc/spack/defaults/packages.yaml`` file. +You can override them in ``~/.config/spack/packages.yaml`` or ``$spack/etc/spack/packages.yaml``. For more details on how this works, see :ref:`configuration-scopes`. .. _sec-external-packages: diff --git a/lib/spack/docs/repositories.rst b/lib/spack/docs/repositories.rst index 5eea211d077c25..f8dcc2f4b10a30 100644 --- a/lib/spack/docs/repositories.rst +++ b/lib/spack/docs/repositories.rst @@ -147,6 +147,11 @@ For example, to use ``~/custom_packages_clone`` for ``my_remote_repo``: git: https://github.com/myorg/spack-custom-pkgs.git destination: ~/custom_packages_clone + +Spack uses the ``repos`` config section to find repositories. +Note that the ``repos.yaml`` configuration file is distinct from the ``repo.yaml`` file in each repository. +For more on the YAML format, where configuration files live, and on how configuration file precedence works in Spack, see :ref:`configuration `. + If the ``git`` URL is defined in a lower-precedence configuration (like Spack's defaults for ``builtin``), you only need to specify the ``destination`` in your user-level ``repos.yaml``. Spack can make the configuration changes for you using ``spack repo set --destination ~/spack-packages builtin``, or you can directly edit your ``repos.yaml`` file: diff --git a/lib/spack/docs/where_spack_writes_data.rst b/lib/spack/docs/where_spack_writes_data.rst new file mode 100644 index 00000000000000..6dd1dda0986d7b --- /dev/null +++ b/lib/spack/docs/where_spack_writes_data.rst @@ -0,0 +1,125 @@ +.. + Copyright Spack Project Developers. See COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. meta:: + :description lang=en: + Learn how to control where Spack generates files and reads files, and how to effectively isolate a Spack installation. + +.. _where_spack_writes_data: + +Controlling where Spack writes data +=================================== + +A fresh checkout of Spack writes nothing into the ``$spack`` prefix; all data goes under the user's home directory in XDG-compliant locations. +A Spack instance that was installed before this layout — where data lived under ``$spack/opt``, ``$spack/var``, etc. — keeps using those legacy locations, so existing installs and environments are not disrupted. + +Spack picks one of two *layout schemes* at startup: + +* **xdg**: data under ``~/.local/share/spack``, state under ``~/.local/state/spack``, cache under ``~/.cache/spack``. +* **old**: installs in ``$spack/opt/spack``, environments in ``$spack/var/spack/environments``, license files in ``$spack/etc/spack/licenses``, etc. — i.e. the pre-1.2 layout. + +The scheme is chosen by ``etc/spack/defaults/include.yaml`` using a ``when:`` clause that calls :func:`spack.paths.detect_layout`. +The included yaml — ``etc/spack/defaults/old/config.yaml`` or ``etc/spack/defaults/xdg/config.yaml`` — sets ``config:locations:*`` (and, for old, the install_tree/environments/etc. paths that don't share a single root). +Everything else flows through normal config: ``config:install_tree:root`` is ``$data_home/installs``, environments root is ``$data_home/environments``, gpg lives at ``$data_home/gpg``, and so on. + +You can see the active scheme and where each path came from with:: + + spack debug paths + +Sample output:: + + layout scheme: xdg (no legacy $spack-local data) + + homes: + $data_home /home/alice/.local/share/spack + config:locations:data (scope: defaults:xdg) + $state_home /home/alice/.local/state/spack + config:locations:state (scope: defaults:xdg) + ... + + config-driven paths: + config:install_tree:root /home/alice/.local/share/spack/installs + $data_home/installs [scope: defaults:base] + ... + +How to override +--------------- + +In order of priority (highest first): + +1. **Env vars** for individual homes — ``SPACK_DATA_HOME``, ``SPACK_STATE_HOME``, ``SPACK_CACHE_HOME``, or ``SPACK_HOME`` (which sets all three via XDG-style subpaths). Any of these also *forces the xdg scheme* even when legacy ``$spack`` data is present, so a partial override never produces a split layout. + +2. **Specific config keys** for individual paths — set ``config:install_tree:root``, ``config:environments_root``, ``config:license_dir``, ``config:source_cache``, ``config:gpg_path``, or ``config:gpg_keys_path`` in any user/site/system scope. + +3. **Layout roots** — set ``config:locations:{home,data,state,cache}`` to redirect everything that uses the corresponding substitution. + +Config locations themselves — ``user_config_path``, ``system_config_path``, the entry-point ``include.yaml`` — are NOT in config (they bootstrap config). Override them with ``SPACK_USER_CONFIG_PATH``, ``SPACK_SYSTEM_CONFIG_PATH``, or ``SPACK_DISABLE_LOCAL_CONFIG``. + +Path substitutions +------------------ + +Config values can reference these in any string field: + +* ``$data_home`` — typically ``~/.local/share/spack`` (xdg) or ``$spack`` (old) +* ``$state_home`` — typically ``~/.local/state/spack`` (xdg) or ``~/.spack`` (old) +* ``$cache_home`` — typically ``~/.cache/spack`` +* ``$spack_home`` — base for spack's user-level data; defaults to ``~`` +* ``$xdg_data_home`` — ``$XDG_DATA_HOME`` if set, else ``~/.local/share`` (no ``/spack`` suffix) +* ``$xdg_state_home`` — ``$XDG_STATE_HOME`` if set, else ``~/.local/state`` +* ``$xdg_cache_home`` — ``$XDG_CACHE_HOME`` if set, else ``~/.cache`` +* ``$user_cache_path`` — alias for ``$state_home`` (legacy) +* ``$spack`` — the Spack instance's prefix +* ``$spack_instance_id`` — hash distinguishing co-installed Spack instances + +The ``$xdg_*_home`` substitutions are used by the xdg scheme yaml so the layout respects XDG env vars without baking that resolution into Python. + +Migrating from the old layout +----------------------------- + +If you have a ``~/.spack`` directory from before 1.2, you'll see a one-time warning. Run:: + + spack migrate --clear + +to copy your config into ``~/.config/spack`` and move ``~/.spack`` to a backup at ``~/.local/share/spack/dotspack_backup``. (The backup location is fixed; it does not move when you set ``SPACK_DATA_HOME``.) Use ``spack migrate --restore`` to undo. + +If you have older Spack instances that can't be upgraded and need ``~/.spack`` to stick around, see ``spack migrate --i-need-old-spack``. + +The location table +------------------ + ++----------------+-----------+--------------------+------------+--------------------+ +| | data_home | state_home | cache_home | somewhere_else | ++================+===========+====================+============+====================+ +| installs | x | | | | ++----------------+-----------+--------------------+------------+--------------------+ +| build stages | | | | x [#wheretable-1]_ | ++----------------+-----------+--------------------+------------+--------------------+ +| download cache | x | | | | ++----------------+-----------+--------------------+------------+--------------------+ +| gpg keys | x | | | | ++----------------+-----------+--------------------+------------+--------------------+ +| modules | x | | | | ++----------------+-----------+--------------------+------------+--------------------+ +| environments | x | | | | ++----------------+-----------+--------------------+------------+--------------------+ +| misc cache | | x [#wheretable-2]_ | | | ++----------------+-----------+--------------------+------------+--------------------+ +| test stages | | x | | | ++----------------+-----------+--------------------+------------+--------------------+ +| licenses | x | | | | ++----------------+-----------+--------------------+------------+--------------------+ +| config files | | | | x [#wheretable-3]_ | ++----------------+-----------+--------------------+------------+--------------------+ + +.. [#wheretable-1] ``cache_home`` is used as a backup, but Spack prefers to write into the user's temp dir if it's available. +.. [#wheretable-2] ``cache_home`` is modeled after ``$XDG_CACHE_HOME``. Spack assumes that ``$XDG_CACHE_HOME`` can be removed on user log-out; misc cache is intended to be longer-lived, so it lives in ``state_home`` instead. +.. [#wheretable-3] User-scope config is controlled with :ref:`environment variables ` or with :ref:`include.yaml ` to avoid recursion issues with configurable locations. + +References +---------- + +* :ref:`include.yaml ` +* :ref:`config.yaml ` +* :ref:`Environment variables controlling config scopes ` diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index ae772db4099df4..8d1ba4a5ec21b0 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -7,7 +7,7 @@ import re from typing import Optional -import spack.paths +import spack.paths_base import spack.util.git #: PEP440 canonical ... string @@ -45,7 +45,7 @@ def get_spack_commit() -> Optional[str]: Returns: (str or None) the commit sha if available, otherwise None """ - git_path = os.path.join(spack.paths.prefix, ".git") + git_path = os.path.join(spack.paths_base.locations.prefix, ".git") if not os.path.exists(git_path): return None @@ -55,7 +55,7 @@ def get_spack_commit() -> Optional[str]: rev = git( "-C", - spack.paths.prefix, + spack.paths_base.locations.prefix, "rev-parse", "HEAD", output=str, diff --git a/lib/spack/spack/bootstrap/config.py b/lib/spack/spack/bootstrap/config.py index 09fec22acea7d5..72185369806e54 100644 --- a/lib/spack/spack/bootstrap/config.py +++ b/lib/spack/spack/bootstrap/config.py @@ -11,7 +11,6 @@ import spack.config import spack.environment import spack.modules -import spack.paths import spack.platforms import spack.repo import spack.store @@ -43,7 +42,7 @@ def spec_for_current_python() -> str: def root_path() -> str: """Root of all the bootstrap related folders""" return spack.util.path.canonicalize_path( - spack.config.get("bootstrap:root", spack.paths.default_user_bootstrap_path) + spack.config.get("bootstrap:root", "$state_home/bootstrap") ) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 84e1e9030cde2a..e19b5348fe0f7f 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -71,7 +71,7 @@ import spack.llnl.util.tty as tty import spack.multimethod import spack.package_base -import spack.paths +import spack.paths_base import spack.platforms import spack.schema.environment import spack.spec @@ -487,7 +487,7 @@ def set_wrapper_variables(pkg, env): env.set(SPACK_DEBUG, "TRUE") env.set(SPACK_SHORT_SPEC, pkg.spec.short_spec) env.set(SPACK_DEBUG_LOG_ID, pkg.spec.format("{name}-{hash:7}")) - env.set(SPACK_DEBUG_LOG_DIR, spack.paths.spack_working_dir) + env.set(SPACK_DEBUG_LOG_DIR, spack.paths_base.spack_working_dir) if spack.config.get("config:ccache"): # Enable ccache in the compiler wrapper diff --git a/lib/spack/spack/caches.py b/lib/spack/spack/caches.py index 528969295f6673..8e8007f0e6c743 100644 --- a/lib/spack/spack/caches.py +++ b/lib/spack/spack/caches.py @@ -9,7 +9,6 @@ import spack.config import spack.fetch_strategy import spack.llnl.util.lang -import spack.paths import spack.util.file_cache import spack.util.path @@ -20,7 +19,7 @@ def misc_cache_location(): Currently the ``MISC_CACHE`` stores indexes for virtual dependency providers and for which packages provide which tags. """ - path = spack.config.get("config:misc_cache", spack.paths.default_misc_cache_path) + path = spack.config.get("config:misc_cache", "$state_home/$spack_instance_id/cache") return spack.util.path.canonicalize_path(path) @@ -39,9 +38,7 @@ def fetch_cache_location(): This prevents Spack from repeatedly fetch the same files when building the same package different ways or multiple times. """ - path = spack.config.get("config:source_cache") - if not path: - path = spack.paths.default_fetch_cache_path + path = spack.config.get("config:source_cache", "$data_home/downloads") path = spack.util.path.canonicalize_path(path) return path diff --git a/lib/spack/spack/ci/common.py b/lib/spack/spack/ci/common.py index 581ce99b5704a0..02bd832a864f92 100644 --- a/lib/spack/spack/ci/common.py +++ b/lib/spack/spack/ci/common.py @@ -579,6 +579,7 @@ def generate_ir(self): { "build-job": { "script": [ + "spack config add 'config:install_tree:root:$spack/installs'", "cd {env_dir}", "spack env activate --without-view .", "spack spec /$SPACK_JOB_SPEC_DAG_HASH", diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 92f97e7b9b7e69..8df8c810521dfa 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -33,6 +33,7 @@ from spack.llnl.util.lang import attr_setdefault, index_by from spack.llnl.util.tty.colify import colify from spack.llnl.util.tty.color import colorize +from spack.paths_base import locations as paths_base from ..enums import InstallRecordStatus @@ -87,7 +88,7 @@ def all_commands(): global _all_commands if _all_commands is None: _all_commands = [] - command_paths = [spack.paths.command_path] # Built-in commands + command_paths = [paths_base.command_path] # Built-in commands command_paths += spack.extensions.get_command_paths() # Extensions for path in command_paths: for file in os.listdir(path): diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index d6ec5795f2a864..965aba9c0bfe70 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -11,11 +11,11 @@ import spack.config import spack.llnl.util.filesystem import spack.llnl.util.tty as tty +import spack.paths import spack.stage import spack.store import spack.util.path from spack.cmd.common import arguments -from spack.paths import lib_path, var_path description = "remove temporary build files and/or downloaded archives" section = "build" @@ -67,7 +67,7 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: def remove_python_cache(): - for directory in [lib_path, var_path]: + for directory in [spack.paths.lib_path]: for root, dirs, files in os.walk(directory): for f in files: if f.endswith(".pyc") or f.endswith(".pyo"): diff --git a/lib/spack/spack/cmd/common/__init__.py b/lib/spack/spack/cmd/common/__init__.py index 4f6bad5c2c88a1..38471bea1ecbd4 100644 --- a/lib/spack/spack/cmd/common/__init__.py +++ b/lib/spack/spack/cmd/common/__init__.py @@ -4,7 +4,7 @@ import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color -import spack.paths +import spack.paths_base def shell_init_instructions(cmd, equivalent): @@ -25,19 +25,19 @@ def shell_init_instructions(cmd, equivalent): "To set up shell support, run the command below for your shell.", "", color.colorize("@*c{For bash/zsh/sh:}"), - " . %s/setup-env.sh" % spack.paths.share_path, + " . %s/setup-env.sh" % spack.paths_base.locations.share_path, "", color.colorize("@*c{For csh/tcsh:}"), - " source %s/setup-env.csh" % spack.paths.share_path, + " source %s/setup-env.csh" % spack.paths_base.locations.share_path, "", color.colorize("@*c{For fish:}"), - " source %s/setup-env.fish" % spack.paths.share_path, + " source %s/setup-env.fish" % spack.paths_base.locations.share_path, "", color.colorize("@*c{For Windows batch:}"), - " %s\\spack_cmd.bat" % spack.paths.bin_path, + " %s\\spack_cmd.bat" % spack.paths_base.locations.bin_path, "", color.colorize("@*c{For PowerShell:}"), - " %s\\setup-env.ps1" % spack.paths.share_path, + " %s\\setup-env.ps1" % spack.paths_base.locations.share_path, "", "Or, if you do not want to use shell support, run " + ("one of these" if shell_specific else "this") diff --git a/lib/spack/spack/cmd/debug.py b/lib/spack/spack/cmd/debug.py index d7e711631d202c..72a8f410731923 100644 --- a/lib/spack/spack/cmd/debug.py +++ b/lib/spack/spack/cmd/debug.py @@ -10,10 +10,12 @@ import spack import spack.config +import spack.paths import spack.platforms import spack.repo import spack.spec import spack.util.git +import spack.util.path description = "debugging commands for troubleshooting Spack" section = "developer" @@ -23,6 +25,11 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="debug_command") sp.add_parser("report", help="print information useful for bug reports") + sp.add_parser( + "paths", + help="show how each Spack data path was resolved (active layout scheme, " + "config keys, env-var overrides)", + ) def _format_repo_info(source, commit): @@ -88,6 +95,102 @@ def report(args): print("* **Platform:**", architecture) +def _row(label, value, source): + return f" {label:<22} {value}\n {source}" + + +def _resolved_home(spack_var, config_var): + """Return (value, source) for state/data/cache home.""" + disable_env = spack.config.get("config:locations:disable_env", False) + if not disable_env and spack_var in os.environ: + return os.path.expanduser(os.environ[spack_var]), f"env: {spack_var}" + if not disable_env and "SPACK_HOME" in os.environ: + # The actual value is computed by SpackPaths._resolve_home; + # we just report SPACK_HOME as the source. + return getattr(spack.paths.locations, f"{config_var}_home"), "env: SPACK_HOME" + cfg = spack.config.get(f"config:locations:{config_var}", None) + if cfg: + scope = _scope_that_set(f"config:locations:{config_var}") + return spack.util.path.canonicalize_path(cfg), f"config:locations:{config_var} ({scope})" + return getattr(spack.paths.locations, f"{config_var}_home"), "default (no source set)" + + +def _scope_that_set(config_path): + """Walk scopes high-to-low and return the name of the first one with this key.""" + section, _, key = config_path.partition(":") + for scope in reversed(list(spack.config.CONFIG.scopes.values())): + data = scope.get_section(section) or {} + # Walk nested keys. + node = data.get(section, {}) + for part in key.split(":") if key else (): + if not isinstance(node, dict) or part not in node: + node = None + break + node = node[part] + if node not in (None, {}): + return f"scope: {scope.name}" + return "scope: (not set)" + + +def paths(args): + p = spack.paths.locations + + # Active layout scheme + if p.old_layout_detected: + env_forces_new = any( + v in os.environ + for v in ("SPACK_DATA_HOME", "SPACK_STATE_HOME", "SPACK_CACHE_HOME", "SPACK_HOME") + ) + if env_forces_new: + scheme = "xdg (forced by SPACK_*_HOME env var; old-layout markers also present)" + else: + scheme = "old ($spack-local layout markers detected)" + else: + scheme = "xdg (no legacy $spack-local data)" + print(f"layout scheme: {scheme}\n") + + print("homes:") + for spack_var, cfg_var in ( + ("SPACK_DATA_HOME", "data"), + ("SPACK_STATE_HOME", "state"), + ("SPACK_CACHE_HOME", "cache"), + ): + value, source = _resolved_home(spack_var, cfg_var) + print(_row(f"${cfg_var}_home", value, source)) + print(_row("$spack_home", p.spack_home, _spack_home_source())) + print() + + # Paths that are read out of config (with substitutions resolved). + print("config-driven paths:") + for key in ( + "config:install_tree:root", + "config:environments_root", + "config:license_dir", + "config:source_cache", + "config:misc_cache", + "config:test_stage", + "config:gpg_path", + "config:gpg_keys_path", + ): + raw = spack.config.get(key, None) + if raw is None: + print(_row(key, "(unset)", "")) + continue + resolved = spack.util.path.canonicalize_path(raw) + scope = _scope_that_set(key) + print(_row(key, resolved, f"{raw} [{scope}]")) + + +def _spack_home_source(): + if "SPACK_HOME" in os.environ and not spack.config.get("config:locations:disable_env", False): + return "env: SPACK_HOME" + if spack.config.get("config:locations:home", None): + return f"config:locations:home ({_scope_that_set('config:locations:home')})" + return "default (~)" + + def debug(parser, args): if args.debug_command == "report": report(args) + elif args.debug_command == "paths": + paths(args) diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 96571ff20341c9..f4f8e1f6258410 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -12,6 +12,7 @@ import spack.paths import spack.repo import spack.stage +import spack.store from spack.cmd.common import arguments description = "print out locations of packages and spack directories" @@ -35,6 +36,9 @@ def setup_parser(subparser: argparse.ArgumentParser) -> None: action="store_true", help="install prefix for spec (spec need not be installed)", ) + directories.add_argument( + "--install-root", action="store_true", help="where spack installs specs" + ) directories.add_argument( "-p", "--package-dir", @@ -110,6 +114,10 @@ def location(parser, args): print(spack.paths.prefix) return + if args.install_root: + print(spack.store.STORE.root) + return + # no -e corresponds to False, -e without arg to None, -e name to the string name. if args.location_env is not False: if args.location_env is None: diff --git a/lib/spack/spack/cmd/migrate.py b/lib/spack/spack/cmd/migrate.py new file mode 100644 index 00000000000000..92aa7713ed7645 --- /dev/null +++ b/lib/spack/spack/cmd/migrate.py @@ -0,0 +1,254 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import argparse +import os +import shutil + +import spack.llnl.util.tty as tty +import spack.paths +from spack.paths_base import locations as paths_base + +description = "migrate user config and cache from old to new locations" +section = "config" +level = "long" + + +def setup_parser(subparser: argparse.ArgumentParser) -> None: + subparser.add_argument( + "--dry-run", + action="store_true", + help="show what would be migrated without actually moving files", + ) + subparser.add_argument( + "--clear", + action="store_true", + help="move entire ~/.spack directory to backup location:" + "use this if no other instances need this old location", + ) + subparser.add_argument( + "--clear-only", + action="store_true", + help="only move ~/.spack to backup without migrating files " + "(useful after running migrate without --clear)", + ) + subparser.add_argument( + "--replace", + action="store_true", + help="replace existing files in new locations (use with --clear " + "if you forgot to use --clear on first run)", + ) + subparser.add_argument( + "--restore", + action="store_true", + help="restore ~/.spack from backup location (after --clear)", + ) + subparser.add_argument( + "--i-need-old-spack", + action="store_true", + help="print help about mixing pre-1.2-Spack and Spack >= 1.2", + ) + + +def restore(args: argparse.Namespace) -> None: + """Restore ~/.spack from backup location.""" + old_location = os.path.expanduser("~/.spack") + backup_location = spack.paths.dotspack_backup + + if not os.path.exists(backup_location): + tty.die(f"No backup found at {backup_location}") + + # Check if ~/.spack already exists + if os.path.exists(old_location): + tty.die(f"Cannot restore: {old_location} already exists") + + if args.dry_run: + tty.msg(f"Would restore from {backup_location} to {old_location}") + return + + tty.msg(f"Restoring from {backup_location} to {old_location}...") + shutil.copytree(backup_location, old_location) + tty.msg("Restore complete!") + + +def i_need_old_spack(): + # Note: I specifically recommend SPACK_USER_CACHE_PATH= rather + # than SPACK_STATE_HOME=..., because if a pre-1.2 instance *is* + # eventually updated, it would take the existence of + # SPACK_STATE_HOME as an indication to use new-style defaults + # for installs etc., and ignore existing installs in + # $spack/opt/spack + print("""\ +If you're getting a warning about using resources in ~/.spack, and +you have pre-1.2 Spack instances that cannot upgrade. Then it is +recommended that both old and new instances use ~/.spack for +configs and the user cache path. + +If a new instance sees SPACK_USER_CACHE_PATH=~/.spack, that will +silence the warning. + +Other explicit uses of ~/.spack will also silence this warning +(e.g. setting `config:locations:state:~/.spack`, or pointing +the user scope's `path` to `~/.spack`). + +TODO: IMO this could also suggest `spack migrate` (no --clear) to +create a divergent cache/config (and once the new spack instance +is isolated from ~/.spack, stop warning). I have to update the +warn logic to accept this though (which is easy). +""") + + +def migrate(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None: + """Migrate user config and package repositories from ~/.spack to new locations. + + This command migrates: + - User config files: ~/.spack/...yaml -> ~/.config/spack/ + - Package repositories: ~/.spack/package_repos -> ~/.local/state/spack/package_repos + """ + if args.i_need_old_spack: + i_need_old_spack() + return + + if args.restore: + restore(args) + return + + old_location = os.path.expanduser("~/.spack") + new_config_location = paths_base.user_config_path + new_state_location = os.path.join(os.path.expanduser("~"), ".local", "state", "spack") + backup_location = spack.paths.dotspack_backup + + if args.clear_only: + if not os.path.exists(old_location): + tty.die(f"Old configuration location does not exist: {old_location}") + if os.path.exists(backup_location): + tty.die(f"Backup location already exists: {backup_location}") + + if args.dry_run: + tty.msg(f"Would move {old_location} to {backup_location}") + return + + os.makedirs(os.path.dirname(backup_location), exist_ok=True) + shutil.move(old_location, backup_location) + tty.msg(f"Backup complete! Original ~/.spack moved to {backup_location}") + return + + # Handle --replace: delete destination directories before migrating + if args.replace: + if os.path.exists(new_config_location): + tty.msg(f"Removing existing config location: {new_config_location}") + shutil.rmtree(new_config_location) + new_package_repos = os.path.join(new_state_location, "package_repos") + if os.path.exists(new_package_repos): + tty.msg(f"Removing existing package repos: {new_package_repos}") + shutil.rmtree(new_package_repos) + + if not os.path.exists(old_location): + tty.die(f"Old configuration location does not exist: {old_location}") + + # Check if backup already exists + if args.clear and os.path.exists(backup_location): + tty.die(f"Backup location already exists: {backup_location}") + + # Track what we'll migrate + migrations = [] + errors = [] + + # 1. Check for config files to migrate (*.yaml and *.yml files in ~/.spack/) + config_files = [] + if os.path.isdir(old_location): + for item in os.listdir(old_location): + if item.endswith(".yaml") or item.endswith(".yml"): + config_files.append(item) + + if config_files: + if os.path.exists(new_config_location): + existing_configs = [ + f + for f in os.listdir(new_config_location) + if f.endswith(".yaml") or f.endswith(".yml") + ] + if existing_configs: + errors.append( + f"New config location already contains config files: {new_config_location}\n" + f" Existing files: {', '.join(existing_configs)}" + ) + + if not errors: + migrations.append(("config", config_files, old_location, new_config_location)) + + # 2. Check for package repositories to migrate + old_package_repos = os.path.join(old_location, "package_repos") + new_package_repos = os.path.join(new_state_location, "package_repos") + + if os.path.exists(old_package_repos) and os.path.isdir(old_package_repos): + repos = os.listdir(old_package_repos) + if repos: + if os.path.exists(new_package_repos): + existing_repos = os.listdir(new_package_repos) + if existing_repos: + errors.append( + f"New package repository location already exists: {new_package_repos}\n" + f" Existing repos: {', '.join(existing_repos)}" + ) + + if not errors: + migrations.append(("package_repos", repos, old_package_repos, new_package_repos)) + + if errors: + tty.msg("Migration conflicts detected (files already in new locations):") + for error in errors: + tty.msg(f" {error}") + tty.msg("\nSkipping migration and backup/clear due to conflicts") + # Exit early here regardless of --clear (we shouldn't move .spack if + # we couldn't copy out the components we want) + return + elif not migrations: + tty.msg("Nothing to migrate - no config files or package repositories found in ~/.spack") + + if args.dry_run: + # Show what will be migrated + if migrations: + tty.msg("Would migrate the following:") + for migration_type, items, src, dst in migrations: + if migration_type == "config": + tty.msg(f"\n Config files from {src}/ to {dst}/:") + for item in items: + tty.msg(f" - {item}") + elif migration_type == "package_repos": + tty.msg(f"\n Package repositories from {src}/ to {dst}/:") + for item in items: + tty.msg(f" - {item}") + return + + if migrations: + for migration_type, items, src, dst in migrations: + # Ensure destination directory exists + os.makedirs(dst, exist_ok=True) + + if migration_type == "config": + for item in items: + src_path = os.path.join(src, item) + dst_path = os.path.join(dst, item) + tty.debug(f"Copying {src_path} -> {dst_path}") + shutil.copy2(src_path, dst_path) + + elif migration_type == "package_repos": + for item in items: + src_path = os.path.join(src, item) + dst_path = os.path.join(dst, item) + tty.debug(f"Copying {src_path} -> {dst_path}") + if os.path.isdir(src_path): + shutil.copytree(src_path, dst_path) + else: + shutil.copy2(src_path, dst_path) + + tty.msg("\nMigration complete!") + tty.msg(f" Config location: {new_config_location}") + tty.msg(f" State location: {new_state_location}") + + if args.clear: + os.makedirs(os.path.dirname(backup_location), exist_ok=True) + shutil.move(old_location, backup_location) + tty.msg(f"Backup complete! Original ~/.spack moved to {backup_location}") diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 7485d5d0e63fa6..1e4bcc3d1aa9b1 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -11,7 +11,10 @@ #. ``spack`` in ``$spack/etc/spack`` - controls all built-in spack scopes, except default #. ``defaults`` in ``$spack/etc/spack/defaults`` - defaults that Spack - needs to function + needs to function. ``defaults/include.yaml`` conditionally pulls in + one of ``defaults/old/`` or ``defaults/xdg/`` (the layout *scheme*) + based on ``layout_detected("old")``; see + :func:`spack.paths.detect_layout`. Important functions in this module are: @@ -43,7 +46,6 @@ from spack.vendor import jsonschema import spack.error -import spack.paths import spack.schema import spack.schema.bootstrap import spack.schema.cdash @@ -71,6 +73,7 @@ import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml from spack.llnl.util import filesystem, lang, tty +from spack.paths_base import locations as paths_base from spack.util.cpus import cpus_available from spack.util.spack_yaml import get_mark_from_yaml_data @@ -105,7 +108,7 @@ } #: Path to the main configuration scope -CONFIGURATION_DEFAULTS_PATH = ("defaults", os.path.join(spack.paths.etc_path, "defaults")) +CONFIGURATION_DEFAULTS_PATH = ("defaults", os.path.join(paths_base.etc_path, "defaults")) #: Hard-coded default values for some key configuration options. #: This ensures that Spack will still work even if config.yaml in @@ -119,7 +122,7 @@ "dirty": False, "build_jobs": min(16, cpus_available()), "build_stage": "$tempdir/spack-stage", - "license_dir": spack.paths.default_license_dir, + "license_dir": os.path.join("$data_home", "licenses"), }, "concretizer": {"externals": {"completion": "default_variants"}}, } @@ -137,6 +140,20 @@ #: safeguard for recursive includes -- maximum include depth MAX_RECURSIVE_INCLUDES = 100 + +#: configurable config vars -- these cannot be used by include paths +#: nor by other paths that can affect config values +CONFIGURABLE_VARS = ( + "config_home", + "state_home", + "cache_home", + "data_home", + "spack_home", + "user_cache_path", +) +_CVARS_RE = "|".join(CONFIGURABLE_VARS) +CONFIGURABLE_VARS_REGEX = r"(\$(" + _CVARS_RE + r")\b)|(\$\{(" + _CVARS_RE + r")\})" + # placeholder object for unspecified default for get methods default_sigil = object() @@ -148,6 +165,7 @@ def __init__(self, name: str, included: bool = False) -> None: self.sections = syaml.syaml_dict() self.prefer_modify = False self.included = included + self.backwards_compat_fallback = False #: included configuration scopes self._included_scopes: Optional[List["ConfigScope"]] = None @@ -1036,6 +1054,22 @@ def override( assert scope is overrides +def substitute_include_path(path, context): + # circular dependencies + import spack.util.path + + banned_var = re.match(CONFIGURABLE_VARS_REGEX, path) + if banned_var: + msg = ( + "Included scope is defined in terms of prohibited config variable." + f" ({banned_var.group(0)}): {path}" + f"\n Context: {context}" + "\n\n Include config paths may not refer to configurable config variables." + ) + raise ValueError(msg) + return spack.util.path.substitute_path_variables(path) + + #: Class for the relevance of an optional path conditioned on a limited #: python code that evaluates to a boolean and or explicit specification #: as optional. @@ -1141,9 +1175,6 @@ def _scope( Raises: ValueError: the required configuration path does not exist """ - # circular dependencies - import spack.util.path - # Ignore included concrete environment files (i.e., ``spack.lock``) # since they are not normal configuration (scope) files and their # processing is handled when the environment is processed. @@ -1162,8 +1193,9 @@ def _scope( # But ensure that name is unique if there are multiple paths. if not self.name or len(getattr(self, "paths", [])) > 1: + context = self.name or (parent_scope.name if parent_scope else "(no context)") + real_path = pathlib.Path(substitute_include_path(path, context)) parent_path = pathlib.Path(getattr(parent_scope, "path", "")) - real_path = pathlib.Path(spack.util.path.substitute_path_variables(path)) try: included_name = real_path.relative_to(parent_path) @@ -1265,16 +1297,28 @@ class IncludePath(OptionalInclude): destination: Optional[str] def __init__(self, entry: dict): - # circular dependencies - import spack.util.path - super().__init__(entry) path_override_env_var = entry.get("path_override_env_var", "") if path_override_env_var and path_override_env_var in os.environ: path = os.environ[path_override_env_var] else: path = entry.get("path", "") - self.path = spack.util.path.substitute_path_variables(path) + + context_prefix = f"({self.name}) " if self.name else "" + context = f"{context_prefix}{path}" + + new_path = substitute_include_path(path, context) + old_path = None + backwards_compat = entry.get("backwards_compat", None) + if backwards_compat: + old_path = substitute_include_path(backwards_compat, context) + + self.backwards_compat_fallback = False + if old_path and os.path.exists(old_path) and not os.path.exists(new_path): + self.path = old_path + self.backwards_compat_fallback = True + else: + self.path = new_path self.sha256 = entry.get("sha256", "") self.remote = "sha256" in entry @@ -1325,6 +1369,8 @@ def scopes(self, parent_scope: ConfigScope) -> List[ConfigScope]: scope = self._scope(self.path, self.destination, parent_scope) if scope is not None: + if self.backwards_compat_fallback: + scope.backwards_compat_fallback = True self._scopes = [scope] return self._scopes @@ -1345,18 +1391,17 @@ class GitIncludePaths(OptionalInclude): destination: Optional[str] def __init__(self, entry: dict): - # circular dependencies - import spack.util.path - super().__init__(entry) - self.git = spack.util.path.substitute_path_variables(entry.get("git", "")) + context = self.name or entry.get("git", "") or "(Git include)" + self.git = substitute_include_path(entry.get("git", ""), context) self.branch = entry.get("branch", "") self.commit = entry.get("commit", "") self.tag = entry.get("tag", "") - self._paths = [ - spack.util.path.substitute_path_variables(path) for path in entry.get("paths", []) - ] + self._paths = [] + for path in entry.get("paths", []): + sub_context = f"{context} - {path}" + self._paths.append(substitute_include_path(path, sub_context)) self.destination = None self.remote = True @@ -1564,7 +1609,7 @@ def create_incremental() -> Generator[Configuration, None, None]: # Initial topmost scope is spack (the config scope in the spack instance). # It includes the user, site, and system scopes. Environments and command # line scopes go above this. - configuration_paths = [("spack", os.path.join(spack.paths.etc_path))] + configuration_paths = [("spack", os.path.join(paths_base.etc_path))] # Python packages can register configuration scopes via entry_points configuration_paths.extend(config_paths_from_entry_points()) diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index 9d53c34c0a438b..56ed3342c97dc6 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -39,7 +39,6 @@ import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as clr import spack.package_base -import spack.paths import spack.repo import spack.schema.env import spack.spec @@ -82,9 +81,6 @@ #: Validation error for a currently activate environment that failed to parse _active_environment_error: Optional[spack.config.ConfigFormatError] = None -#: default path where environments are stored in the spack tree -default_env_path = os.path.join(spack.paths.var_path, "environments") - #: Name of the input yaml file for an environment manifest_name = "spack.yaml" @@ -98,10 +94,21 @@ env_subdir_name = ".spack-env" +def default_env_path() -> str: + """Default for ``config:environments_root`` when no scope sets it. + + The base default scope sets the config key to ``$data_home/environments``; + this function is the runtime equivalent. Tests that mock the + environments directory monkeypatch this function (see the + ``mutable_mock_env_path`` fixture). + """ + return spack.util.path.canonicalize_path("$data_home/environments") + + def env_root_path() -> str: - """Override default root path if the user specified it""" + """Resolve config:environments_root with substitutions and ~ expansion.""" return spack.util.path.canonicalize_path( - spack.config.get("config:environments_root", default=default_env_path) + spack.config.get("config:environments_root", default=default_env_path()) ) diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py index 5cc542664b255a..4653bdfe50d922 100644 --- a/lib/spack/spack/install_test.py +++ b/lib/spack/spack/install_test.py @@ -19,7 +19,6 @@ import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.llnl.util.tty.log -import spack.paths import spack.repo import spack.report import spack.spec @@ -99,7 +98,7 @@ def get_test_stage_dir() -> str: absolute path to the configured test stage root or, if none, the default test stage path """ return spack.util.path.canonicalize_path( - spack.config.get("config:test_stage", spack.paths.default_test_path) + spack.config.get("config:test_stage", "$state_home/test") ) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index cb634fedf0f399..778bc020bd7e34 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -40,9 +40,11 @@ import spack.llnl.util.tty.colify import spack.llnl.util.tty.color as color import spack.paths +import spack.paths_base import spack.platforms import spack.solver.asp import spack.spec +import spack.trace import spack.util.environment import spack.util.lock @@ -519,6 +521,11 @@ def make_argument_parser(**kwargs): default="SPACK_STACKTRACE" in os.environ, help="add stacktraces to all printed statements", ) + debug.add_argument( + "--warn-writes-into-spack", + action="store_true", + help="Warn when Spack tries to write into its own prefix", + ) locks = general lock_mutex = locks.add_mutually_exclusive_group() @@ -580,7 +587,7 @@ def setup_main_options(args): # override lock configuration if passed on command line if args.locks is not None: if args.locks is False: - spack.util.lock.check_lock_safety(spack.paths.prefix) + spack.util.lock.check_lock_safety(spack.paths_base.locations.prefix) spack.config.set("config:locks", args.locks, scope="command_line") if args.mock: @@ -589,7 +596,7 @@ def setup_main_options(args): key = syaml.syaml_str("repos") key.override = True spack.config.CONFIG.scopes["command_line"].sections["repos"] = syaml.syaml_dict( - [(key, [spack.paths.mock_packages_path])] + [(key, [spack.paths_base.locations.mock_packages_path])] ) # If the user asked for it, don't check ssl certs. @@ -920,6 +927,79 @@ def add_command_line_scopes( cfg.push_scope(scope, priority=ConfigScopePriority.CUSTOM) +def _warn_about_old_dotspack(): + """Warn if ~/.spack exists and is in use (not explicitly configured).""" + old_dotspack = os.path.expanduser("~/.spack") + + # Don't warn if it doesn't exist + if not os.path.exists(old_dotspack): + tty.debug("Skip .spack warning: no ~/.spack directory") + return + + # Helper to check if old_dotspack is a prefix + def uses_old_dotspack(path): + return path == old_dotspack or path.startswith(old_dotspack + os.sep) + + # Check if any config scope is using ~/.spack (means explicit configuration) + reasons = [] + for scope in spack.config.CONFIG.scopes.values(): + if hasattr(scope, "path") and uses_old_dotspack(scope.path): + # Explicitly configured via a config scope, don't warn + if scope.backwards_compat_fallback: + reasons.append(f"Used by config scope: {scope.name}") + else: + # A config scope explicitly targets ~/.spack + tty.debug( + f"Skip .spack warning: scope {scope.name} includes ~/.spack: {scope.path}" + ) + return + + explicit_path = False + if uses_old_dotspack(spack.paths.locations.state_home): + # state_home is ~/.spack: distinguish "default from the old-layout + # scheme yaml" (warn) from "user explicitly pointed an env var + # there" (don't warn). + env_override = any( + v in os.environ for v in ("SPACK_USER_CACHE_PATH", "SPACK_STATE_HOME", "SPACK_HOME") + ) + if not env_override: + reasons.append("User cache path fallback") + else: + tty.debug( + "Skip .spack warning:" + f" state_home includes .spack {spack.paths.locations.state_home}" + ) + explicit_path = True + if uses_old_dotspack(spack.paths.locations.data_home): + tty.debug( + f"Skip .spack warning: data_home includes .spack {spack.paths.locations.data_home}" + ) + explicit_path = True + if uses_old_dotspack(spack.paths.locations.cache_home): + tty.debug( + f"Skip .spack warning: cache_home includes .spack {spack.paths.locations.cache_home}" + ) + explicit_path = True + + if explicit_path: + return + + msg = "Old config/user-cache-path in `~/.spack`" + if reasons: + msg += " - it is currently in use:" + for reason in reasons: + msg += f"\n\t{reason}" + else: + msg += " - it is not currently used by this spack instance." + msg += ( + "\nIf all spack instances are >= 1.2, you can use" + " `spack migrate --clear` to silence this warning" + "\nIf not, run `spack migrate --i-need-old-spack` to" + " see what manual steps you can take to silence this warning" + ) + tty.warn(msg) + + def _main(argv=None): """Logic for the main entry point for the Spack command. @@ -1034,6 +1114,9 @@ def add_environment_scope(): cmd_name = args.command[0] cmd_name, args.command = resolve_alias(cmd_name, args.command) + if args.warn_writes_into_spack: + spack.trace.warn_writes_into_spack() + # set up a bootstrap context, if asked. # bootstrap context needs to include parsing the command, b/c things # like `ConstraintAction` and `ConfigSetAction` happen at parse time. @@ -1044,7 +1127,9 @@ def add_environment_scope(): bootstrap_context = bootstrap.ensure_bootstrap_configuration() with bootstrap_context: - return finish_parse_and_run(parser, cmd_name, args, env_format_error) + result = finish_parse_and_run(parser, cmd_name, args, env_format_error) + _warn_about_old_dotspack() + return result def finish_parse_and_run(parser, cmd_name, main_args, env_format_error): @@ -1064,7 +1149,7 @@ def finish_parse_and_run(parser, cmd_name, main_args, env_format_error): raise env_format_error # many operations will fail without a working directory. - spack.paths.set_working_dir() + spack.paths_base.set_working_dir() # now we can actually execute the command. if main_args.spack_profile or main_args.sorted_profile or main_args.profile_file: diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py index 472e40973b9749..9ed331b0b19b2e 100644 --- a/lib/spack/spack/modules/common.py +++ b/lib/spack/spack/modules/common.py @@ -210,14 +210,11 @@ def root_path(name, module_set_name): Returns: root folder for module file installation """ - defaults = {"lmod": "$spack/share/spack/lmod", "tcl": "$spack/share/spack/modules"} # Root folders where the various module files should be written roots = spack.config.get(f"modules:{module_set_name}:roots", {}) - # Merge config values into the defaults so we prefer configured values - roots = spack.schema.merge_yaml(defaults, roots) + path = roots.get(name, os.path.join(spack.paths.data_home, name)) - path = roots.get(name, os.path.join(spack.paths.share_path, name)) return spack.util.path.canonicalize_path(path) diff --git a/lib/spack/spack/new_installer.py b/lib/spack/spack/new_installer.py index 5b7c92251a26f6..b20b1a2bd80363 100644 --- a/lib/spack/spack/new_installer.py +++ b/lib/spack/spack/new_installer.py @@ -72,6 +72,7 @@ import spack.llnl.util.tty.color import spack.mirrors.mirror import spack.paths +import spack.paths_base import spack.report import spack.sandbox import spack.spec @@ -345,15 +346,17 @@ class GlobalState: but excludes the Spack environment, which is slow to serialize and should not be needed during the build.""" - __slots__ = ("store", "config", "monkey_patches", "spack_working_dir", "repo_cache") + __slots__ = ("store", "config", "monkey_patches", "spack_working_dir", "paths_state") def __init__(self): + paths_state = spack.paths.freeze() if multiprocessing.get_start_method() == "fork": return + self.paths_state = paths_state self.config = spack.config.CONFIG.ensure_unwrapped() self.store = spack.store.STORE self.monkey_patches = spack.subprocess_context.TestPatches.create() - self.spack_working_dir = spack.paths.spack_working_dir + self.spack_working_dir = spack.paths_base.spack_working_dir def restore(self): if multiprocessing.get_start_method() == "fork": @@ -366,10 +369,11 @@ def restore(self): opener.urlopen._instance = None s3_client_cache.clear() return + spack.paths.restore(self.paths_state) spack.store.STORE = self.store spack.config.CONFIG = self.config self.monkey_patches.restore() - spack.paths.spack_working_dir = self.spack_working_dir + spack.paths_base.spack_working_dir = self.spack_working_dir class PrefixPivoter: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index cd361702c83d3b..bdd3ee354cbc90 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -13,6 +13,7 @@ import spack.builder import spack.llnl.util.tty as _tty +import spack.paths from spack.archspec import microarchitecture_flags, microarchitecture_flags_from_target from spack.build_environment import ( MakeExecutable, @@ -125,7 +126,6 @@ zsh_completion_path, ) from spack.package_test import compare_output, compare_output_file, compile_c_and_execute -from spack.paths import spack_script from spack.phase_callbacks import run_after, run_before from spack.platforms import host as host_platform from spack.spec import Spec @@ -186,6 +186,8 @@ #: Alias for :func:`os.symlink` (with certain Windows-specific changes) symlink = symlink +spack_script = spack.paths.locations.spack_script + # Not an import alias because black and isort disagree about style create_builder = spack.builder.create diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 707dc60184ac91..96f4891d783c16 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -9,150 +9,321 @@ dependencies. """ +import itertools import os -from pathlib import PurePath +import pathlib +import sys as _sys +import types as _types +from contextlib import contextmanager +from enum import Enum + +import spack.config as config +import spack.paths_base as paths_base -import spack.llnl.util.filesystem -import spack.util.hash as hash -#: This file lives in $prefix/lib/spack/spack/__file__ -prefix = str(PurePath(spack.llnl.util.filesystem.ancestor(__file__, 4))) +def dir_is_occupied(x, except_for=None): + x = pathlib.Path(x) + except_for = except_for or set() + if not x.is_dir(): + return False + for path in x.iterdir(): + if path.parts[-1] not in except_for: + return True + return False -#: synonym for prefix -spack_root = prefix -#: bin directory in the spack prefix -bin_path = os.path.join(prefix, "bin") - -#: The spack script itself -spack_script = os.path.join(bin_path, "spack") - -#: The sbang script in the spack installation -sbang_script = os.path.join(bin_path, "sbang") - -# spack directory hierarchy -lib_path = os.path.join(prefix, "lib", "spack") -module_path = os.path.join(lib_path, "spack") -vendor_path = os.path.join(module_path, "vendor") -command_path = os.path.join(module_path, "cmd") -analyzers_path = os.path.join(module_path, "analyzers") -platform_path = os.path.join(module_path, "platforms") -compilers_path = os.path.join(module_path, "compilers") -operating_system_path = os.path.join(module_path, "operating_systems") -test_path = os.path.join(module_path, "test") -hooks_path = os.path.join(module_path, "hooks") -opt_path = os.path.join(prefix, "opt") -share_path = os.path.join(prefix, "share", "spack") -etc_path = os.path.join(prefix, "etc", "spack") - -# -# Things in $spack/etc/spack -# -default_license_dir = os.path.join(etc_path, "licenses") - -# -# Things in $spack/var/spack -# -var_path = os.path.join(prefix, "var", "spack") - -# read-only things in $spack/var/spack -repos_path = os.path.join(var_path, "repos") -test_repos_path = os.path.join(var_path, "test_repos") -mock_packages_path = os.path.join(test_repos_path, "spack_repo", "builtin_mock") - -# -# Writable things in $spack/var/spack -# TODO: Deprecate these, as we want a read-only spack prefix by default. -# TODO: These should probably move to user cache, or some other location. -# -# fetch cache for downloaded files -default_fetch_cache_path = os.path.join(var_path, "cache") - -# GPG paths. -gpg_keys_path = os.path.join(var_path, "gpg") -mock_gpg_data_path = os.path.join(var_path, "gpg.mock", "data") -mock_gpg_keys_path = os.path.join(var_path, "gpg.mock", "keys") -gpg_path = os.path.join(opt_path, "spack", "gpg") - - -#: Not a location itself, but used for when Spack instances -#: share the same cache base directory for caches that should -#: not be shared between those instances. -spack_instance_id = hash.b32_hash(spack_root)[:7] - - -# Below paths are where Spack can write information for the user. -# Some are caches, some are not exactly caches. -# -# The options that start with `default_` below are overridable in -# `config.yaml`, but they default to use `user_cache_path/`. -# -# You can override the top-level directory (the user cache path) by -# setting `SPACK_USER_CACHE_PATH`. Otherwise it defaults to ~/.spack. -# -def _get_user_cache_path(): - return os.path.expanduser(os.getenv("SPACK_USER_CACHE_PATH") or "~%s.spack" % os.sep) - - -user_cache_path = str(PurePath(_get_user_cache_path())) - -#: junit, cdash, etc. reports about builds -reports_path = os.path.join(user_cache_path, "reports") - -#: installation test (spack test) output -default_test_path = os.path.join(user_cache_path, "test") - -#: spack monitor analysis directories -default_monitor_path = os.path.join(reports_path, "monitor") - -#: git repositories fetched to compare commits to versions -user_repos_cache_path = os.path.join(user_cache_path, "git_repos") - -#: default location where remote package repositories are cloned -package_repos_path = os.path.join(user_cache_path, "package_repos") - -#: bootstrap store for bootstrapping clingo and other tools -default_user_bootstrap_path = os.path.join(user_cache_path, "bootstrap") - -#: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) -default_misc_cache_path = os.path.join(user_cache_path, spack_instance_id, "cache") - -# Below paths pull configuration from the host environment. -# -# There are three environment variables you can use to isolate spack from -# the host environment: -# - `SPACK_USER_CONFIG_PATH`: override `~/.spack` location (for config and caches) -# - `SPACK_SYSTEM_CONFIG_PATH`: override `/etc/spack` configuration scope. -# - `SPACK_DISABLE_LOCAL_CONFIG`: disable both of these locations. - - -# User configuration and caches in $HOME/.spack -def _get_user_config_path(): - return os.path.expanduser(os.getenv("SPACK_USER_CONFIG_PATH") or "~%s.spack" % os.sep) - - -# Configuration in /etc/spack on the system -def _get_system_config_path(): - return os.path.expanduser( - os.getenv("SPACK_SYSTEM_CONFIG_PATH") or os.sep + os.path.join("etc", "spack") - ) - - -#: User configuration location -user_config_path = _get_user_config_path() - -#: System configuration location -system_config_path = _get_system_config_path() - -#: Recorded directory where spack command was originally invoked -spack_working_dir = None - - -def set_working_dir(): - """Change the working directory to getcwd, or spack prefix if no cwd.""" - global spack_working_dir - try: - spack_working_dir = os.getcwd() - except OSError: - os.chdir(prefix) - spack_working_dir = prefix +class XDG_vars(Enum): + state_home = "XDG_STATE_HOME" + data_home = "XDG_DATA_HOME" + cache_home = "XDG_CACHE_HOME" + + +class Spack_vars(Enum): + state_home = "SPACK_STATE_HOME" + data_home = "SPACK_DATA_HOME" + cache_home = "SPACK_CACHE_HOME" + user_cache_path = "SPACK_USER_CACHE_PATH" + home = "SPACK_HOME" + + @classmethod + def new_layout(cls): + # Exclude SPACK_USER_CACHE_PATH (legacy env var, doesn't signal new + # layout preference the way the others do). + return [ + Spack_vars.state_home, + Spack_vars.data_home, + Spack_vars.cache_home, + Spack_vars.home, + ] + + +# Used by tests to scrub the environment of XDG_ variables that would +# affect Spack's path resolution. Does not influence config:test_stage. +def _unset_path_vars(env): + for env_var in itertools.chain(XDG_vars, Spack_vars): + env.pop(env_var.value, None) + + +# Relative paths under SPACK_HOME used when SPACK_HOME or +# config:locations:home is set without explicit data/state/cache homes. +_RELATIVE_STATE = os.path.join(".local", "state") +_RELATIVE_DATA = os.path.join(".local", "share") +_RELATIVE_CACHE = ".cache" + + +class SpackPaths: + """Per-instance Spack path resolution. + + The four "home" properties (``state_home``, ``data_home``, + ``cache_home``, ``spack_home``) resolve to whatever the active layout + scheme yaml provides via ``config:locations:*`` — with two + higher-priority overrides preserved in Python: + + 1. ``SPACK_x_HOME`` env var (highest) + 2. ``SPACK_HOME`` env var with the XDG-style subpath appended + 3. ``config:locations:x`` (from scheme yaml or higher scopes) + 4. ``config:locations:home`` with subpath (rarely needed when + scheme yaml is loaded, but kept for completeness) + + Everything else that used to live in this module as ``default_*`` + properties (default install root, envs root, license dir, gpg paths, + download cache) now comes directly from config, with the active + scheme yaml supplying the defaults. See + ``etc/spack/defaults/{old,xdg}/config.yaml``. + """ + + def __init__(self, base): + self.base = base + + self._state_home = None + self._data_home = None + self._cache_home = None + + self.old_layout_detected = detect_old_spack_layout(base) + + def _resolve_home(self, spack_var, config_var, home_rel): + """Resolve one of state/data/cache home with env-var precedence. + + See SpackPaths docstring for the precedence order. + """ + disable_env = config.get("config:locations:disable_env", False) + + if not disable_env and spack_var in os.environ: + return os.path.expanduser(os.environ[spack_var]) + + if not disable_env and "SPACK_HOME" in os.environ: + return os.path.join(os.path.expanduser(os.environ["SPACK_HOME"]), home_rel, "spack") + + cfg = config.get(f"config:locations:{config_var}", None) + if cfg: + import spack.util.path + + return spack.util.path.canonicalize_path(cfg) + + cfg_home = config.get("config:locations:home", None) + if cfg_home: + import spack.util.path + + return os.path.join(spack.util.path.canonicalize_path(cfg_home), home_rel, "spack") + + # Final fallback. Shouldn't be hit when the scheme yamls are + # loaded — they always set config:locations:*. + return os.path.join(os.path.expanduser("~"), home_rel, "spack") + + @property + def state_home(self): + if not self._state_home: + # SPACK_USER_CACHE_PATH is a legacy alias for SPACK_STATE_HOME. + disable_env = config.get("config:locations:disable_env", False) + if not disable_env and "SPACK_USER_CACHE_PATH" in os.environ: + self._state_home = os.path.expanduser(os.environ["SPACK_USER_CACHE_PATH"]) + else: + self._state_home = self._resolve_home("SPACK_STATE_HOME", "state", _RELATIVE_STATE) + return self._state_home + + @property + def cache_home(self): + if not self._cache_home: + self._cache_home = self._resolve_home("SPACK_CACHE_HOME", "cache", _RELATIVE_CACHE) + return self._cache_home + + @property + def data_home(self): + if not self._data_home: + self._data_home = self._resolve_home("SPACK_DATA_HOME", "data", _RELATIVE_DATA) + return self._data_home + + @property + def spack_home(self): + disable_env = config.get("config:locations:disable_env", False) + if not disable_env: + env_home = os.environ.get("SPACK_HOME") + if env_home: + return os.path.expanduser(env_home) + + cfg_home = config.get("config:locations:home", None) + if cfg_home: + import spack.util.path + + return spack.util.path.canonicalize_path(cfg_home) + + return os.path.expanduser("~") + + @property + def user_cache_path(self): + return self.state_home + + @property + def reports_path(self): + #: junit, cdash, etc. reports about builds + return os.path.join(self.state_home, "reports") + + @property + def default_monitor_path(self): + #: spack monitor analysis directories + return os.path.join(self.reports_path, "monitor") + + @property + def user_repos_cache_path(self): + #: git repositories fetched to compare commits to versions + if hasattr(self, "_user_repos_cache_path"): + return self._user_repos_cache_path + return os.path.join(self.state_home, "git_repos") + + @contextmanager + def redirect_state_home(self, x): + old = self._state_home + self._state_home = x + yield + self._state_home = old + + @contextmanager + def redirect_user_repos_cache_path(self, x): + # This is under state_home, but tests may want to selectively redirect + # only this attribute (e.g. to avoid regenerating provider cache) + self._user_repos_cache_path = x + yield + delattr(self, "_user_repos_cache_path") + + @property + def package_repos_path(self): + #: default location where remote package repositories are cloned + return os.path.join(self.state_home, "package_repos") + + @property + def dotspack_backup(self): + #: backup location for old ~/.spack directory during migration. + # Pinned to the XDG default (~/.local/share/spack/dotspack_backup) + # rather than the active scheme's data_home, because the backup is + # a one-time migration artifact and shouldn't move around when the + # user sets SPACK_DATA_HOME or config:locations:data. + return os.path.join(os.path.expanduser("~"), _RELATIVE_DATA, "spack", "dotspack_backup") + + @property + def gpg_path(self): + cfg = config.get("config:gpg_path", None) + if cfg: + import spack.util.path + + return spack.util.path.canonicalize_path(cfg) + return os.path.join(self.data_home, "gpg") + + @property + def gpg_keys_path(self): + cfg = config.get("config:gpg_keys_path", None) + if cfg: + import spack.util.path + + return spack.util.path.canonicalize_path(cfg) + return os.path.join(self.data_home, "gpg-keys") + + def __getattr__(self, name): + # Things that aren't sensitive to import cycles can import the + # paths module and access all items from paths_base + try: + base = object.__getattribute__(self, "base") + except AttributeError: + raise AttributeError(name) + return getattr(base, name) + + +def detect_old_spack_layout(paths: paths_base.SpackPathsBase): + checks = [ + # It's important if this directory is occupied but we have a separate + # check for that, so exclude that directory here + (paths.old_install_path, ["gpg"]), + (paths.old_envs_path, []), + (paths.old_fetch_cache_path, []), + (paths.old_gpg_path, []), + (paths.old_gpg_keys_path, ["README.md"]), + (paths.old_licenses_path, []), + ] + for x, y in checks: + if dir_is_occupied(x, except_for=set(y)): + return True + return False + + +def detect_layout(scheme): + """True if ``scheme`` is the active layout (``"old"`` or ``"xdg"``). + + Used by ``etc/spack/defaults/include.yaml`` to choose which scheme + yaml to include. Honors "unilateral override": if the user has set + any new-style location env var (SPACK_DATA_HOME, SPACK_STATE_HOME, + SPACK_CACHE_HOME, SPACK_HOME), xdg is selected even when legacy + $spack-local data is present. + + Cannot call ``config.get(...)`` here: this runs during config + initialization (an include's ``when:`` is evaluated before the scope + is pushed), so reading config would recurse into the singleton init. + Env-var + filesystem probes only. config:locations:* set in + user/site/system scopes still overrides specific values once those + scopes are loaded. + """ + if scheme not in ("old", "xdg"): + raise ValueError(f"unknown layout scheme: {scheme!r} (expected 'old' or 'xdg')") + env_forces_new = any(v.value in os.environ for v in Spack_vars.new_layout()) + is_old = detect_old_spack_layout(paths_base.locations) and not env_forces_new + return is_old if scheme == "old" else not is_old + + +locations = SpackPaths(paths_base.locations) + + +def freeze(): + """Snapshot resolved homes for a child build process. + + Builds may set their own XDG_* env vars, which would otherwise change + Spack's path resolution mid-build. Call this in the parent, ship the + dict to the child, and call ``restore()`` there. + """ + return { + "state_home": locations.state_home, + "data_home": locations.data_home, + "cache_home": locations.cache_home, + "old_layout_detected": locations.old_layout_detected, + } + + +def restore(bundled_state): + locations._state_home = bundled_state["state_home"] + locations._data_home = bundled_state["data_home"] + locations._cache_home = bundled_state["cache_home"] + locations.old_layout_detected = bundled_state["old_layout_detected"] + + +# Module shim: lets callers keep using `spack.paths.X` for any attribute on +# `locations` (e.g. `spack.paths.gpg_path`), which itself delegates to +# `paths_base.locations` for static attributes (e.g. `spack.paths.prefix`). +# Uses a sys.modules swap because Spack still supports Python 3.6, which +# predates PEP 562 (module-level `__getattr__`). When 3.6 support is dropped, +# replace this block with a plain +# `def __getattr__(name): return getattr(locations, name)`. +class _PathsModule(_types.ModuleType): + def __getattr__(self, name): + return getattr(locations, name) + + +_shim = _PathsModule(__name__) +_shim.__dict__.update(_sys.modules[__name__].__dict__) +_sys.modules[__name__] = _shim diff --git a/lib/spack/spack/paths_base.py b/lib/spack/spack/paths_base.py new file mode 100644 index 00000000000000..177f76e6b83f18 --- /dev/null +++ b/lib/spack/spack/paths_base.py @@ -0,0 +1,93 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +from pathlib import PurePath + +import spack.llnl.util.filesystem +import spack.util.hash as hash + + +class SpackPathsBase: + def __init__(self, _prefix=None): + #: This file lives in $prefix/lib/spack/spack/__file__ + self.prefix = _prefix or str(PurePath(spack.llnl.util.filesystem.ancestor(__file__, 4))) + + #: synonym for prefix + self.spack_root = self.prefix + + #: bin directory in the spack prefix + self.bin_path = os.path.join(self.prefix, "bin") + + #: The spack script itself + self.spack_script = os.path.join(self.bin_path, "spack") + + #: The sbang script in the spack installation + self.sbang_script = os.path.join(self.bin_path, "sbang") + + # spack directory hierarchy + self.lib_path = os.path.join(self.prefix, "lib", "spack") + self.external_path = os.path.join(self.lib_path, "external") + self.module_path = os.path.join(self.lib_path, "spack") + self.vendor_path = os.path.join(self.module_path, "vendor") + self.command_path = os.path.join(self.module_path, "cmd") + self.platform_path = os.path.join(self.module_path, "platforms") + self.compilers_path = os.path.join(self.module_path, "compilers") + self.operating_system_path = os.path.join(self.module_path, "operating_systems") + self.test_path = os.path.join(self.module_path, "test") + self.hooks_path = os.path.join(self.module_path, "hooks") + self.share_path = os.path.join(self.prefix, "share", "spack") + self.etc_path = os.path.join(self.prefix, "etc", "spack") + self.var_path = os.path.join(self.prefix, "var", "spack") + + # $spack/var/spack is generally read-only. Older instances may + # write gpg keys or environments into ...var/ + self.test_repos_path = os.path.join(self.var_path, "test_repos") + self.mock_packages_path = os.path.join(self.test_repos_path, "spack_repo", "builtin_mock") + + self.mock_gpg_data_path = os.path.join(self.var_path, "gpg.mock", "data") + self.mock_gpg_keys_path = os.path.join(self.var_path, "gpg.mock", "keys") + + self.old_install_path = os.path.join(self.prefix, "opt", "spack") + self.old_envs_path = os.path.join(self.var_path, "environments") + self.old_fetch_cache_path = os.path.join(self.var_path, "cache") + self.old_gpg_path = os.path.join(self.prefix, "opt", "spack", "gpg") + self.old_gpg_keys_path = os.path.join(self.var_path, "gpg") + self.old_licenses_path = os.path.join(self.etc_path, "licenses") + + expanded_home = os.path.expanduser("~") + + # If this exists, this was the location for configs and for the package repository + self.old_default_dot_spack = os.path.join(expanded_home, ".spack") + + #: User configuration location + self.user_config_path = os.path.expanduser( + os.getenv("SPACK_USER_CONFIG_PATH") or os.path.join(expanded_home, ".config", "spack") + ) + + #: System configuration location + self.system_config_path = os.path.expanduser( + os.getenv("SPACK_SYSTEM_CONFIG_PATH") or os.sep + os.path.join("etc", "spack") + ) + + #: Not a location itself, but used for when Spack instances + #: share the same cache base directory for caches that should + #: not be shared between those instances. + self.spack_instance_id = hash.b32_hash(self.prefix)[:7] + + +locations = SpackPathsBase() + +#: Recorded directory where spack command was originally invoked +spack_working_dir = None + + +def set_working_dir(): + """Change the working directory to getcwd, or spack prefix if no cwd.""" + global spack_working_dir + try: + spack_working_dir = os.getcwd() + except OSError: + os.chdir(locations.prefix) + spack_working_dir = locations.prefix diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py index 188cad095ad17b..21853b408576df 100644 --- a/lib/spack/spack/schema/config.py +++ b/lib/spack/spack/schema/config.py @@ -75,6 +75,26 @@ **spack.schema.projections.ref_properties, }, }, + "locations": { + "type": "object", + "description": "Roots for Spack's user-level data. Used by " + "$data_home/$state_home/$cache_home/$spack_home substitutions. " + "Defaults come from the active layout scheme yaml " + "(etc/spack/defaults/{old,xdg}/config.yaml).", + "properties": { + "home": {"type": "string"}, + "data": {"type": "string"}, + "cache": {"type": "string"}, + "state": {"type": "string"}, + "disable_env": {"type": "boolean"}, + }, + "additionalProperties": False, + }, + "gpg_path": {"type": "string", "description": "Directory holding Spack's gpg keyring"}, + "gpg_keys_path": { + "type": "string", + "description": "Directory of pre-imported public keys", + }, "install_hash_length": { "type": "integer", "minimum": 1, diff --git a/lib/spack/spack/schema/include.py b/lib/spack/spack/schema/include.py index 6938dee47df929..5f06a0dbbd0d29 100644 --- a/lib/spack/spack/schema/include.py +++ b/lib/spack/spack/schema/include.py @@ -45,6 +45,11 @@ "raw form). Supports file, ftp, http, https schemes and " "Spack/environment variables", }, + "backwards_compat": { + "type": "string", + "description": "For default scopes, old paths where they may be " + "found, if they have moved", + }, "sha256": { "type": "string", "description": "Required SHA256 hash for remote URLs to verify " diff --git a/lib/spack/spack/solver/compat.py b/lib/spack/spack/solver/compat.py index 07c7747ac6a8a8..fb5820d2ee1259 100644 --- a/lib/spack/spack/solver/compat.py +++ b/lib/spack/spack/solver/compat.py @@ -57,7 +57,6 @@ def _ensure_clingo_or_raise(clingo_mod: ModuleType) -> None: # These are imports that may be problematic at top level (circular imports). They are used # only to provide exhaustive details when erroring due to a broken clingo module. import spack.config - import spack.paths as sp import spack.util.path as sup try: @@ -83,7 +82,7 @@ def _ensure_clingo_or_raise(clingo_mod: ModuleType) -> None: if ( pathlib.Path( sup.canonicalize_path( - spack.config.CONFIG.get("bootstrap:root", sp.default_user_bootstrap_path) + spack.config.CONFIG.get("bootstrap:root", "$state_home/bootstrap") ) ) in pathlib.Path(clingo_mod.__file__).parents diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index fab65cba4a46fa..aaccb7a146b519 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -5783,8 +5783,12 @@ def get_host_environment() -> Dict[str, Any]: def eval_conditional(string): """Evaluate conditional definitions using restricted variable scope.""" + import spack.paths # local import to avoid cycle at module load + valid_variables = get_host_environment() - valid_variables.update({"re": re, "env": os.environ}) + valid_variables.update( + {"re": re, "env": os.environ, "layout_detected": spack.paths.detect_layout} + ) return eval(string, valid_variables) diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index e8f07e70f19bb4..046ad70e9bc38e 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -39,9 +39,6 @@ from spack.llnl.util import filesystem as fs from spack.llnl.util import tty -#: default installation root, relative to the Spack install path -DEFAULT_INSTALL_TREE_ROOT = os.path.join(spack.paths.opt_path, "spack") - def parse_install_tree(config_dict: dict) -> Tuple[str, str, Dict[str, str]]: """Parse config settings and return values relevant to the store object. @@ -85,7 +82,9 @@ def parse_install_tree(config_dict: dict) -> Tuple[str, str, Dict[str, str]]: projections = {"all": all_projection} else: - unpadded_root = install_tree.get("root", DEFAULT_INSTALL_TREE_ROOT) + # config:install_tree:root is always set by the active layout + # scheme yaml (etc/spack/defaults/{old,xdg}/config.yaml via base). + unpadded_root = install_tree["root"] unpadded_root = spack.util.path.canonicalize_path(unpadded_root) padded_length = install_tree.get("padded_length", False) diff --git a/lib/spack/spack/subprocess_context.py b/lib/spack/spack/subprocess_context.py index 0d34234ee929b3..04a22394b63375 100644 --- a/lib/spack/spack/subprocess_context.py +++ b/lib/spack/spack/subprocess_context.py @@ -22,6 +22,7 @@ import spack.config import spack.paths +import spack.paths_base import spack.platforms import spack.repo import spack.store @@ -74,10 +75,10 @@ def __init__( ctx = ctx or multiprocessing.get_context() self.global_state = GlobalStateMarshaler(ctx=ctx, serialize_env=True) self.pkg = pkg if ctx.get_start_method() == "fork" else serialize(pkg) - self.spack_working_dir = spack.paths.spack_working_dir + self.spack_working_dir = spack.paths_base.spack_working_dir def restore(self) -> "spack.package_base.PackageBase": - spack.paths.spack_working_dir = self.spack_working_dir + spack.paths_base.spack_working_dir = self.spack_working_dir self.global_state.restore() return deserialize(self.pkg) if isinstance(self.pkg, io.BytesIO) else self.pkg @@ -97,9 +98,12 @@ def __init__( ) -> None: ctx = ctx or multiprocessing.get_context() self.is_forked = ctx.get_start_method() == "fork" + # Make sure paths have been resolved before fork + paths_state = spack.paths.freeze() if self.is_forked: return + self.paths_state = paths_state self.config = spack.config.CONFIG.ensure_unwrapped() self.platform = spack.platforms.host self.store = spack.store.STORE @@ -128,6 +132,7 @@ def restore(self): spack.platforms.host = self.platform spack.store.STORE = self.store self.test_patches.restore() + spack.paths.restore(self.paths_state) if self.env: from spack.environment import activate diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index 72916555c1a3c8..29a1437c3eada2 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -262,7 +262,7 @@ def test_ci_copy_test_logs_to_artifacts_fail(tmp_path: pathlib.Path, capfd): def test_setup_spack_repro_version( - tmp_path: pathlib.Path, capfd, last_two_git_commits, monkeypatch + tmp_path: pathlib.Path, capfd, last_two_git_commits, monkeypatch, override_path ): c1, c2 = last_two_git_commits repro_dir = tmp_path / "repro" @@ -270,7 +270,7 @@ def test_setup_spack_repro_version( spack_dir.mkdir(parents=True) prefix_save = spack.paths.prefix - monkeypatch.setattr(spack.paths, "prefix", "/garbage") + override_path("prefix", "/garbage") ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1) _, err = capfd.readouterr() @@ -278,7 +278,7 @@ def test_setup_spack_repro_version( assert not ret assert "Unable to find the path" in err - monkeypatch.setattr(spack.paths, "prefix", prefix_save) + override_path("prefix", prefix_save) monkeypatch.setattr(spack.util.git, "git", lambda: None) ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1) diff --git a/lib/spack/spack/test/cmd/blame.py b/lib/spack/spack/test/cmd/blame.py index f9b8d6345a2d5e..b08cd43ca096ab 100644 --- a/lib/spack/spack/test/cmd/blame.py +++ b/lib/spack/spack/test/cmd/blame.py @@ -70,10 +70,10 @@ def test_blame_file_outside_spack_repo(tmp_path: Path): assert "not within a spack repo" in out -def test_blame_spack_not_git_clone(monkeypatch): +def test_blame_spack_not_git_clone(override_path): """Ensure attempt to get blame when spack not a git clone fails.""" non_git_dir = os.path.join(spack.paths.prefix, "..") - monkeypatch.setattr(spack.paths, "prefix", non_git_dir) + override_path("prefix", non_git_dir) with pytest.raises(SpackCommandError): out = blame(".") diff --git a/lib/spack/spack/test/cmd/buildcache.py b/lib/spack/spack/test/cmd/buildcache.py index 4e30908694a3b5..810aeeb6f301c6 100644 --- a/lib/spack/spack/test/cmd/buildcache.py +++ b/lib/spack/spack/test/cmd/buildcache.py @@ -22,13 +22,13 @@ import spack.error import spack.main import spack.mirrors.mirror +import spack.paths import spack.spec import spack.util.url as url_util import spack.util.web as web_util from spack.installer import PackageInstaller from spack.llnl.util.filesystem import copy_tree, find, getuid from spack.llnl.util.lang import nullcontext -from spack.paths import test_path from spack.url_buildcache import ( BuildcacheComponent, URLBuildcacheEntry, @@ -563,7 +563,9 @@ def test_push_without_build_deps( @pytest.fixture(scope="function") def v2_buildcache_layout(tmp_path: pathlib.Path): def _layout(signedness: str = "signed"): - source_path = str(pathlib.Path(test_path) / "data" / "mirrors" / "v2_layout" / signedness) + source_path = str( + pathlib.Path(spack.paths.test_path) / "data" / "mirrors" / "v2_layout" / signedness + ) test_mirror_path = tmp_path / "mirror" copy_tree(source_path, test_mirror_path) return test_mirror_path diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index c111bbba4728e5..a2fabac1b24608 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -333,10 +333,11 @@ def test_ci_generate_with_custom_settings( "git checkout ${SPACK_REF}", "popd", ] - assert ci_obj["script"][1].startswith("cd ") - ci_obj["script"][1] = "cd ENV" + assert ci_obj["script"][2].startswith("cd ") + ci_obj["script"][2] = "cd ENV" assert ci_obj["script"] == [ "spack -d ci rebuild", + "spack config add 'config:install_tree:root:$spack/installs'", "cd ENV", "spack env activate --without-view .", "spack spec /$SPACK_JOB_SPEC_DAG_HASH", @@ -1095,10 +1096,10 @@ def test_ci_rebuild_index( assert concrete_spec.dag_hash() + " callpath" in output -def test_ci_get_stack_changed(mock_git_repo, monkeypatch): +def test_ci_get_stack_changed(mock_git_repo, monkeypatch, override_path): """Test that we can detect the change to .gitlab-ci.yml in a mock spack git repo.""" - monkeypatch.setattr(spack.paths, "prefix", mock_git_repo) + override_path("prefix", mock_git_repo) fake_env_path = os.path.join( spack.paths.prefix, os.path.sep.join(("no", "such", "env", "path")) ) diff --git a/lib/spack/spack/test/cmd/clean.py b/lib/spack/spack/test/cmd/clean.py index c3ecb0ae8a53af..883890d52a9575 100644 --- a/lib/spack/spack/test/cmd/clean.py +++ b/lib/spack/spack/test/cmd/clean.py @@ -71,7 +71,7 @@ def test_function_calls(command_line, effects, mock_calls_for_clean, mutable_con assert mock_calls_for_clean[name] == (1 if name in effects else 0) -def test_remove_python_cache(tmp_path: pathlib.Path, monkeypatch): +def test_remove_python_cache(tmp_path: pathlib.Path, monkeypatch, override_path): cache_files = ["file1.pyo", "file2.pyc"] source_file = "file1.py" @@ -92,20 +92,14 @@ def _check_files(directory): assert not os.path.exists(fs.join_path(directory, cache_files[0])) assert not os.path.exists(fs.join_path(directory, "__pycache__")) - source_dir = fs.join_path(str(tmp_path), "lib", "spack", "spack") - var_dir = fs.join_path(str(tmp_path), "var", "spack", "stuff") + source_dir = fs.join_path(tmp_path, "lib", "spack", "spack") - for d in [source_dir, var_dir]: - _setup_files(d) + _setup_files(source_dir) - # Patching the path variables from-import'd by spack.cmd.clean is needed - # to ensure the paths used by the command for this test reflect the - # temporary directory locations and not those from spack.paths when - # the clean command's module was imported. - monkeypatch.setattr(spack.cmd.clean, "lib_path", source_dir) - monkeypatch.setattr(spack.cmd.clean, "var_path", var_dir) + # spack.cmd.clean references paths from spack.paths: we want to + # update them for the duration of this test. + override_path("lib_path", source_dir) spack.cmd.clean.remove_python_cache() - for d in [source_dir, var_dir]: - _check_files(d) + _check_files(source_dir) diff --git a/lib/spack/spack/test/cmd/commands.py b/lib/spack/spack/test/cmd/commands.py index c8362b539bc2ea..22e96ee4b5579f 100644 --- a/lib/spack/spack/test/cmd/commands.py +++ b/lib/spack/spack/test/cmd/commands.py @@ -278,7 +278,7 @@ def test_update_completion_arg(shell, tmp_path: pathlib.Path, monkeypatch): # Note: this test is never expected to be supported on Windows @pytest.mark.not_on_windows("Shell completion script generator fails on windows") @pytest.mark.parametrize("shell", ["bash", "fish"]) -def test_updated_completion_scripts(shell, tmp_path: pathlib.Path): +def test_updated_completion_scripts(shell, tmp_path: pathlib.Path, config): """Make sure our shell tab completion scripts remain up-to-date.""" width = 72 diff --git a/lib/spack/spack/test/cmd/gpg.py b/lib/spack/spack/test/cmd/gpg.py index 5a745e10f1b945..3bf80930bd3eb1 100644 --- a/lib/spack/spack/test/cmd/gpg.py +++ b/lib/spack/spack/test/cmd/gpg.py @@ -9,9 +9,9 @@ import spack.binary_distribution import spack.llnl.util.filesystem as fs +import spack.paths import spack.util.gpg from spack.main import SpackCommand -from spack.paths import mock_gpg_data_path, mock_gpg_keys_path from spack.util.executable import ProcessError #: spack command used by tests below @@ -62,10 +62,10 @@ def test_no_gpg_in_path(tmp_path: pathlib.Path, mock_gnupghome, monkeypatch, mut def test_gpg(tmp_path: pathlib.Path, mutable_config, mock_gnupghome): # Verify a file with an empty keyring. with pytest.raises(ProcessError): - gpg("verify", os.path.join(mock_gpg_data_path, "content.txt")) + gpg("verify", os.path.join(spack.paths.mock_gpg_data_path, "content.txt")) # Import the default key. - gpg("init", "--from", mock_gpg_keys_path) + gpg("init", "--from", spack.paths.mock_gpg_keys_path) # List the keys. # TODO: Test the output here. @@ -73,14 +73,14 @@ def test_gpg(tmp_path: pathlib.Path, mutable_config, mock_gnupghome): gpg("list", "--signing") # Verify the file now that the key has been trusted. - gpg("verify", os.path.join(mock_gpg_data_path, "content.txt")) + gpg("verify", os.path.join(spack.paths.mock_gpg_data_path, "content.txt")) # Untrust the default key. gpg("untrust", "Spack testing") # Now that the key is untrusted, verification should fail. with pytest.raises(ProcessError): - gpg("verify", os.path.join(mock_gpg_data_path, "content.txt")) + gpg("verify", os.path.join(spack.paths.mock_gpg_data_path, "content.txt")) # Create a file to test signing. test_path = tmp_path / "to-sign.txt" diff --git a/lib/spack/spack/test/cmd/migrate.py b/lib/spack/spack/test/cmd/migrate.py new file mode 100644 index 00000000000000..c380a10ac05120 --- /dev/null +++ b/lib/spack/spack/test/cmd/migrate.py @@ -0,0 +1,298 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import pathlib + +import pytest + +import spack +import spack.cmd.migrate +import spack.main +import spack.paths +import spack.paths_base +from spack.paths import SpackPaths +from spack.paths_base import SpackPathsBase + +migrate = spack.main.SpackCommand("migrate") + + +@pytest.fixture(autouse=True) +def clear_env_vars(working_env): + spack.paths._unset_path_vars(os.environ) + + +@pytest.fixture +def migrate_setup(tmp_path, set_home, monkeypatch, mutable_config): + """Set up common test environment for migrate tests. + + Yields: + tuple: (dotspack, created, new_config, new_state, paths) + """ + spack_root = tmp_path / "spack-root" + spack_root.mkdir() + home = tmp_path / "home" + home.mkdir() + + set_home(str(home)) + + # Create ~/.spack with test files + dotspack = home / ".spack" + dotspack.mkdir() + created = create_dotspack_files(dotspack) + + new_config = home / ".config" / "spack" + new_state = home / ".local" / "state" / "spack" + + # Create fresh SpackPaths object and patch the module + base_paths = SpackPathsBase(str(spack_root)) + paths = SpackPaths(base_paths) + monkeypatch.setattr(spack.paths, "locations", paths) + monkeypatch.setattr(spack.paths_base, "locations", base_paths) + + # cmd.migrate imports `paths_base` directly; patch that too. + monkeypatch.setattr(spack.cmd.migrate, "paths_base", base_paths) + + yield (dotspack, created, new_config, new_state, paths) + + +def create_dotspack_files(base_path): + """Create minimal test files under a base path to simulate ~/.spack structure. + + Args: + base_path: Path-like object or string where files should be created + + Returns: + dict: mapping of created file/dir paths to their expected content/structure + """ + base = pathlib.Path(base_path) + + # Create config files with verifiable content + config_yaml = base / "config.yaml" + config_yaml.write_text("config:\n build_jobs: 8\n") + + packages_yaml = base / "packages.yaml" + packages_yaml.write_text("packages:\n all:\n compiler: [gcc]\n") + + # Create package_repos directory with a test repo + package_repos_dir = base / "package_repos" + package_repos_dir.mkdir(parents=True, exist_ok=True) + + test_repo = package_repos_dir / "test_repo" + test_repo.mkdir(parents=True, exist_ok=True) + + # Add a minimal repo.yaml to the test repo + repo_yaml = test_repo / "repo.yaml" + repo_yaml.write_text("repo:\n namespace: test\n") + + # Add a packages directory + packages_dir = test_repo / "packages" + packages_dir.mkdir(parents=True, exist_ok=True) + + # Return mapping of what we created + return { + "config_files": ["config.yaml", "packages.yaml"], + "package_repos": ["test_repo"], + "test_repo_structure": { + "repo.yaml": "repo:\n namespace: test\n", + "packages": {}, # directory + }, + } + + +def verify_files_copied(source_base, dest_base, created_files): + """Verify that files from source_base have been copied to dest_base. + + Args: + source_base: Path-like source directory + dest_base: Path-like destination directory + created_files: dict returned from create_dotspack_files + + Returns: + bool: True if all expected files exist in dest_base with correct content + """ + source = pathlib.Path(source_base) + dest = pathlib.Path(dest_base) + + # Check if dest directory exists + if not dest.exists(): + print(f"Destination directory does not exist: {dest}") + return False + + # Check config files + for config_file in created_files["config_files"]: + source_file = source / config_file + dest_file = dest / config_file + + if not dest_file.exists(): + print(f"Expected file does not exist: {dest_file}") + if dest.exists(): + print(f"Files in {dest}: {list(dest.iterdir())}") + else: + print(f"{dest} does not exist") + return False + + # Verify content matches + if source_file.read_text() != dest_file.read_text(): + print(f"Content mismatch for {config_file}") + return False + + return True + + +def verify_package_repos_copied(source_repos_base, dest_repos_base, created_files): + """Verify that package repos have been copied correctly. + + Args: + source_repos_base: Path-like source package_repos directory + dest_repos_base: Path-like destination package_repos directory + created_files: dict returned from create_dotspack_files + + Returns: + bool: True if all expected repos exist in dest with correct structure + """ + source_base = pathlib.Path(source_repos_base) + dest_base = pathlib.Path(dest_repos_base) + + for repo_name in created_files["package_repos"]: + source_repo = source_base / repo_name + dest_repo = dest_base / repo_name + + if not dest_repo.exists(): + return False + + # Check for repo.yaml + source_repo_yaml = source_repo / "repo.yaml" + dest_repo_yaml = dest_repo / "repo.yaml" + + if not dest_repo_yaml.exists(): + return False + + if source_repo_yaml.read_text() != dest_repo_yaml.read_text(): + return False + + # Check for packages directory + if not (dest_repo / "packages").is_dir(): + return False + + return True + + +def test_migrate_basic(migrate_setup): + """`spack migrate` with no additional arguments moves config files + out of ~/.spack into ~/.config/spack, moves package repositories + from ~/.spack/package_repos into ~/.local/state/spack/package_repos, + and does not delete any files in ~/.spack (e.g. if some spack + instances are not getting updated to 1.2). + """ + dotspack, created, new_config, new_state, paths = migrate_setup + + migrate() + + # Verify config files were copied + assert verify_files_copied(dotspack, new_config, created) + + old_repos = dotspack / "package_repos" + new_repos = new_state / "package_repos" + assert verify_package_repos_copied(old_repos, new_repos, created) + + # Verify old location still exists (not deleted) + assert dotspack.exists() + for config_file in created["config_files"]: + assert (dotspack / config_file).exists() + + +def test_migrate_with_clear(migrate_setup): + """Test --clear: does everything `spack migrate` does, plus + removes ~/.spack (moves it into a backup location in + $data_home/dotspack_backup).""" + dotspack, created, new_config, new_state, paths = migrate_setup + + # The backup location is now paths.dotspack_backup + backup_location = pathlib.Path(paths.dotspack_backup) + + migrate("--clear") + + # Verify config files were copied + assert verify_files_copied(backup_location, new_config, created) + + old_repos = backup_location / "package_repos" + new_repos = new_state / "package_repos" + assert verify_package_repos_copied(old_repos, new_repos, created) + + # Verify old location no longer exists + assert not dotspack.exists() + + # Verify backup exists + assert backup_location.exists() + + +def test_migrate_then_clear_replace(migrate_setup): + """A user who runs `spack migrate` but forgets --clear should + run `spack migrate --clear --replace`, which deletes everything + in ~/.config/spack and ~/.local/state/spack/package_repos + (--replace is telling Spack it's OK to do this, because + otherwise configs in the old/new locations may have diverged + and Spack has no notion of how to merge them). + """ + dotspack, created, new_config, new_state, paths = migrate_setup + + migrate() + + assert verify_files_copied(dotspack, new_config, created) + old_repos = dotspack / "package_repos" + new_repos = new_state / "package_repos" + assert verify_package_repos_copied(old_repos, new_repos, created) + + # Verify old location still exists + assert dotspack.exists() + + # Now run migrate --clear --replace (to remove existing files and clear ~/.spack) + backup_location = pathlib.Path(paths.dotspack_backup) + + migrate("--clear", "--replace") + + # Verify old location is gone + assert not dotspack.exists() + + # Verify backup exists with the original content + assert backup_location.exists() + for config_file in created["config_files"]: + assert (backup_location / config_file).exists() + + +def test_migrate_then_clear_only(migrate_setup): + """If the user runs `spack migrate` but forgets --clear, then + using `spack migrate --clear-only` is telling spack it's ok + to get rid of `~/.spack` (it won't try coping files out of + ~/.spack, which would trigger a collision if `spack migrate` + copied anything into ~/.config/spack, for example).""" + dotspack, created, new_config, new_state, paths = migrate_setup + + migrate() + + assert verify_files_copied(dotspack, new_config, created) + old_repos = dotspack / "package_repos" + new_repos = new_state / "package_repos" + assert verify_package_repos_copied(old_repos, new_repos, created) + + # Verify old location still exists + assert dotspack.exists() + + # Now run migrate --clear-only (just move ~/.spack without re-migrating) + backup_location = pathlib.Path(paths.dotspack_backup) + + migrate("--clear-only") + + # Verify old location is gone + assert not dotspack.exists() + + # Verify backup exists with the original content + assert backup_location.exists() + for config_file in created["config_files"]: + assert (backup_location / config_file).exists() + + # Verify new locations still have the files (weren't touched) + assert verify_files_copied(backup_location, new_config, created) + assert verify_package_repos_copied(backup_location / "package_repos", new_repos, created) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 30ec284cc75a16..39a04a7ff6d0d1 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -19,10 +19,7 @@ import spack.environment as ev import spack.error import spack.llnl.util.filesystem as fs -import spack.package_base -import spack.paths import spack.platforms -import spack.repo import spack.schema.compilers import spack.schema.config import spack.schema.env @@ -37,6 +34,7 @@ import spack.util.spack_yaml as syaml from spack.enums import ConfigScopePriority from spack.llnl.util.filesystem import getuid, join_path, touch +from spack.paths import locations as spack_paths from spack.util.spack_yaml import DictWithLineInfo # sample config data @@ -334,17 +332,17 @@ def __init__(self, path): def test_substitute_config_variables(mock_low_high_config, monkeypatch, tmp_path: pathlib.Path): # Test $spack substitution at the start (valid on all platforms) - assert os.path.join(spack.paths.prefix, "foo", "bar", "baz") == spack_path.canonicalize_path( + assert os.path.join(spack_paths.prefix, "foo", "bar", "baz") == spack_path.canonicalize_path( "$spack/foo/bar/baz/" ) - assert os.path.join(spack.paths.prefix, "foo", "bar", "baz") == spack_path.canonicalize_path( + assert os.path.join(spack_paths.prefix, "foo", "bar", "baz") == spack_path.canonicalize_path( "${spack}/foo/bar/baz/" ) # Test $spack substitution in the middle. This only makes sense when using posix paths. if sys.platform != "win32": - prefix = spack.paths.prefix.lstrip(os.sep) + prefix = spack_paths.prefix.lstrip(os.sep) base = str(tmp_path) assert os.path.join(base, "foo", "bar", "baz", prefix) == spack_path.canonicalize_path( @@ -467,7 +465,7 @@ def test_substitute_user(mock_low_high_config, tmp_path: pathlib.Path): def test_substitute_user_cache(mock_low_high_config): - user_cache_path = spack.paths.user_cache_path + user_cache_path = spack_paths.user_cache_path assert os.path.join(user_cache_path, "baz") == spack_path.canonicalize_path( os.path.join("$user_cache_path", "baz") ) @@ -1174,20 +1172,6 @@ def test_bad_path_double_override(config): pass -def test_license_dir_config(mutable_config, mock_packages, tmp_path): - """Ensure license directory is customizable""" - expected_dir = spack.paths.default_license_dir - assert spack.config.get("config:license_dir") == expected_dir - assert spack.package_base.PackageBase.global_license_dir == expected_dir - assert spack.repo.PATH.get_pkg_class("pkg-a").global_license_dir == expected_dir - - abs_path = str(tmp_path / "foo" / "bar" / "baz") - spack.config.set("config:license_dir", abs_path) - assert spack.config.get("config:license_dir") == abs_path - assert spack.package_base.PackageBase.global_license_dir == abs_path - assert spack.repo.PATH.get_pkg_class("pkg-a").global_license_dir == abs_path - - @pytest.mark.regression("22547") def test_single_file_scope_cache_clearing(env_yaml): scope = spack.config.SingleFileScope( @@ -1222,28 +1206,6 @@ def test_internal_config_scope_cache_clearing(): assert internal_scope.sections["config"] == data -def test_system_config_path_is_overridable(working_env): - p = "/some/path" - os.environ["SPACK_SYSTEM_CONFIG_PATH"] = p - assert spack.paths._get_system_config_path() == p - - -def test_system_config_path_is_default_when_env_var_is_empty(working_env): - os.environ["SPACK_SYSTEM_CONFIG_PATH"] = "" - assert os.sep + os.path.join("etc", "spack") == spack.paths._get_system_config_path() - - -def test_user_config_path_is_overridable(working_env): - p = "/some/path" - os.environ["SPACK_USER_CONFIG_PATH"] = p - assert p == spack.paths._get_user_config_path() - - -def test_user_config_path_is_default_when_env_var_is_empty(working_env): - os.environ["SPACK_USER_CONFIG_PATH"] = "" - assert os.path.expanduser("~%s.spack" % os.sep) == spack.paths._get_user_config_path() - - def test_default_install_tree(monkeypatch, default_config): s = spack.spec.Spec("nonexistent@x.y.z arch=foo-bar-baz") monkeypatch.setattr(s, "dag_hash", lambda length: "abc123") @@ -1514,17 +1476,6 @@ def test_override_included_config(working_env, tmp_path, include_config_factory) assert "test3" in includes -def test_user_cache_path_is_overridable(working_env): - p = "/some/path" - os.environ["SPACK_USER_CACHE_PATH"] = p - assert spack.paths._get_user_cache_path() == p - - -def test_user_cache_path_is_default_when_env_var_is_empty(working_env): - os.environ["SPACK_USER_CACHE_PATH"] = "" - assert os.path.expanduser("~%s.spack" % os.sep) == spack.paths._get_user_cache_path() - - def test_config_file_dir_failure(tmp_path: pathlib.Path, mutable_empty_config): with pytest.raises(spack.config.ConfigFileError, match="not a file"): spack.config.read_config_file(str(tmp_path)) @@ -1697,6 +1648,13 @@ def test_included_optional_include_scopes(): spack.config.OptionalInclude({}).scopes(spack.config.ConfigScope("fail")) +def test_included_path_forbidden_var(tmp_path: pathlib.Path, mock_low_high_config, ensure_debug): + with pytest.raises(ValueError, match="defined in terms of.*data_home"): + include = spack.config.included_path("$data_home/test") + parent_scope = mock_low_high_config.scopes["low"] + include.scopes(parent_scope) + + def test_included_path_string( tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, capfd ): @@ -1746,11 +1704,6 @@ def test_included_path_string_no_parent_path( def test_included_path_substitution(): - # check a straight path substitution - entry = {"path": "$user_cache_path/path/to/config.yaml"} - include = spack.config.included_path(entry) - assert spack.paths.user_cache_path in include.path - # check path through an environment variable path = "/path/to/project/packages.yaml" os.environ["SPACK_TEST_PATH_SUB"] = path @@ -1778,6 +1731,37 @@ def test_included_path_conditional_bad_when( assert not scopes +@pytest.fixture +def backwards_compat_setup(tmp_path: pathlib.Path): + src_dir = tmp_path / "src" + src_dir.mkdir() + (src_dir / "example.yaml").touch() + + # Path doesn't exist yet + to_dir = tmp_path / "to" + + entry = {"path": str(to_dir), "optional": True, "backwards_compat": str(src_dir)} + + yield src_dir, to_dir, entry + + +def test_backwards_compat_use_old(backwards_compat_setup): + """If target dir doesn't exist, make it and copy over yaml files.""" + src_dir, to_dir, entry = backwards_compat_setup + + x = spack.config.included_path(entry) + assert x.paths == [str(src_dir)] + + +def test_backwards_compat_skip_old(backwards_compat_setup): + """If target dir exists, copy over nothing from source.""" + src_dir, to_dir, entry = backwards_compat_setup + + to_dir.mkdir() + x = spack.config.included_path(entry) + assert x.paths == [str(to_dir)] + + def test_included_path_conditional_success(tmp_path: pathlib.Path, mock_low_high_config): path = tmp_path / "local" path.mkdir() @@ -1989,9 +1973,7 @@ def test_included_path_git_temp_dest(mock_low_high_config): assert dest_dir == temp_dir, pre + rest -def test_included_path_git_errs(tmp_path: pathlib.Path, mock_low_high_config, monkeypatch): - monkeypatch.setattr(spack.paths, "user_cache_path", str(tmp_path)) - +def test_included_path_git_errs(mock_low_high_config, monkeypatch, redirect_state_home): paths = ["concretizer.yaml"] entry = { "git": "https://example.com/linux/configs.git", diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 6c9e12a8fcb2cf..e244fd89695246 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -48,6 +48,7 @@ import spack.modules.common import spack.package_base import spack.paths +import spack.paths_base import spack.platforms import spack.repo import spack.solver.asp @@ -134,7 +135,7 @@ def git(): # @pytest.fixture(scope="session") def last_two_git_commits(git): - spack_git_path = spack.paths.prefix + spack_git_path = spack.paths_base.locations.prefix with working_dir(spack_git_path): git_log_out = git("log", "-n", "2", output=str, error=os.devnull) @@ -151,17 +152,47 @@ def write_file(filename, contents): @pytest.fixture -def override_git_repos_cache_path(tmp_path: Path): - saved = spack.paths.user_repos_cache_path +def override_path(monkeypatch): + def _override(path_attr, new_path): + if hasattr(spack.paths_base.locations, path_attr): + monkeypatch.setattr(spack.paths_base.locations, path_attr, new_path) + elif hasattr(spack.paths.locations, path_attr): + raise ValueError( + "To redirect individual spack.paths attributes, use (or add)" + " an appropriate redirect to spack.paths and conftest" + ) + + return _override + + +@pytest.fixture +def redirect_user_repos_cache_path(tmp_path: Path): tmp_git_path = tmp_path / "git-repo-cache-path-for-tests" tmp_git_path.mkdir() - spack.paths.user_repos_cache_path = str(tmp_git_path) - yield - spack.paths.user_repos_cache_path = saved + spack.paths.locations.redirect_user_repos_cache_path(tmp_git_path) + + +@pytest.fixture +def redirect_state_home(tmp_path: Path): + spack.paths.locations.redirect_state_home(tmp_path) + + +@pytest.fixture +def set_home(working_env): + def _set_home(val): + # Clear some env vars that can interfere w/ expanduser(~) on Windows + os.environ.pop("USERPROFILE", None) + os.environ.pop("HOMEDRIVE", None) + os.environ["HOMEPATH"] = val + + # For expanduser on Linux + os.environ["HOME"] = val + + yield _set_home @pytest.fixture -def mock_git_version_info(git, tmp_path: Path, override_git_repos_cache_path): +def mock_git_version_info(git, tmp_path: Path, redirect_user_repos_cache_path): """Create a mock git repo with known structure The structure of commits in this repo is as follows:: @@ -267,7 +298,7 @@ def latest_commit(): @pytest.fixture -def mock_git_package_changes(git, tmp_path: Path, override_git_repos_cache_path, monkeypatch): +def mock_git_package_changes(git, tmp_path: Path, redirect_user_repos_cache_path): """Create a mock git repo with known structure of package edits The structure of commits in this repo is as follows:: @@ -320,25 +351,37 @@ def latest_commit(): os.makedirs(os.path.dirname(filename)) # add diff-test as a new package to the repository - shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-0.txt", filename) + shutil.copy2( + f"{spack.paths_base.locations.test_path}/data/conftest/diff-test/package-0.txt", + filename, + ) git("add", filename) commit("diff-test: new package") commits.append(latest_commit()) # add v2.1.5 to diff-test - shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-1.txt", filename) + shutil.copy2( + f"{spack.paths_base.locations.test_path}/data/conftest/diff-test/package-1.txt", + filename, + ) git("add", filename) commit("diff-test: add v2.1.5") commits.append(latest_commit()) # add v2.1.6 to diff-test - shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-2.txt", filename) + shutil.copy2( + f"{spack.paths_base.locations.test_path}/data/conftest/diff-test/package-2.txt", + filename, + ) git("add", filename) commit("diff-test: add v2.1.6") commits.append(latest_commit()) # add v2.1.7 and v2.1.8 to diff-test - shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-3.txt", filename) + shutil.copy2( + f"{spack.paths_base.locations.test_path}/data/conftest/diff-test/package-3.txt", + filename, + ) git("add", filename) commit("diff-test: add v2.1.7 and v2.1.8") commits.append(latest_commit()) @@ -696,7 +739,7 @@ def _use_test_platform(test_platform): # @pytest.fixture(scope="session") def mock_packages_repo(): - yield spack.repo.from_path(spack.paths.mock_packages_path) + yield spack.repo.from_path(spack.paths_base.locations.mock_packages_path) def _pkg_install_fn(pkg, spec, prefix): @@ -743,7 +786,7 @@ def mock_packages(mock_packages_repo, mock_pkg_install, request): def mutable_mock_repo(mock_packages_repo, request): """Function-scoped mock packages, for tests that need to modify them.""" ensure_configuration_fixture_run_before(request) - mock_repo = spack.repo.from_path(spack.paths.mock_packages_path) + mock_repo = spack.repo.from_path(spack.paths_base.locations.mock_packages_path) with spack.repo.use_repositories(mock_repo) as mock_packages_repo: yield mock_packages_repo @@ -758,7 +801,7 @@ def __init__(self, root_directory: str) -> None: namespace = f"test_namespace_{RepoBuilder._counter}" repo_root = os.path.join(root_directory, namespace) os.makedirs(repo_root, exist_ok=True) - self.template_dirs = (os.path.join(spack.paths.share_path, "templates"),) + self.template_dirs = (os.path.join(spack.paths_base.locations.share_path, "templates"),) self.root, self.namespace = spack.repo.create_repo(repo_root, namespace) self.build_system_name = f"test_build_system_{self.namespace}" self._add_build_system() @@ -857,7 +900,7 @@ def default_config(): This ensures we can test the real default configuration without having tests fail when the user overrides the defaults that we test against.""" - defaults_path = os.path.join(spack.paths.etc_path, "defaults") + defaults_path = os.path.join(spack.paths_base.locations.etc_path, "defaults") if sys.platform == "win32": defaults_path = os.path.join(defaults_path, "windows") with spack.config.use_configuration(defaults_path) as defaults_config: @@ -870,7 +913,10 @@ def mock_uarch_json(tmp_path_factory: pytest.TempPathFactory): tmpdir = tmp_path_factory.mktemp("microarchitectures") uarch_json_source = ( - Path(spack.paths.test_path) / "data" / "microarchitectures" / "microarchitectures.json" + Path(spack.paths_base.locations.test_path) + / "data" + / "microarchitectures" + / "microarchitectures.json" ) uarch_json_dest = tmpdir / "microarchitectures.json" shutil.copy2(uarch_json_source, uarch_json_dest) @@ -914,7 +960,7 @@ def configuration_dir(tmp_path_factory: pytest.TempPathFactory, linux_os): # /data/config has mock config yaml files in it # copy these to the site config. - test_config = Path(spack.paths.test_path) / "data" / "config" + test_config = Path(spack.paths_base.locations.test_path) / "data" / "config" shutil.copytree(test_config, tmp_path / "site") # Create temporary 'defaults', 'site' and 'user' folders @@ -1399,7 +1445,9 @@ def module_configuration(monkeypatch, request, mutable_config): # Key for specific settings relative to this module type writer_key = str(writer_mod.__name__).split(".")[-1] # Root folder for configuration - root_for_conf = os.path.join(spack.paths.test_path, "data", "modules", writer_key) + root_for_conf = os.path.join( + spack.paths_base.locations.test_path, "data", "modules", writer_key + ) # ConfigUpdate, when called, will modify configuration, so we need to use # the mutable_config fixture @@ -1918,12 +1966,22 @@ def get_rev(): yield t +class LazyValue: + def __init__(self, val): + self.val = val + + def __call__(self): + return self.val + + @pytest.fixture(scope="function") def mutable_mock_env_path(tmp_path: Path, mutable_config, monkeypatch): """Fixture for mocking the internal spack environments directory.""" mock_path = tmp_path / "mock-env-path" mutable_config.set("config:environments_root", str(mock_path)) - monkeypatch.setattr(ev.environment, "default_env_path", str(mock_path)) + # Also pin the runtime default so tests that compose with other config + # fixtures (which may reset the active config) still see the mock. + monkeypatch.setattr(ev.environment, "default_env_path", lambda: str(mock_path)) return mock_path @@ -2007,7 +2065,7 @@ def mock_clone_repo(tmp_path_factory: pytest.TempPathFactory): ) shutil.copytree( - os.path.join(spack.paths.mock_packages_path, spack.repo.packages_dir_name), + os.path.join(spack.paths_base.locations.mock_packages_path, spack.repo.packages_dir_name), os.path.join(str(repodir), spack.repo.packages_dir_name), ) @@ -2167,7 +2225,7 @@ def noncyclical_dir_structure(tmp_path: Path): @pytest.fixture(scope="function") def mock_config_data(): - config_data_dir = os.path.join(spack.paths.test_path, "data", "config") + config_data_dir = os.path.join(spack.paths_base.locations.test_path, "data", "config") return config_data_dir, os.listdir(config_data_dir) @@ -2482,10 +2540,6 @@ def _write(config, data, scope): return _write -def _include_cache_root(): - return join_path(str(tempfile.mkdtemp()), "user_cache", "includes") - - @pytest.fixture() def wrapper_dir(install_mockery): """Installs the compiler wrapper and returns the prefix where the script is installed.""" diff --git a/lib/spack/spack/test/data/config/config.yaml b/lib/spack/spack/test/data/config/config.yaml index 20bcd2a7e1af88..a03d9b52fcc3f0 100644 --- a/lib/spack/spack/test/data/config/config.yaml +++ b/lib/spack/spack/test/data/config/config.yaml @@ -8,7 +8,7 @@ config: build_stage: - $tempdir/$user/spack-stage source_cache: $user_cache_path/source - misc_cache: $user_cache_path/cache + misc_cache: $spack_state_home/$spack_instance_id/cache verify_ssl: true ssl_certs: $SSL_CERT_FILE checksum: true diff --git a/lib/spack/spack/test/environment_modifications.py b/lib/spack/spack/test/environment_modifications.py index 23e23f9d553988..d3402c06d99711 100644 --- a/lib/spack/spack/test/environment_modifications.py +++ b/lib/spack/spack/test/environment_modifications.py @@ -8,8 +8,8 @@ import pytest +import spack.paths import spack.util.environment as environment -from spack.paths import spack_root from spack.util.environment import ( AppendPath, EnvironmentModifications, @@ -21,7 +21,7 @@ is_system_path, ) -datadir = os.path.join(spack_root, "lib", "spack", "spack", "test", "data") +datadir = os.path.join(spack.paths.spack_root, "lib", "spack", "spack", "test", "data") shell_extension = ".bat" if sys.platform == "win32" else ".sh" diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 0eccf2dab412f4..1607f09f721932 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -186,7 +186,7 @@ def test_adhoc_version_submodules( mutable_mock_repo, monkeypatch, mock_stage, - override_git_repos_cache_path, + redirect_user_repos_cache_path, ): t = mock_git_repository.checks["tag"] # Construct the package under test diff --git a/lib/spack/spack/test/link_paths.py b/lib/spack/spack/test/link_paths.py index 5c07c8e987700f..eba591388065be 100644 --- a/lib/spack/spack/test/link_paths.py +++ b/lib/spack/spack/test/link_paths.py @@ -199,6 +199,6 @@ def test_obscure_parsing_rules(): # TODO: add a comment explaining why this happens if sys.platform == "win32": - paths.remove(os.path.join(root, "second", "path")) + spack.paths.remove(os.path.join(root, "second", "path")) check_link_paths("obscure-parsing-rules.txt", paths) diff --git a/lib/spack/spack/test/llnl/util/filesystem.py b/lib/spack/spack/test/llnl/util/filesystem.py index fc3a485c4799ec..bb5f296e9016c3 100644 --- a/lib/spack/spack/test/llnl/util/filesystem.py +++ b/lib/spack/spack/test/llnl/util/filesystem.py @@ -15,7 +15,7 @@ import pytest import spack.llnl.util.filesystem as fs -import spack.paths +from spack.paths import locations as spack_paths @pytest.fixture() @@ -505,7 +505,7 @@ def test_filter_files_with_different_encodings( # All files given as input to this test must satisfy the pre-requisite # that the 'replacement' string is not present in the file initially and # that there's at least one match for the regex - original_file = os.path.join(spack.paths.test_path, "data", "filter_file", filename) + original_file = os.path.join(spack_paths.test_path, "data", "filter_file", filename) target_file = os.path.join(str(tmp_path), filename) shutil.copy(original_file, target_file) # This should not raise exceptions @@ -553,7 +553,7 @@ def test_filter_files_multiple(tmp_path: pathlib.Path): # All files given as input to this test must satisfy the pre-requisite # that the 'replacement' string is not present in the file initially and # that there's at least one match for the regex - original_file = os.path.join(spack.paths.test_path, "data", "filter_file", "x86_cpuid_info.c") + original_file = os.path.join(spack_paths.test_path, "data", "filter_file", "x86_cpuid_info.c") target_file = os.path.join(str(tmp_path), "x86_cpuid_info.c") shutil.copy(original_file, target_file) # This should not raise exceptions @@ -568,7 +568,7 @@ def test_filter_files_multiple(tmp_path: pathlib.Path): def test_filter_files_start_stop(tmp_path: pathlib.Path): - original_file = os.path.join(spack.paths.test_path, "data", "filter_file", "start_stop.txt") + original_file = os.path.join(spack_paths.test_path, "data", "filter_file", "start_stop.txt") target_file = os.path.join(str(tmp_path), "start_stop.txt") shutil.copy(original_file, target_file) # None of the following should happen: diff --git a/lib/spack/spack/test/main.py b/lib/spack/spack/test/main.py index ba18643bca7f8b..c4ce6496e4a43b 100644 --- a/lib/spack/spack/test/main.py +++ b/lib/spack/spack/test/main.py @@ -15,7 +15,6 @@ import spack.error import spack.llnl.util.filesystem as fs import spack.main -import spack.paths import spack.platforms import spack.util.executable as exe import spack.util.git @@ -76,8 +75,8 @@ def test_git_sha_output(tmp_path: pathlib.Path, working_env, monkeypatch): assert expected == spack.get_version() -def test_get_version_no_repo(tmp_path: pathlib.Path, monkeypatch): - monkeypatch.setattr(spack.paths, "prefix", str(tmp_path)) +def test_get_version_no_repo(tmp_path: pathlib.Path, override_path): + override_path("prefix", str(tmp_path)) assert spack.spack_version == spack.get_version() diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 75378a0e974473..b576fdba96446b 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -15,8 +15,8 @@ import spack.fetch_strategy import spack.package import spack.package_base +import spack.paths import spack.repo -from spack.paths import mock_packages_path from spack.spec import Spec from spack.util.naming import pkg_name_to_class_name from spack.version import VersionChecksumError @@ -46,15 +46,17 @@ def test_package_name(self): assert pkg_cls.name == "mpich" def test_package_filename(self): - repo = spack.repo.from_path(mock_packages_path) + repo = spack.repo.from_path(spack.paths.mock_packages_path) filename = repo.filename_for_package_name("mpich") - assert filename == os.path.join(mock_packages_path, "packages", "mpich", "package.py") + assert filename == os.path.join( + spack.paths.mock_packages_path, "packages", "mpich", "package.py" + ) def test_nonexisting_package_filename(self): - repo = spack.repo.from_path(mock_packages_path) + repo = spack.repo.from_path(spack.paths.mock_packages_path) filename = repo.filename_for_package_name("some-nonexisting-package") assert filename == os.path.join( - mock_packages_path, "packages", "some_nonexisting_package", "package.py" + spack.paths.mock_packages_path, "packages", "some_nonexisting_package", "package.py" ) def test_package_class_names(self): diff --git a/lib/spack/spack/test/packaging.py b/lib/spack/spack/test/packaging.py index 3cfae44d84ed33..d744ff2f5ffd7e 100644 --- a/lib/spack/spack/test/packaging.py +++ b/lib/spack/spack/test/packaging.py @@ -24,6 +24,7 @@ import spack.error import spack.fetch_strategy import spack.package_base +import spack.paths import spack.stage import spack.util.gpg import spack.util.url as url_util @@ -31,7 +32,6 @@ from spack.installer import PackageInstaller from spack.llnl.util import filesystem as fs from spack.llnl.util.filesystem import readlink, symlink -from spack.paths import mock_gpg_keys_path from spack.relocate import _macho_find_paths, relocate_links, relocate_text pytestmark = pytest.mark.not_on_windows("does not run on windows") @@ -110,7 +110,9 @@ def test_buildcache(mock_archive, tmp_path: pathlib.Path, monkeypatch, mutable_c buildcache.buildcache(parser, args) # Copy a key to the mirror to have something to download - shutil.copyfile(mock_gpg_keys_path + "/external.key", mirror_path + "/external.key") + shutil.copyfile( + spack.paths.mock_gpg_keys_path + "/external.key", mirror_path + "/external.key" + ) args = parser.parse_args(["keys"]) buildcache.buildcache(parser, args) diff --git a/lib/spack/spack/test/paths.py b/lib/spack/spack/test/paths.py new file mode 100644 index 00000000000000..46359cd48eb9ce --- /dev/null +++ b/lib/spack/spack/test/paths.py @@ -0,0 +1,319 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import pathlib + +import pytest + +import spack.paths +import spack.paths_base +from spack.paths import SpackPaths +from spack.paths_base import SpackPathsBase + + +def _ensure_dir(pathlike): + pathlike.mkdir(parents=True, exist_ok=True) + return str(pathlike) + + +@pytest.fixture(autouse=True) +def clear_env_vars(working_env): + """Scrub XDG_*/SPACK_* env vars so each test starts from a known state.""" + spack.paths._unset_path_vars(os.environ) + + +def _set_locations(cfg, **kwargs): + """Set config:locations:* keys (avoiding the leftover state pitfall of + overwriting the whole locations block).""" + for k, v in kwargs.items(): + cfg.set(f"config:locations:{k}", v) + + +# --------------------------------------------------------------------------- +# Home property resolution +# --------------------------------------------------------------------------- + + +def test_data_home_from_config(working_env, tmp_path, mutable_config, set_home): + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, data=str(tmp_path / "datadir")) + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.data_home == str(tmp_path / "datadir") + + +def test_state_home_from_config(working_env, tmp_path, mutable_config, set_home): + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, state=str(tmp_path / "statedir")) + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.state_home == str(tmp_path / "statedir") + + +def test_cache_home_from_config(working_env, tmp_path, mutable_config, set_home): + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, cache=str(tmp_path / "cachedir")) + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.cache_home == str(tmp_path / "cachedir") + + +def test_spack_home_from_config(working_env, tmp_path, mutable_config, set_home): + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, home=str(tmp_path / "spackhome")) + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.spack_home == str(tmp_path / "spackhome") + + +def test_data_home_env_overrides_config(working_env, tmp_path, mutable_config, set_home): + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, data=str(tmp_path / "fromconfig")) + os.environ["SPACK_DATA_HOME"] = str(tmp_path / "fromenv") + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.data_home == str(tmp_path / "fromenv") + + +def test_spack_home_env_overrides_subhomes(working_env, tmp_path, mutable_config, set_home): + """SPACK_HOME (no _DATA_/_STATE_/_CACHE_) provides a root and the homes + derive from it via their XDG-style subpaths.""" + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, data=str(tmp_path / "fromconfig")) + os.environ["SPACK_HOME"] = str(tmp_path / "alt-home") + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.data_home == str(tmp_path / "alt-home" / ".local" / "share" / "spack") + assert p.state_home == str(tmp_path / "alt-home" / ".local" / "state" / "spack") + assert p.cache_home == str(tmp_path / "alt-home" / ".cache" / "spack") + + +def test_specific_env_overrides_spack_home(working_env, tmp_path, mutable_config, set_home): + """SPACK_DATA_HOME beats SPACK_HOME for data_home.""" + set_home(_ensure_dir(tmp_path / "home")) + os.environ["SPACK_HOME"] = str(tmp_path / "alt-home") + os.environ["SPACK_DATA_HOME"] = str(tmp_path / "specific-data") + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.data_home == str(tmp_path / "specific-data") + + +def test_disable_env_ignores_env_vars(working_env, tmp_path, mutable_config, set_home): + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, data=str(tmp_path / "fromconfig")) + mutable_config.set("config:locations:disable_env", True) + os.environ["SPACK_DATA_HOME"] = str(tmp_path / "fromenv") + os.environ["SPACK_HOME"] = str(tmp_path / "alt-home") + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.data_home == str(tmp_path / "fromconfig") + + +# --------------------------------------------------------------------------- +# SPACK_USER_CACHE_PATH is a legacy alias for SPACK_STATE_HOME +# --------------------------------------------------------------------------- + + +def test_user_cache_path_env_sets_state_home(working_env, tmp_path): + target = str(tmp_path / "cache") + os.environ["SPACK_USER_CACHE_PATH"] = target + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "base"))) + assert p.user_cache_path == target + assert p.state_home == target + # Things rooted under state_home follow. + assert p.package_repos_path == os.path.join(target, "package_repos") + + +def test_state_home_env_overrides_user_cache_path(working_env, tmp_path): + """When both SPACK_USER_CACHE_PATH and SPACK_STATE_HOME are set, the + older one (SPACK_USER_CACHE_PATH) wins, matching pre-1.2 behavior.""" + legacy = str(tmp_path / "legacy") + new = str(tmp_path / "new") + os.environ["SPACK_USER_CACHE_PATH"] = legacy + os.environ["SPACK_STATE_HOME"] = new + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "base"))) + assert p.state_home == legacy + + +# --------------------------------------------------------------------------- +# Config path overrides (SPACK_USER_CONFIG_PATH, SPACK_SYSTEM_CONFIG_PATH) +# These still live in paths_base because they bootstrap config itself. +# --------------------------------------------------------------------------- + + +def test_user_config_path_is_overridable(working_env, tmp_path): + target = str(tmp_path / "redirected_usrcfg") + os.environ["SPACK_USER_CONFIG_PATH"] = target + pb = SpackPathsBase(_ensure_dir(tmp_path / "base-prefix")) + assert pb.user_config_path == target + + +def test_user_config_path_default(working_env, tmp_path): + os.environ["SPACK_USER_CONFIG_PATH"] = "" + pb = SpackPathsBase(str(tmp_path)) + assert pb.user_config_path == os.path.expanduser(os.path.join("~", ".config", "spack")) + + +def test_system_config_path_is_overridable(working_env, tmp_path): + target = str(tmp_path / "redirected_syscfg") + os.environ["SPACK_SYSTEM_CONFIG_PATH"] = target + pb = SpackPathsBase(_ensure_dir(tmp_path / "base-prefix")) + assert pb.system_config_path == target + + +def test_system_config_path_default(working_env, tmp_path): + os.environ["SPACK_SYSTEM_CONFIG_PATH"] = "" + pb = SpackPathsBase(str(tmp_path)) + assert pb.system_config_path == os.sep + os.path.join("etc", "spack") + + +# --------------------------------------------------------------------------- +# Layout detection (detect_layout / layout_detected) — the helper exposed +# to include `when:` clauses. +# --------------------------------------------------------------------------- + + +@pytest.fixture +def empty_spack_root(tmp_path): + base = SpackPathsBase(_ensure_dir(tmp_path / "spack-root")) + yield base + + +@pytest.fixture +def occupied_spack_root(tmp_path): + base = SpackPathsBase(_ensure_dir(tmp_path / "spack-root")) + # Put something in old install location to trigger old-layout detection. + inst = pathlib.Path(base.old_install_path) + inst.mkdir(parents=True, exist_ok=True) + (inst / "marker").touch() + yield base + + +def test_detect_layout_empty_root_is_xdg(working_env, empty_spack_root, monkeypatch): + monkeypatch.setattr(spack.paths_base, "locations", empty_spack_root) + assert spack.paths.detect_layout("old") is False + assert spack.paths.detect_layout("xdg") is True + + +def test_detect_layout_old_data_present(working_env, occupied_spack_root, monkeypatch): + monkeypatch.setattr(spack.paths_base, "locations", occupied_spack_root) + assert spack.paths.detect_layout("old") is True + assert spack.paths.detect_layout("xdg") is False + + +def test_detect_layout_env_var_forces_xdg(working_env, occupied_spack_root, monkeypatch): + """Even with old-layout markers, any SPACK_*_HOME env var forces xdg.""" + monkeypatch.setattr(spack.paths_base, "locations", occupied_spack_root) + for var in ("SPACK_DATA_HOME", "SPACK_STATE_HOME", "SPACK_CACHE_HOME", "SPACK_HOME"): + os.environ.pop(var, None) + os.environ[var] = "/tmp/somewhere" + try: + assert spack.paths.detect_layout("old") is False, f"{var} should force xdg" + finally: + del os.environ[var] + + +def test_detect_layout_rejects_unknown_scheme(): + with pytest.raises(ValueError, match="unknown layout scheme"): + spack.paths.detect_layout("eggplant") + + +def test_layout_detected_in_eval_conditional(occupied_spack_root, monkeypatch): + """The helper is reachable from include `when:` clauses via eval_conditional.""" + import spack.spec + + monkeypatch.setattr(spack.paths_base, "locations", occupied_spack_root) + assert spack.spec.eval_conditional("layout_detected('old')") is True + assert spack.spec.eval_conditional("not layout_detected('old')") is False + assert spack.spec.eval_conditional("layout_detected('xdg')") is False + + +# --------------------------------------------------------------------------- +# $xdg_data_home / $xdg_state_home / $xdg_cache_home substitutions +# --------------------------------------------------------------------------- + + +def test_xdg_substitutions_respect_env_vars(working_env, tmp_path): + from spack.util.path import canonicalize_path + + os.environ["XDG_DATA_HOME"] = str(tmp_path / "xdg-data") + os.environ["XDG_STATE_HOME"] = str(tmp_path / "xdg-state") + os.environ["XDG_CACHE_HOME"] = str(tmp_path / "xdg-cache") + + assert canonicalize_path("$xdg_data_home/spack") == str(tmp_path / "xdg-data" / "spack") + assert canonicalize_path("$xdg_state_home/spack") == str(tmp_path / "xdg-state" / "spack") + assert canonicalize_path("$xdg_cache_home/spack") == str(tmp_path / "xdg-cache" / "spack") + + +def test_xdg_substitutions_fall_back_to_defaults(working_env, set_home, tmp_path): + from spack.util.path import canonicalize_path + + home = _ensure_dir(tmp_path / "home") + set_home(home) + assert canonicalize_path("$xdg_data_home/spack") == os.path.join( + home, ".local", "share", "spack" + ) + assert canonicalize_path("$xdg_state_home/spack") == os.path.join( + home, ".local", "state", "spack" + ) + assert canonicalize_path("$xdg_cache_home/spack") == os.path.join(home, ".cache", "spack") + + +# --------------------------------------------------------------------------- +# Derived paths (reports, package_repos, etc.) follow state_home +# --------------------------------------------------------------------------- + + +def test_derived_paths_follow_state_home(working_env, tmp_path, mutable_config, set_home): + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, state=str(tmp_path / "statedir")) + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.reports_path == str(tmp_path / "statedir" / "reports") + assert p.default_monitor_path == str(tmp_path / "statedir" / "reports" / "monitor") + assert p.user_repos_cache_path == str(tmp_path / "statedir" / "git_repos") + assert p.package_repos_path == str(tmp_path / "statedir" / "package_repos") + + +def test_dotspack_backup_pinned_to_xdg_default(working_env, tmp_path, mutable_config, set_home): + """dotspack_backup is pinned to ~/.local/share/spack/dotspack_backup and + does NOT follow data_home, so the migration backup lives in a stable + location regardless of SPACK_DATA_HOME / config:locations:data.""" + home = _ensure_dir(tmp_path / "home") + set_home(home) + _set_locations(mutable_config, data=str(tmp_path / "datadir")) # ignored by dotspack_backup + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.dotspack_backup == os.path.join(home, ".local", "share", "spack", "dotspack_backup") + + +# --------------------------------------------------------------------------- +# gpg_path / gpg_keys_path read config first, fall back to data_home +# --------------------------------------------------------------------------- + + +def test_gpg_path_from_config(working_env, tmp_path, mutable_config, set_home): + set_home(_ensure_dir(tmp_path / "home")) + mutable_config.set("config:gpg_path", str(tmp_path / "my-gpg")) + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.gpg_path == str(tmp_path / "my-gpg") + + +def test_gpg_path_default_from_scheme(working_env, tmp_path, mutable_config, set_home): + """The xdg scheme yaml sets config:gpg_path to $data_home/gpg by default; + the property simply reads that.""" + set_home(_ensure_dir(tmp_path / "home")) + _set_locations(mutable_config, data=str(tmp_path / "datadir")) + # base/config.yaml sets gpg_path to $data_home/gpg + mutable_config.set("config:gpg_path", "$data_home/gpg") + p = SpackPaths(SpackPathsBase(_ensure_dir(tmp_path / "spack-root"))) + assert p.gpg_path == str(tmp_path / "datadir" / "gpg") + + +# --------------------------------------------------------------------------- +# Module shim: spack.paths.X works for both static and dynamic attributes +# --------------------------------------------------------------------------- + + +def test_module_shim_static_attribute(): + # `prefix` lives on paths_base.SpackPathsBase; spack.paths.prefix + # should reach it via the SpackPaths.__getattr__ delegation. + assert spack.paths.prefix == spack.paths.locations.prefix + + +def test_module_shim_dynamic_attribute(): + # `state_home` is computed by SpackPaths; access via the module + # should also work. + assert spack.paths.state_home == spack.paths.locations.state_home diff --git a/lib/spack/spack/test/repo.py b/lib/spack/spack/test/repo.py index 0cba89b5167fb1..a647425bccd1d4 100644 --- a/lib/spack/spack/test/repo.py +++ b/lib/spack/spack/test/repo.py @@ -210,8 +210,6 @@ def test_path_computation_with_names(method_name, mock_packages_repo): def test_use_repositories_and_import(): """Tests that use_repositories changes the import search too""" - import spack.paths - repo_dir = pathlib.Path(spack.paths.test_repos_path) with spack.repo.use_repositories(str(repo_dir / "spack_repo" / "compiler_runtime_test")): import spack_repo.compiler_runtime_test.packages.gcc_runtime.package # type: ignore[import] # noqa: E501 diff --git a/lib/spack/spack/test/util/compression.py b/lib/spack/spack/test/util/compression.py index 020addce8bf3f8..1b82191b30be52 100644 --- a/lib/spack/spack/test/util/compression.py +++ b/lib/spack/spack/test/util/compression.py @@ -12,12 +12,14 @@ import pytest import spack.llnl.url +import spack.paths from spack.llnl.util.filesystem import working_dir -from spack.paths import spack_root from spack.util import compression from spack.util.executable import CommandNotFoundError -datadir = os.path.join(spack_root, "lib", "spack", "spack", "test", "data", "compression") +datadir = os.path.join( + spack.paths.spack_root, "lib", "spack", "spack", "test", "data", "compression" +) ext_archive = { ext: f"Foo.{ext}" for ext in spack.llnl.url.ALLOWED_ARCHIVE_TYPES if "TAR" not in ext diff --git a/lib/spack/spack/trace.py b/lib/spack/spack/trace.py new file mode 100644 index 00000000000000..776c6ab513e153 --- /dev/null +++ b/lib/spack/spack/trace.py @@ -0,0 +1,90 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import inspect +import pathlib +import warnings + +from spack.paths_base import locations as paths_base + + +def _most_recent_internal_call(): + """If called within an audit for a Python library function, finds + the most recent spot within Spack's source code that generated + the call. + """ + + stack = inspect.stack() + this_file = str(pathlib.Path(__file__).resolve()) + spack_prefix = pathlib.Path(paths_base.prefix).resolve() + for frame in stack: + frame_loc = pathlib.Path(frame.filename).resolve() + if str(frame_loc) != this_file and spack_prefix in frame_loc.parents: + return frame_loc, frame.lineno + + return None, None + + +_recorded_accesses = set() + + +def _attempted_modify_internal(msg): + loc, line = _most_recent_internal_call() + if loc: + if (loc, line) not in _recorded_accesses: + _recorded_accesses.add((loc, line)) + msg += f" at {loc}:{line}" + warnings.warn(msg) + else: + msg += " (no location)" + warnings.warn(msg) + + +def _real(path): + return pathlib.Path(path).absolute().resolve() + + +_real_spack_prefix = _real(paths_base.prefix) + + +def _is_in_spack_prefix(path): + return _real_spack_prefix in _real(path).parents + + +def _guard_writes(event, args): + # Note: this doesn't catch files opened in "r" mode and then + # later upgraded to "w" mode (e.g. our locks). I think to track + # that properly we would need to (a) patch builtins.open to + # map all paths to FDs as they are opened (and delete on close) + # and (b) audit fcntl.fcntl events, using reverse mapping on + # FD to check associated path + if event == "open": + path, mode = args[:2] + if not mode: + # Some internal Python libs can call open(..., mode=None) + return + if not isinstance(path, str): + # Skip instances of open() that function like fdopen + return + intent_to_modify = bool((set(mode) & set("wax")) or "r+" in mode) + if _is_in_spack_prefix(path) and intent_to_modify: + _attempted_modify_internal(f"Open {path} in mode [{mode}]") + elif event in ["shutil.copyfile", "os.rename", "shutil.move"]: + _, dst = args[:2] + if _is_in_spack_prefix(dst): + _attempted_modify_internal(f"copy dst {str(_real(dst))}") + elif event == "os.mkdir": + path = args[0] + if _is_in_spack_prefix(path): + _attempted_modify_internal(f"mkdir {str(_real(path))}") + + +def warn_writes_into_spack(): + import sys + + if sys.version_info[:2] >= (3, 8): + sys.addaudithook(_guard_writes) # novermin + else: + raise ValueError( + f"Cannot detect writes for Python {sys.version_info[:2]} (supported in 3.8 and later)" + ) diff --git a/lib/spack/spack/util/gpg.py b/lib/spack/spack/util/gpg.py index 406c60b77fd695..a2232d6c7b2e9b 100644 --- a/lib/spack/spack/util/gpg.py +++ b/lib/spack/spack/util/gpg.py @@ -57,7 +57,9 @@ def init(gnupghome=None, force=False): return # Set the value of GNUPGHOME to be used in this module - GNUPGHOME = gnupghome or os.getenv("SPACK_GNUPGHOME") or spack.paths.gpg_path + GNUPGHOME = gnupghome or os.getenv("SPACK_GNUPGHOME") + if not GNUPGHOME: + GNUPGHOME = spack.paths.gpg_path # Set the executable objects for "gpg" and "gpgconf" with spack.bootstrap.ensure_bootstrap_configuration(): diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index bbd6a27f506dac..11b48e6ec82e61 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -59,15 +59,19 @@ def replacements(): import spack import spack.environment as ev import spack.paths + import spack.paths_base + paths = spack.paths.locations arch = architecture() - return { - "spack": lambda: spack.paths.prefix, + def xdg_or_default(var, default): + return os.environ.get(var) or os.path.expanduser(default) + + replace = { + "spack": lambda: spack.paths_base.locations.prefix, "user": lambda: get_user(), "tempdir": lambda: tempfile.gettempdir(), - "user_cache_path": lambda: spack.paths.user_cache_path, - "spack_instance_id": lambda: spack.paths.spack_instance_id, + "spack_instance_id": lambda: spack.paths_base.locations.spack_instance_id, "architecture": lambda: arch, "arch": lambda: arch, "platform": lambda: arch.platform, @@ -78,8 +82,21 @@ def replacements(): "date": lambda: date.today().strftime("%Y-%m-%d"), "env": lambda: ev.active_environment().path if ev.active_environment() else NOMATCH, "spack_short_version": lambda: spack.get_short_version(), + "user_cache_path": lambda: paths.user_cache_path, + "data_home": lambda: paths.data_home, + "cache_home": lambda: paths.cache_home, + "state_home": lambda: paths.state_home, + "spack_home": lambda: paths.spack_home, + # XDG base-dir env vars with their spec defaults. Used by the xdg + # scheme yaml so $XDG_*_HOME is respected without forcing all of + # paths.py through Python. See etc/spack/defaults/xdg/config.yaml. + "xdg_data_home": lambda: xdg_or_default("XDG_DATA_HOME", "~/.local/share"), + "xdg_state_home": lambda: xdg_or_default("XDG_STATE_HOME", "~/.local/state"), + "xdg_cache_home": lambda: xdg_or_default("XDG_CACHE_HOME", "~/.cache"), } + return replace + # This is intended to be longer than the part of the install path # spack generates from the root path we give it. Included in the @@ -161,21 +178,28 @@ def substitute_config_variables(path): Spack allows paths in configs to have some placeholders, as follows: - - $env The active Spack environment. - - $spack The Spack instance's prefix - - $tempdir Default temporary directory returned by tempfile.gettempdir() - - $user The current user's username - - $user_cache_path The user cache directory (~/.spack, unless overridden) - - $spack_instance_id Hash that distinguishes Spack instances on the filesystem - - $architecture The spack architecture triple for the current system - - $arch The spack architecture triple for the current system - - $platform The spack platform for the current system - - $os The OS of the current system - - $operating_system The OS of the current system - - $target The ISA target detected for the system - - $target_family The family of the target detected for the system - - $date The current date (YYYY-MM-DD) - - $spack_short_version The spack short version + - $env The active Spack environment. + - $spack The Spack instance's prefix + - $spack_home Base for spack's user-level data; defaults to ``~`` + - $tempdir Default temporary directory returned by tempfile.gettempdir() + - $user The current user's username + - $data_home SPACK_DATA_HOME, config:locations:data, XDG_DATA_HOME, or default + - $cache_home SPACK_CACHE_HOME, config:locations:cache, XDG_CACHE_HOME, or default + - $state_home SPACK_STATE_HOME, config:locations:state, XDG_STATE_HOME, or default + - $user_cache_path The user cache directory (same as state_home) + - $xdg_data_home $XDG_DATA_HOME or ``~/.local/share`` (no ``/spack`` suffix) + - $xdg_state_home $XDG_STATE_HOME or ``~/.local/state`` + - $xdg_cache_home $XDG_CACHE_HOME or ``~/.cache`` + - $spack_instance_id Hash that distinguishes Spack instances on the filesystem + - $architecture The spack architecture triple for the current system + - $arch The spack architecture triple for the current system + - $platform The spack platform for the current system + - $os The OS of the current system + - $operating_system The OS of the current system + - $target The ISA target detected for the system + - $target_family The family of the target detected for the system + - $date The current date (YYYY-MM-DD) + - $spack_short_version The spack short version These are substituted case-insensitively into the path, and users can use either ``$var`` or ``${var}`` syntax for the variables. $env is only @@ -192,7 +216,12 @@ def repl(match): return m if repl is NOMATCH else str(repl) # Replace $var or ${var}. - return re.sub(r"(\$\w+\b|\$\{\w+\})", repl, path) + # Iterate in case values refer to other config variables + new_path = re.sub(r"(\$\w+\b|\$\{\w+\})", repl, path) + while path != new_path: + path = new_path + new_path = re.sub(r"(\$\w+\b|\$\{\w+\})", repl, path) + return new_path def substitute_path_variables(path): diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 315bc0edf75237..392650a9cb6d69 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -394,9 +394,9 @@ SPACK_ALIASES="concretise:concretize;containerise:containerize;rm:remove" _spack() { if $list_options then - SPACK_COMPREPLY="--color -v --verbose -k --insecure -b --bootstrap -V --version -h --help -H --all-help -c --config -C --config-scope -e --env -D --env-dir -E --no-env --use-env-repo -d --debug -t --backtrace --pdb --timestamp -m --mock --print-shell-vars --stacktrace -l --enable-locks -L --disable-locks -p --profile --profile-file --sorted-profile --lines" + SPACK_COMPREPLY="--color -v --verbose -k --insecure -b --bootstrap -V --version -h --help -H --all-help -c --config -C --config-scope -e --env -D --env-dir -E --no-env --use-env-repo -d --debug -t --backtrace --pdb --timestamp -m --mock --print-shell-vars --stacktrace --warn-writes-into-spack -l --enable-locks -L --disable-locks -p --profile --profile-file --sorted-profile --lines" else - SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse logs maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" + SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse logs maintainers make-installer mark migrate mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" fi } @@ -677,7 +677,7 @@ _spack_buildcache_migrate() { _spack_cd() { if $list_options then - SPACK_COMPREPLY="-h --help -m --module-dir -r --spack-root -i --install-dir -p --package-dir --repo --packages -P -s --stage-dir -S --stages -c --source-dir -b --build-dir -e --env -v --view --first" + SPACK_COMPREPLY="-h --help -m --module-dir -r --spack-root -i --install-dir --install-root -p --package-dir --repo --packages -P -s --stage-dir -S --stages -c --source-dir -b --build-dir -e --env -v --view --first" else _all_packages fi @@ -961,7 +961,7 @@ _spack_debug() { then SPACK_COMPREPLY="-h --help" else - SPACK_COMPREPLY="report" + SPACK_COMPREPLY="report paths" fi } @@ -969,6 +969,10 @@ _spack_debug_report() { SPACK_COMPREPLY="-h --help" } +_spack_debug_paths() { + SPACK_COMPREPLY="-h --help" +} + _spack_deconcretize() { if $list_options then @@ -1406,7 +1410,7 @@ _spack_load() { _spack_location() { if $list_options then - SPACK_COMPREPLY="-h --help -m --module-dir -r --spack-root -i --install-dir -p --package-dir --repo --packages -P -s --stage-dir -S --stages -c --source-dir -b --build-dir -e --env -v --view --first" + SPACK_COMPREPLY="-h --help -m --module-dir -r --spack-root -i --install-dir --install-root -p --package-dir --repo --packages -P -s --stage-dir -S --stages -c --source-dir -b --build-dir -e --env -v --view --first" else _all_packages fi @@ -1457,6 +1461,10 @@ _spack_mark() { fi } +_spack_migrate() { + SPACK_COMPREPLY="-h --help --dry-run --clear --clear-only --replace --restore --i-need-old-spack" +} + _spack_mirror() { if $list_options then diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 5a0a496923cdf7..90f14d28a1dc71 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -346,7 +346,7 @@ complete -c spack --erase # Everything below here is auto-generated. # spack -set -g __fish_spack_optspecs_spack color= v/verbose k/insecure b/bootstrap V/version h/help H/all-help c/config= C/config-scope= e/env= D/env-dir= E/no-env use-env-repo d/debug t/backtrace pdb timestamp m/mock print-shell-vars= stacktrace l/enable-locks L/disable-locks p/profile profile-file= sorted-profile= lines= +set -g __fish_spack_optspecs_spack color= v/verbose k/insecure b/bootstrap V/version h/help H/all-help c/config= C/config-scope= e/env= D/env-dir= E/no-env use-env-repo d/debug t/backtrace pdb timestamp m/mock print-shell-vars= stacktrace warn-writes-into-spack l/enable-locks L/disable-locks p/profile profile-file= sorted-profile= lines= complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a add -d 'add a spec to an environment' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a arch -d 'print architecture information about this machine' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a audit -d 'audit configuration files, packages, etc.' @@ -398,6 +398,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a logs -d 'print ou complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a maintainers -d 'get information about package maintainers' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a make-installer -d 'generate Windows installer' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a mark -d 'mark packages as explicitly or implicitly installed' +complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a migrate -d 'migrate user config and cache from old to new locations' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a mirror -d 'manage mirrors (source and binary)' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a module -d 'generate/manage module files' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a patch -d 'patch expanded sources in preparation for install' @@ -466,6 +467,8 @@ complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -f -a complete -c spack -n '__fish_spack_using_command ' -l print-shell-vars -r -d 'print info needed by setup-env.*sh' complete -c spack -n '__fish_spack_using_command ' -l stacktrace -f -a stacktrace complete -c spack -n '__fish_spack_using_command ' -l stacktrace -d 'add stacktraces to all printed statements' +complete -c spack -n '__fish_spack_using_command ' -l warn-writes-into-spack -f -a warn_writes_into_spack +complete -c spack -n '__fish_spack_using_command ' -l warn-writes-into-spack -d 'Warn when Spack tries to write into its own prefix' complete -c spack -n '__fish_spack_using_command ' -s l -l enable-locks -f -a locks complete -c spack -n '__fish_spack_using_command ' -s l -l enable-locks -d 'use filesystem locking (default)' complete -c spack -n '__fish_spack_using_command ' -s L -l disable-locks -f -a locks @@ -598,7 +601,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_enable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap enable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap enable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap enable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap disable @@ -606,7 +609,7 @@ set -g __fish_spack_optspecs_spack_bootstrap_disable h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap disable' -f -a '(__fish_spack_bootstrap_names)' complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap disable' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap disable' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap reset @@ -621,14 +624,14 @@ set -g __fish_spack_optspecs_spack_bootstrap_root h/help scope= complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap root' -f -a '(__fish_complete_directories)' complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap root' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap root' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap list set -g __fish_spack_optspecs_spack_bootstrap_list h/help scope= complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap list' -l scope -r -d 'configuration scope to read/modify' # spack bootstrap add @@ -637,7 +640,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 bootstrap add' -f -a '(__ complete -c spack -n '__fish_spack_using_command_pos 1 bootstrap add' -f -a '(__fish_spack_environments)' complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command bootstrap add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command bootstrap add' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -f -a trust complete -c spack -n '__fish_spack_using_command bootstrap add' -l trust -d 'enable the source immediately upon addition' @@ -823,7 +826,7 @@ complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirro complete -c spack -n '__fish_spack_using_command buildcache check' -s m -l mirror-url -r -d 'override any configured mirrors with this mirror URL' complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -f -a output_file complete -c spack -n '__fish_spack_using_command buildcache check' -s o -l output-file -r -d 'file where rebuild info should be written' -complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command buildcache check' -l scope -r -d 'configuration scope containing mirrors to check' # spack buildcache download @@ -921,7 +924,7 @@ complete -c spack -n '__fish_spack_using_command buildcache migrate' -s y -l yes complete -c spack -n '__fish_spack_using_command buildcache migrate' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request' # spack cd -set -g __fish_spack_optspecs_spack_cd h/help m/module-dir r/spack-root i/install-dir p/package-dir repo= s/stage-dir S/stages c/source-dir b/build-dir e/env= v/view= first +set -g __fish_spack_optspecs_spack_cd h/help m/module-dir r/spack-root i/install-dir install-root p/package-dir repo= s/stage-dir S/stages c/source-dir b/build-dir e/env= v/view= first complete -c spack -n '__fish_spack_using_command_pos_remainder 0 cd' -f -k -a '(__fish_spack_specs)' complete -c spack -n '__fish_spack_using_command cd' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command cd' -s h -l help -d 'show this help message and exit' @@ -931,6 +934,8 @@ complete -c spack -n '__fish_spack_using_command cd' -s r -l spack-root -f -a sp complete -c spack -n '__fish_spack_using_command cd' -s r -l spack-root -d 'spack installation root' complete -c spack -n '__fish_spack_using_command cd' -s i -l install-dir -f -a install_dir complete -c spack -n '__fish_spack_using_command cd' -s i -l install-dir -d 'install prefix for spec (spec need not be installed)' +complete -c spack -n '__fish_spack_using_command cd' -l install-root -f -a install_root +complete -c spack -n '__fish_spack_using_command cd' -l install-root -d 'where spack installs specs' complete -c spack -n '__fish_spack_using_command cd' -s p -l package-dir -f -a package_dir complete -c spack -n '__fish_spack_using_command cd' -s p -l package-dir -d 'directory enclosing a spec'"'"'s package.py file' complete -c spack -n '__fish_spack_using_command cd' -l repo -l packages -s P -r -f -a repo @@ -1131,7 +1136,7 @@ set -g __fish_spack_optspecs_spack_compiler_find h/help scope= j/jobs= complete -c spack -n '__fish_spack_using_command compiler find' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler find' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1141,7 +1146,7 @@ set -g __fish_spack_optspecs_spack_compiler_add h/help scope= j/jobs= complete -c spack -n '__fish_spack_using_command compiler add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -1153,7 +1158,7 @@ complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help - complete -c spack -n '__fish_spack_using_command compiler remove' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler remove' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler remove' -l scope -r -d 'configuration scope to modify' # spack compiler rm @@ -1163,14 +1168,14 @@ complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command compiler rm' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -f -a all complete -c spack -n '__fish_spack_using_command compiler rm' -s a -l all -d 'remove ALL compilers that match spec' -complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler rm' -l scope -r -d 'configuration scope to modify' # spack compiler list set -g __fish_spack_optspecs_spack_compiler_list h/help scope= remote complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler list' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler list' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler list' -l remote -d 'list also compilers from registered buildcaches' @@ -1179,7 +1184,7 @@ complete -c spack -n '__fish_spack_using_command compiler list' -l remote -d 'li set -g __fish_spack_optspecs_spack_compiler_ls h/help scope= remote complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler ls' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler ls' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler ls' -l remote -d 'list also compilers from registered buildcaches' @@ -1189,7 +1194,7 @@ set -g __fish_spack_optspecs_spack_compiler_info h/help scope= remote complete -c spack -n '__fish_spack_using_command_pos 0 compiler info' -f -a '(__fish_spack_installed_compilers)' complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler info' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compiler info' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command compiler info' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compiler info' -l remote -d 'list also compilers from registered buildcaches' @@ -1198,7 +1203,7 @@ complete -c spack -n '__fish_spack_using_command compiler info' -l remote -d 'li set -g __fish_spack_optspecs_spack_compilers h/help scope= remote complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compilers' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command compilers' -l scope -r -d 'configuration scope to read/modify' complete -c spack -n '__fish_spack_using_command compilers' -l remote -f -a remote complete -c spack -n '__fish_spack_using_command compilers' -l remote -d 'list also compilers from registered buildcaches' @@ -1265,7 +1270,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a update -d ' complete -c spack -n '__fish_spack_using_command_pos 0 config' -f -a revert -d 'revert configuration files to their state before update' complete -c spack -n '__fish_spack_using_command config' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command config' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command config' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command config' -l scope -r -d 'configuration scope to read/modify' # spack config get @@ -1405,6 +1410,7 @@ complete -c spack -n '__fish_spack_using_command create' -s b -l batch -d 'don'" # spack debug set -g __fish_spack_optspecs_spack_debug h/help complete -c spack -n '__fish_spack_using_command_pos 0 debug' -f -a report -d 'print information useful for bug reports' +complete -c spack -n '__fish_spack_using_command_pos 0 debug' -f -a paths -d 'show how each Spack data path was resolved (active layout scheme, config keys, env-var overrides)' complete -c spack -n '__fish_spack_using_command debug' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command debug' -s h -l help -d 'show this help message and exit' @@ -1413,6 +1419,11 @@ set -g __fish_spack_optspecs_spack_debug_report h/help complete -c spack -n '__fish_spack_using_command debug report' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command debug report' -s h -l help -d 'show this help message and exit' +# spack debug paths +set -g __fish_spack_optspecs_spack_debug_paths h/help +complete -c spack -n '__fish_spack_using_command debug paths' -s h -l help -f -a help +complete -c spack -n '__fish_spack_using_command debug paths' -s h -l help -d 'show this help message and exit' + # spack deconcretize set -g __fish_spack_optspecs_spack_deconcretize h/help root y/yes-to-all a/all complete -c spack -n '__fish_spack_using_command_pos_remainder 0 deconcretize' -f -k -a '(__fish_spack_specs)' @@ -1825,7 +1836,7 @@ complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -f complete -c spack -n '__fish_spack_using_command external find' -l exclude -r -d 'packages to exclude from search' complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -f -a path complete -c spack -n '__fish_spack_using_command external find' -s p -l path -r -d 'one or more alternative search paths for finding externals' -complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command external find' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command external find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command external find' -l all -f -a all complete -c spack -n '__fish_spack_using_command external find' -l all -d 'search for all packages that Spack knows about' @@ -2262,7 +2273,7 @@ complete -c spack -n '__fish_spack_using_command load' -l list -f -a list complete -c spack -n '__fish_spack_using_command load' -l list -d 'show loaded packages: same as ``spack find --loaded``' # spack location -set -g __fish_spack_optspecs_spack_location h/help m/module-dir r/spack-root i/install-dir p/package-dir repo= s/stage-dir S/stages c/source-dir b/build-dir e/env= v/view= first +set -g __fish_spack_optspecs_spack_location h/help m/module-dir r/spack-root i/install-dir install-root p/package-dir repo= s/stage-dir S/stages c/source-dir b/build-dir e/env= v/view= first complete -c spack -n '__fish_spack_using_command_pos_remainder 0 location' -f -k -a '(__fish_spack_specs)' complete -c spack -n '__fish_spack_using_command location' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command location' -s h -l help -d 'show this help message and exit' @@ -2272,6 +2283,8 @@ complete -c spack -n '__fish_spack_using_command location' -s r -l spack-root -f complete -c spack -n '__fish_spack_using_command location' -s r -l spack-root -d 'spack installation root' complete -c spack -n '__fish_spack_using_command location' -s i -l install-dir -f -a install_dir complete -c spack -n '__fish_spack_using_command location' -s i -l install-dir -d 'install prefix for spec (spec need not be installed)' +complete -c spack -n '__fish_spack_using_command location' -l install-root -f -a install_root +complete -c spack -n '__fish_spack_using_command location' -l install-root -d 'where spack installs specs' complete -c spack -n '__fish_spack_using_command location' -s p -l package-dir -f -a package_dir complete -c spack -n '__fish_spack_using_command location' -s p -l package-dir -d 'directory enclosing a spec'"'"'s package.py file' complete -c spack -n '__fish_spack_using_command location' -l repo -l packages -s P -r -f -a repo @@ -2351,6 +2364,23 @@ complete -c spack -n '__fish_spack_using_command mark' -s e -l explicit -d 'mark complete -c spack -n '__fish_spack_using_command mark' -s i -l implicit -f -a implicit complete -c spack -n '__fish_spack_using_command mark' -s i -l implicit -d 'mark packages as implicitly installed' +# spack migrate +set -g __fish_spack_optspecs_spack_migrate h/help dry-run clear clear-only replace restore i-need-old-spack +complete -c spack -n '__fish_spack_using_command migrate' -s h -l help -f -a help +complete -c spack -n '__fish_spack_using_command migrate' -s h -l help -d 'show this help message and exit' +complete -c spack -n '__fish_spack_using_command migrate' -l dry-run -f -a dry_run +complete -c spack -n '__fish_spack_using_command migrate' -l dry-run -d 'show what would be migrated without actually moving files' +complete -c spack -n '__fish_spack_using_command migrate' -l clear -f -a clear +complete -c spack -n '__fish_spack_using_command migrate' -l clear -d 'move entire ~/.spack directory to backup location:use this if no other instances need this old location' +complete -c spack -n '__fish_spack_using_command migrate' -l clear-only -f -a clear_only +complete -c spack -n '__fish_spack_using_command migrate' -l clear-only -d 'only move ~/.spack to backup without migrating files (useful after running migrate without --clear)' +complete -c spack -n '__fish_spack_using_command migrate' -l replace -f -a replace +complete -c spack -n '__fish_spack_using_command migrate' -l replace -d 'replace existing files in new locations (use with --clear if you forgot to use --clear on first run)' +complete -c spack -n '__fish_spack_using_command migrate' -l restore -f -a restore +complete -c spack -n '__fish_spack_using_command migrate' -l restore -d 'restore ~/.spack from backup location (after --clear)' +complete -c spack -n '__fish_spack_using_command migrate' -l i-need-old-spack -f -a i_need_old_spack +complete -c spack -n '__fish_spack_using_command migrate' -l i-need-old-spack -d 'print help about mixing pre-1.2-Spack and Spack >= 1.2' + # spack mirror set -g __fish_spack_optspecs_spack_mirror h/help n/no-checksum complete -c spack -n '__fish_spack_using_command_pos 0 mirror' -f -a create -d 'create a directory to be used as a spack mirror, and fill it with package archives' @@ -2417,7 +2447,7 @@ set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsig complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source' complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use ``--type binary --type source`` (default)' @@ -2453,7 +2483,7 @@ set -g __fish_spack_optspecs_spack_mirror_remove h/help scope= all-scopes complete -c spack -n '__fish_spack_using_command_pos 0 mirror remove' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror remove' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror remove' -l all-scopes -f -a all_scopes complete -c spack -n '__fish_spack_using_command mirror remove' -l all-scopes -d 'remove from all config scopes (default: highest scope with matching mirror)' @@ -2463,7 +2493,7 @@ set -g __fish_spack_optspecs_spack_mirror_rm h/help scope= all-scopes complete -c spack -n '__fish_spack_using_command_pos 0 mirror rm' -f -a '(__fish_spack_mirrors)' complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror rm' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror rm' -l all-scopes -f -a all_scopes complete -c spack -n '__fish_spack_using_command mirror rm' -l all-scopes -d 'remove from all config scopes (default: highest scope with matching mirror)' @@ -2477,7 +2507,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -f -a p complete -c spack -n '__fish_spack_using_command mirror set-url' -l push -d 'set only the URL used for uploading' complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -f -a fetch complete -c spack -n '__fish_spack_using_command mirror set-url' -l fetch -d 'set only the URL used for downloading' -complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror set-url' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set-url' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2519,7 +2549,7 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a s complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache' complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed complete -c spack -n '__fish_spack_using_command mirror set' -l signed -d 'require signing and signature verification when pushing and installing from this build cache' -complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror set' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -f -a s3_access_key_id complete -c spack -n '__fish_spack_using_command mirror set' -l s3-access-key-id -r -d 'ID string to use to connect to this S3 mirror' @@ -2544,14 +2574,14 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l oci-password-var set -g __fish_spack_optspecs_spack_mirror_list h/help scope= complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror list' -l scope -r -d 'configuration scope to read from' # spack mirror ls set -g __fish_spack_optspecs_spack_mirror_ls h/help scope= complete -c spack -n '__fish_spack_using_command mirror ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command mirror ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command mirror ls' -l scope -r -d 'configuration scope to read from' # spack module @@ -2865,7 +2895,7 @@ complete -c spack -n '__fish_spack_using_command repo create' -s d -l subdirecto set -g __fish_spack_optspecs_spack_repo_list h/help scope= names namespaces json complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo list' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo list' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command repo list' -l names -f -a names complete -c spack -n '__fish_spack_using_command repo list' -l names -d 'show configuration names only' @@ -2878,7 +2908,7 @@ complete -c spack -n '__fish_spack_using_command repo list' -l json -d 'output r set -g __fish_spack_optspecs_spack_repo_ls h/help scope= names namespaces json complete -c spack -n '__fish_spack_using_command repo ls' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo ls' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo ls' -l scope -r -d 'configuration scope to read from' complete -c spack -n '__fish_spack_using_command repo ls' -l names -f -a names complete -c spack -n '__fish_spack_using_command repo ls' -l names -d 'show configuration names only' @@ -2896,7 +2926,7 @@ complete -c spack -n '__fish_spack_using_command repo add' -l name -r -f -a name complete -c spack -n '__fish_spack_using_command repo add' -l name -r -d 'config name for the package repository, defaults to the namespace of the repository' complete -c spack -n '__fish_spack_using_command repo add' -l path -r -f -a path complete -c spack -n '__fish_spack_using_command repo add' -l path -r -d 'relative path to the Spack package repository inside a git repository. Can be repeated to add multiple package repositories in case of a monorepo' -complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo add' -l scope -r -d 'configuration scope to modify' # spack repo set @@ -2908,7 +2938,7 @@ complete -c spack -n '__fish_spack_using_command repo set' -l destination -r -f complete -c spack -n '__fish_spack_using_command repo set' -l destination -r -d 'destination to clone git repository into' complete -c spack -n '__fish_spack_using_command repo set' -l path -r -f -a path complete -c spack -n '__fish_spack_using_command repo set' -l path -r -d 'relative path to the Spack package repository inside a git repository. Can be repeated to add multiple package repositories in case of a monorepo' -complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo set' -l scope -r -d 'configuration scope to modify' # spack repo remove @@ -2916,7 +2946,7 @@ set -g __fish_spack_optspecs_spack_repo_remove h/help scope= all-scopes complete -c spack -n '__fish_spack_using_command_pos 0 repo remove' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo remove' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo remove' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command repo remove' -l all-scopes -f -a all_scopes complete -c spack -n '__fish_spack_using_command repo remove' -l all-scopes -d 'remove from all config scopes (default: highest scope with matching repo)' @@ -2926,7 +2956,7 @@ set -g __fish_spack_optspecs_spack_repo_rm h/help scope= all-scopes complete -c spack -n '__fish_spack_using_command_pos 0 repo rm' $__fish_spack_force_files -a '(__fish_spack_repos)' complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show this help message and exit' -complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command repo rm' -l all-scopes -f -a all_scopes complete -c spack -n '__fish_spack_using_command repo rm' -l all-scopes -d 'remove from all config scopes (default: highest scope with matching repo)' @@ -2948,7 +2978,7 @@ complete -c spack -n '__fish_spack_using_command repo update' -s h -l help -f -a complete -c spack -n '__fish_spack_using_command repo update' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command repo update' -l remote -s r -r -f -a remote complete -c spack -n '__fish_spack_using_command repo update' -l remote -s r -r -d 'name of remote to check for branches, tags, or commits' -complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -f -a '_builtin defaults:base defaults system site user spack command_line' +complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -f -a '_builtin defaults:base defaults:old defaults system site user spack command_line' complete -c spack -n '__fish_spack_using_command repo update' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command repo update' -l branch -s b -r -f -a branch complete -c spack -n '__fish_spack_using_command repo update' -l branch -s b -r -d 'name of a branch to change to' diff --git a/shared-spack.txt b/shared-spack.txt new file mode 100644 index 00000000000000..8b8d7e7a41be24 --- /dev/null +++ b/shared-spack.txt @@ -0,0 +1,146 @@ +Shared spack +============ + +This document captures the design decisions behind moving Spack's mutable +state out of $spack so the prefix can be treated as immutable (apt/pip +installable, multi-user readable). The implementation is described in +lib/spack/docs/where_spack_writes_data.rst. + +# What's moving out of $spack + +For new clones, the following are written under the user's home directory +in XDG-compliant locations instead of inside the Spack prefix: + +1. installs -> $data_home/installs (default ~/.local/share/spack/installs) +2. environments -> $data_home/environments +3. cached downloads -> $data_home/downloads +4. gpg keys -> $data_home/gpg, gpg-keys +5. modules -> $data_home +6. licenses -> $data_home/licenses + +Existing clones with data already in $spack continue to use those legacy +locations. The decision is made once, at startup, by the layout *scheme*. + +# Layout schemes + +`etc/spack/defaults/include.yaml` includes one of two scheme yamls based +on `layout_detected("old")` (a helper exposed to include `when:` clauses +by spack.spec.eval_conditional): + +- `etc/spack/defaults/old/config.yaml` — selected when legacy data exists + in $spack/opt, $spack/var/spack/environments, $spack/var/cache, etc. + Sets install_tree:root, environments_root, license_dir, source_cache, + gpg_path, gpg_keys_path to their $spack-local paths, plus + config:locations:* for general substitutions. + +- `etc/spack/defaults/xdg/config.yaml` — selected otherwise. Just sets + config:locations:{home,data,state,cache} using $xdg_*_home + substitutions; base/config.yaml's $data_home-relative defaults then + resolve to the right XDG paths. + +The schemes are pulled in EARLIER in defaults/include.yaml than +defaults/base/config.yaml, giving them higher priority — they override +base for paths that diverge between layouts. + +# Unilateral override + +`layout_detected("old")` returns False (forces xdg) if any of the +following env vars is set: SPACK_DATA_HOME, SPACK_STATE_HOME, +SPACK_CACHE_HOME, SPACK_HOME. Reason: setting one of these is an +intentional opt-in to the new layout; defaulting back to old anyway +would produce a split state where data goes new but envs/installs stay +in $spack. + +config:locations:* set in user/site/system scopes does NOT force the +scheme (those scopes aren't loaded at include-evaluation time). They +still override individual values once loaded. + +# Override priority + +In order from highest to lowest: + +1. SPACK_DATA_HOME / SPACK_STATE_HOME / SPACK_CACHE_HOME / SPACK_HOME + env vars (env-var, also forces scheme). +2. Specific config keys: config:install_tree:root, + config:environments_root, config:license_dir, config:source_cache, + config:gpg_path, config:gpg_keys_path — set in any scope. +3. config:locations:{home,data,state,cache} — redirects everything that + uses $data_home etc. substitutions. +4. The active scheme yaml's defaults. + +User/system config paths bootstrap the config system and are not +themselves in config. They are controlled by SPACK_USER_CONFIG_PATH, +SPACK_SYSTEM_CONFIG_PATH, and SPACK_DISABLE_LOCAL_CONFIG, exactly as +before. + +# Path substitutions + +Available in any string-valued config field: + +- $data_home / $state_home / $cache_home / $spack_home — read + config:locations:* with the env-var overrides applied +- $xdg_data_home / $xdg_state_home / $xdg_cache_home — $XDG_*_HOME or + the freedesktop default (with no /spack suffix). Used in the xdg + scheme yaml. +- $user_cache_path — legacy alias for $state_home +- $spack, $spack_instance_id — instance-specific; unchanged + +# Diagnostic: spack debug paths + +Prints active scheme, each resolved home with its source (env var? +config? which scope?), and every config-driven path with raw template +and resolved value. Replaces the per-command warnings the previous +attempt emitted; users hit this once when something looks wrong, not on +every invocation. + +# Migration: spack migrate + +`spack migrate` copies config from ~/.spack to ~/.config/spack and moves +package repos from ~/.spack/package_repos to +~/.local/state/spack/package_repos. + +`spack migrate --clear` additionally moves the entire ~/.spack into a +backup at ~/.local/share/spack/dotspack_backup. The backup location is +pinned to the XDG default and does NOT follow SPACK_DATA_HOME / +config:locations:data — the backup is a one-time migration artifact, +not legacy state to preserve, and shouldn't move around if the user +later redirects $data_home. + +`spack migrate --restore` reverses --clear from that fixed location. + +`spack migrate --i-need-old-spack` prints guidance for users who have +older Spack instances that can't upgrade and need to keep using +~/.spack. + +# Avoiding ~ + +To prevent any writes into $HOME from a new clone: + +- SPACK_HOME=x SPACK_USER_CONFIG_PATH=x/config (everything to x) +- or SPACK_HOME=x SPACK_DISABLE_LOCAL_CONFIG=1 +- or `config:locations:home: x` in an env's `spack.yaml` (with + `include: []` to avoid pulling in user config) + +The pre-1.2 isolation incantation still works for old instances that +pull this code: + +- SPACK_USER_CACHE_PATH=x SPACK_DISABLE_LOCAL_CONFIG=1 + +# Crosstalk between Spack instances + +* The misc. cache key (config:misc_cache) includes $spack_instance_id + (a hash of the Spack root), so co-installed Spack instances have + separate misc caches by default. Override config:misc_cache to share + one. +* Multiple Spack instances all writing to ~/.config/spack is still + possible — admins who want isolation between instances should set + config:locations:* in a site/system scope. + +# Admin management, pip-installed Spack + +- Admins can ship users an upstream + a curated set of config via + `include:` in the `spack` scope, using `when: SPACK_ADMIN not in env` + / `when: SPACK_ADMIN in env` to distinguish admin from user runs. +- A pip-installed Spack works as long as $spack (the install prefix) is + not written to. Read-only prefixes are now the default; users land in + the xdg scheme automatically. diff --git a/var/spack/gpg/README.md b/var/spack/gpg/README.md deleted file mode 100644 index 122d24f84126b4..00000000000000 --- a/var/spack/gpg/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# GPG Keys - -This directory contains keys that should be trusted by this installation of -Spack. They are imported when running `spack gpg init`, but may also be -imported manually with `spack gpg trust path/to/key`. diff --git a/var/spack/test_repos/spack_repo/builtin_mock/packages/url_list_test/package.py b/var/spack/test_repos/spack_repo/builtin_mock/packages/url_list_test/package.py index 5fa78405eafdbb..9196e4f13e0d9f 100644 --- a/var/spack/test_repos/spack_repo/builtin_mock/packages/url_list_test/package.py +++ b/var/spack/test_repos/spack_repo/builtin_mock/packages/url_list_test/package.py @@ -4,8 +4,8 @@ from spack_repo.builtin_mock.build_systems.generic import Package -import spack.paths from spack.package import * +from spack.paths import locations as spack_paths from spack.util.url import path_to_file_url @@ -14,9 +14,9 @@ class UrlListTest(Package): homepage = "http://www.url-list-example.com" - web_data_path = join_path(spack.paths.test_path, "data", "web") - url = path_to_file_url(join_path(spack.paths.test_path, "data", "web") + "/foo-0.0.0.tar.gz") - list_url = path_to_file_url(join_path(spack.paths.test_path, "data", "web") + "/index.html") + web_data_path = join_path(spack_paths.test_path, "data", "web") + url = path_to_file_url(join_path(spack_paths.test_path, "data", "web") + "/foo-0.0.0.tar.gz") + list_url = path_to_file_url(join_path(spack_paths.test_path, "data", "web") + "/index.html") list_depth = 3 version("0.0.0", md5="00000000000000000000000000000000")