$ spack info mvapich2

# ~/.spack/linux/packages.yaml
# Paste the following so that the dependency is marked as external and is not installed by spack
# Edit the value of spec to be rdma-core/psm/opa-psm2 depending on the dependency of you chosen fabrics.

packages:
  rdma-core:
    buildable: False
    externals:
    - spec: rdma-core
      prefix: /usr

$ spack install mvapich2@2.3.7 %gcc@4.8.5

$ spack find -l -v -p mvapich2
suqign2 mvapich2@2.3.7~alloca~cuda~debug+regcache+wrapperrpath ch3_rank_bits=32 fabrics=mrail file_systems=auto process_managers=auto threads=multiple  /home/user/spack/opt/spack/linux-centos7-x86_64/gcc-4.8.5/mvapich2-2.3.7-suqign2xxq63oao52wtgu4kohkpmhte6

# Note the hash of the required version
$ spack load /suqign2
$ which mpirun_rsh 
$ ~ /spack/opt/spack/linux-centos7-x86_64/gcc-4.8.5/mvapich2-2.3.7-suqign2dc5uwyir7oaqqtwobrlwabz7k/bin/mpirun_rsh
              

$ git clone https://gitlab.com/hjelmn/xpmem.git

$ cd xpmem
$ ./autogen.sh
$ ./configure --prefix=<>
$ sudo make -j8 install

$ sudo insmod /opt/xpmem/lib/modules/4.14.123-111.109.amzn2.x86_64/xpmem.ko
$ sudo chmod 666 /dev/xpmem

$ lsmod | grep xpmem
xpmem                  32569  0

$ spack mirror add mvapich2x http://mvapich.cse.ohio-state.edu/download/mvapich/spack-mirror/mvapich2x
$ spack mirror add mvapich2-gdr http://mvapich.cse.ohio-state.edu/download/mvapich/spack-mirror/mvapich2-gdr

$ wget http://mvapich.cse.ohio-state.edu/download/mvapich/spack-mirror/mvapich2x/build_cache/public.key
$ spack gpg trust public.key

$ spack buildcache list -L -v -a
==> 80 cached builds.
-- linux-centos7-x86_64 / gcc@4.8.5 -----------------------------
ru6d56jm7q4jsqjpt2rjilj47pcqbmjt mvapich2x@2.3 distribution=mofed4.5 feature=advanced pmi_version=pmi1 process_managers=mpirun
.
.
.


# Note: ru6d56jm7q4jsqjpt2rjilj47pcqbmjt is the hash of this binary 

 $ spack compiler list
# if the compiler version needed is not listed, use "spack find" to add it

# this command will search the PATH variable for compilers
$ spack compiler find

# it might be easier to explictly specify the path the gcc executable
$ spack compiler find /path/to/gcc

1. First install the dependencies of your chosen MVAPICH2-GDR variants. Don't forget to append target=(x86_64/ppc64le).
   Add the correct cuda dependency or configure your packages.yaml with external packages to use system installed cuda. (Refer to spack docs)
$ spack install --only dependencies mvapich2-gdr@2.3.7%gcc@4.8.5 distribution=mofed4.7 pmi_version=pmix process_managers=slurm target=x86_64 ^cuda@11.1.0%gcc@4.8.5 target=x86_64

2. Now get the hash of the binary that you want to install. Refer further below for the binary hash.Use this hash to install the correct binary. (You might run into some errors which will be addressed below)
login4.frontera(1009)$ spack buildcache install -o /44aoflbqebw3xbgp4rbncvrjwbjcwhs4     # (Use -u switch if facing problems with the gpg signatures)

3. If the installation proceeds smoothly, you can skip the next steps.

4. If you run into dependency errors, it is likely that certain dependencies were updated by their maintainers after the binaries were built.
   In these scenarios, we must install the missing dependencies by analyzing the error and warning messages for the full spec of the missing dependencies.

[senthilkumar.16@head spack]$ spack buildcache install -o /44aoflbqebw3xbgp4rbncvrjwbjcwhs4
==> buildcache spec(s) matching /44aoflbqebw3xbgp4rbncvrjwbjcwhs4 

==> Installing buildcache for dependency spec cuda@11.1.0%gcc@4.8.5 arch=linux-centos7-x86_64 ^libiconv@1.16%gcc@4.8.5 arch=linux-centos7-x86_64 ^libxml2@2.9.10%gcc@4.8.5~python arch=linux-centos7-x86_64 ^xz@5.2.5%gcc@4.8.5~pic arch=linux-centos7-x86_64 ^zlib@1.2.11%gcc@4.8.5+optimize+pic+shared arch=linux-centos7-x86_64
==> Installing buildcache for dependency spec libxml2@2.9.10%gcc@4.8.5~python arch=linux-centos7-x86_64 ^libiconv@1.16%gcc@4.8.5 arch=linux-centos7-x86_64 ^xz@5.2.5%gcc@4.8.5~pic arch=linux-centos7-x86_64 ^zlib@1.2.11%gcc@4.8.5+optimize+pic+shared arch=linux-centos7-x86_64
==> Installing buildcache for dependency spec libiconv@1.16%gcc@4.8.5 arch=linux-centos7-x86_64
==> Warning: download_tarball() was unable to download libiconv@1.16%gcc@4.8.5 arch=linux-centos7-x86_64 from any configured mirrors
==> Error: Download of binary cache file for spec libiconv@1.16%gcc@4.8.5 arch=linux-centos7-x86_64 failed.

5. From line above we can see the missing spec. Install that by using the $ spack install command. You may want to add --fresh to the command if installing from build cache complains about a dependency that you just installed.
[senthilkumar.16@head spack]$ spack install libiconv@1.16%gcc@4.8.5
==> Installing libiconv-1.16-qazxaa4lnd2ukd44fjzdhayyxvcg3wyu
==> No binary for libiconv-1.16-qazxaa4lnd2ukd44fjzdhayyxvcg3wyu found: installing from source
==> Using cached archive: /home/contini.26/spack-related/spack/var/spack/cache/_source-cache/archive/e6/e6a1b1b589654277ee790cce3734f07876ac4ccfaecbee8afa0b649cf529cc04.tar.gz
==> libiconv: Executing phase: 'autoreconf'
==> libiconv: Executing phase: 'configure'
==> libiconv: Executing phase: 'build'
==> libiconv: Executing phase: 'install'
[+] /home/contini.26/spack-related/spack/opt/spack/linux-centos7-haswell/gcc-4.8.5/libiconv-1.16-qazxaa4lnd2ukd44fjzdhayyxvcg3wyu

6. Follow this by the buildcache install command and keep repeating the steps untill all dependencies are installed
[senthilkumar.16@head spack]$ spack buildcache install -o /44aoflbqebw3xbgp4rbncvrjwbjcwhs4

$ spack find -l -v -p mvapich2-gdr
-- linux-centos7-x86_64 / gcc@4.8.5 -----------------------------
44aoflb mvapich2-gdr@2.3.7+cuda+mcast~openacc~rocm distribution=mofed4.7 pmi_version=pmix process_managers=slurm  /home/user/spack/opt/spack/linux-centos7-x86_64/gcc-4.8.5/mvapich2-gdr-2.3.7-44aoflbqebw3xbgp4rbncvrjwbjcwhs4

# Note the hash of the required version - It's the first word of the previous command's output
$ spack load /44aoflb
$ which mpirun_rsh