[Date Prev][Date Next] [Thread Prev][Thread Next] [Date Index] [Thread Index]

Bug#954272: slurmd: SLURM not working with OpenMPI

On 20-07-2020 16:27, Alastair McKinstry wrote:


On 20/07/2020 14:52, Lars Veldscholte wrote:

Hi,

I believe I have found a solution.
I must confess that I still don't fully understand the difference between the various PMI APIs, and which ones are supported by OpenMPI, but I found that the recommended way is to use PMIx.
However, PMIx was not working on my system even though libpmix2 is installed: 

# srun --mpi pmix ./a.out
srun: error: (null) [0] /mpi_pmix.c:133 [init] mpi/pmix: ERROR: pmi/pmix: can not load PMIx library
srun: error: Couldn't load specified plugin name for mpi/pmix: Plugin init() callback failed 

srun: error: cannot create mpi context for mpi/pmix

srun: error: invalid MPI type 'pmix', --mpi=list for acceptable types
Running `strace srun --mpi=pmix ./a.out` revealed that SLURM is looking for the pmix library at `/usr/lib/x86_64-linux-gnu/pmix/lib/libpmix.so`, which does not exist, only `libpmix.so.2` exists.
Which code is trying to load libpmix.so ? The compiled code should be loading libpmix.so.2 directly; the libpmix.so should only be needed 
at build-time.
srun is, I suppose. I've attached the entire strace output in case it's of any help.
Perhaps the problem is that libpmix-dev is not installed at compile time. I can add it as a Dependency of libopenmpi-dev.
Installing the package `libpmix-dev` installs this library (it symlinks it to the same file `libpmix.so.2` is symlinked to). 

Now, `srun --mpi=pmix ./a.out` is working!
I'm not 100% sure, but I think that the package `libpmix2` should also install the file `libpmix.so`. The dev package shouldn't be required for that, right? 

Lars


Regards

Alastair



Regards,

Lars


# strace srun --mpi=pmix_v3 ./a.out
execve("/usr/bin/srun", ["srun", "--mpi=pmix_v3", "./a.out"], 0x7ffd4f380370 /* 17 vars */) = 0
brk(NULL)                               = 0x563aea4e4000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/tls/x86_64/x86_64/libz.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm/tls/x86_64/x86_64", 0x7ffe2154f320) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/tls/x86_64/libz.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm/tls/x86_64", 0x7ffe2154f320) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/tls/x86_64/libz.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm/tls/x86_64", 0x7ffe2154f320) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/tls/libz.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm/tls", 0x7ffe2154f320) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/x86_64/x86_64/libz.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm/x86_64/x86_64", 0x7ffe2154f320) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/x86_64/libz.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm/x86_64", 0x7ffe2154f320) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/x86_64/libz.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm/x86_64", 0x7ffe2154f320) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/libz.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm", {st_mode=S_IFDIR|0755, st_size=20480, ...}) = 0
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=34055, ...}) = 0
mmap(NULL, 34055, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f2eddeec000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libz.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0203\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=113088, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2eddeea000
mmap(NULL, 115088, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2eddecd000
mprotect(0x7f2edded0000, 98304, PROT_NONE) = 0
mmap(0x7f2edded0000, 69632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f2edded0000
mmap(0x7f2eddee1000, 24576, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x7f2eddee1000
mmap(0x7f2eddee8000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1a000) = 0x7f2eddee8000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/liblz4.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/liblz4.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\3402\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=133464, ...}) = 0
mmap(NULL, 135544, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2eddeab000
mmap(0x7f2eddeae000, 106496, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f2eddeae000
mmap(0x7f2eddec8000, 12288, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1d000) = 0x7f2eddec8000
mmap(0x7f2eddecb000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1f000) = 0x7f2eddecb000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/libslurmfull.so", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\33\4\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=2084504, ...}) = 0
mmap(NULL, 2109984, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2eddca7000
mprotect(0x7f2eddce3000, 1789952, PROT_NONE) = 0
mmap(0x7f2eddce3000, 1368064, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3c000) = 0x7f2eddce3000
mmap(0x7f2edde31000, 417792, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x18a000) = 0x7f2edde31000
mmap(0x7f2edde98000, 53248, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1f0000) = 0x7f2edde98000
mmap(0x7f2eddea5000, 21024, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f2eddea5000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/libpthread.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`|\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=146912, ...}) = 0
mmap(NULL, 132256, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2eddc86000
mmap(0x7f2eddc8d000, 61440, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7f2eddc8d000
mmap(0x7f2eddc9c000, 20480, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f2eddc9c000
mmap(0x7f2eddca1000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1a000) = 0x7f2eddca1000
mmap(0x7f2eddca3000, 13472, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f2eddca3000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 o\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1831600, ...}) = 0
mmap(NULL, 1844568, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2eddac3000
mmap(0x7f2eddae8000, 1351680, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x25000) = 0x7f2eddae8000
mmap(0x7f2eddc32000, 303104, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16f000) = 0x7f2eddc32000
mmap(0x7f2eddc7c000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b8000) = 0x7f2eddc7c000
mmap(0x7f2eddc82000, 13656, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f2eddc82000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000\21\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=14592, ...}) = 0
mmap(NULL, 16656, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2eddabe000
mmap(0x7f2eddabf000, 4096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x7f2eddabf000
mmap(0x7f2eddac0000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f2eddac0000
mmap(0x7f2eddac1000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f2eddac1000
close(3)                                = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2eddabc000
mmap(NULL, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2eddab9000
arch_prctl(ARCH_SET_FS, 0x7f2eddab9740) = 0
mprotect(0x7f2eddc7c000, 12288, PROT_READ) = 0
mprotect(0x7f2eddac1000, 4096, PROT_READ) = 0
mprotect(0x7f2eddca1000, 4096, PROT_READ) = 0
mprotect(0x7f2edde98000, 8192, PROT_READ) = 0
mprotect(0x7f2eddecb000, 4096, PROT_READ) = 0
mprotect(0x7f2eddee8000, 4096, PROT_READ) = 0
mprotect(0x563ae9ba4000, 4096, PROT_READ) = 0
mprotect(0x7f2eddf1d000, 4096, PROT_READ) = 0
munmap(0x7f2eddeec000, 34055)           = 0
set_tid_address(0x7f2eddab9a10)         = 3944334
set_robust_list(0x7f2eddab9a20, 24)     = 0
rt_sigaction(SIGRTMIN, {sa_handler=0x7f2eddc8d6b0, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7f2eddc9a110}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0x7f2eddc8d750, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7f2eddc9a110}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
brk(NULL)                               = 0x563aea4e4000
brk(0x563aea505000)                     = 0x563aea505000
stat("/etc/slurm-llnl/slurm.conf", {st_mode=S_IFREG|0644, st_size=1370, ...}) = 0
openat(AT_FDCWD, "/etc/slurm-llnl/slurm.conf", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=1370, ...}) = 0
read(3, "# slurm.conf file generated by c"..., 4096) = 1370
read(3, "", 4096)                       = 0
close(3)                                = 0
stat("/bin/mail", {st_mode=S_IFREG|0755, st_size=218160, ...}) = 0
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm", {st_mode=S_IFDIR|0755, st_size=20480, ...}) = 0
socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
connect(3, {sa_family=AF_UNIX, sun_path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(3)                                = 0
socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
connect(3, {sa_family=AF_UNIX, sun_path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(3)                                = 0
openat(AT_FDCWD, "/etc/nsswitch.conf", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=494, ...}) = 0
read(3, "# /etc/nsswitch.conf\n#\n# Example"..., 4096) = 494
read(3, "", 4096)                       = 0
close(3)                                = 0
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=34055, ...}) = 0
mmap(NULL, 34055, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f2eddeec000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libnss_files.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0003\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=51696, ...}) = 0
mmap(NULL, 79672, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2eddaa5000
mmap(0x7f2eddaa8000, 28672, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f2eddaa8000
mmap(0x7f2eddaaf000, 8192, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa000) = 0x7f2eddaaf000
mmap(0x7f2eddab1000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xb000) = 0x7f2eddab1000
mmap(0x7f2eddab3000, 22328, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f2eddab3000
close(3)                                = 0
mprotect(0x7f2eddab1000, 4096, PROT_READ) = 0
munmap(0x7f2eddeec000, 34055)           = 0
openat(AT_FDCWD, "/etc/passwd", O_RDONLY|O_CLOEXEC) = 3
lseek(3, 0, SEEK_CUR)                   = 0
fstat(3, {st_mode=S_IFREG|0644, st_size=1947, ...}) = 0
read(3, "root:x:0:0:root:/root:/bin/bash\n"..., 4096) = 1947
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
fstat(3, {st_mode=S_IFDIR|0755, st_size=20480, ...}) = 0
brk(0x563aea526000)                     = 0x563aea526000
getdents64(3, /* 102 entries */, 32768) = 4320
getdents64(3, /* 0 entries */, 32768)   = 0
close(3)                                = 0
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm/select_cray_aries.so", {st_mode=S_IFREG|0644, st_size=60872, ...}) = 0
access("/usr/lib/x86_64-linux-gnu/slurm-wlm/select_cray_aries.so", R_OK) = 0
futex(0x7f2eddac20c8, FUTEX_WAKE_PRIVATE, 2147483647) = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/select_cray_aries.so", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220I\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=60872, ...}) = 0
mmap(NULL, 66120, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2edda94000
mprotect(0x7f2edda98000, 40960, PROT_NONE) = 0
mmap(0x7f2edda98000, 24576, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x7f2edda98000
mmap(0x7f2edda9e000, 12288, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa000) = 0x7f2edda9e000
mmap(0x7f2eddaa2000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd000) = 0x7f2eddaa2000
mmap(0x7f2eddaa4000, 584, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f2eddaa4000
close(3)                                = 0
mprotect(0x7f2eddaa2000, 4096, PROT_READ) = 0
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm/select_cons_res.so", {st_mode=S_IFREG|0644, st_size=118144, ...}) = 0
access("/usr/lib/x86_64-linux-gnu/slurm-wlm/select_cons_res.so", R_OK) = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/select_cons_res.so", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`I\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=118144, ...}) = 0
mmap(NULL, 122376, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2edda76000
mprotect(0x7f2edda7a000, 98304, PROT_NONE) = 0
mmap(0x7f2edda7a000, 77824, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x7f2edda7a000
mmap(0x7f2edda8d000, 16384, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7f2edda8d000
mmap(0x7f2edda92000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b000) = 0x7f2edda92000
close(3)                                = 0
mprotect(0x7f2edda92000, 4096, PROT_READ) = 0
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm/select_linear.so", {st_mode=S_IFREG|0644, st_size=64128, ...}) = 0
access("/usr/lib/x86_64-linux-gnu/slurm-wlm/select_linear.so", R_OK) = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/select_linear.so", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\3005\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=64128, ...}) = 0
mmap(NULL, 68376, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2edda65000
mmap(0x7f2edda68000, 40960, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f2edda68000
mmap(0x7f2edda72000, 8192, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd000) = 0x7f2edda72000
mmap(0x7f2edda74000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xe000) = 0x7f2edda74000
close(3)                                = 0
mprotect(0x7f2edda74000, 4096, PROT_READ) = 0
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm/select_cons_tres.so", {st_mode=S_IFREG|0644, st_size=151024, ...}) = 0
access("/usr/lib/x86_64-linux-gnu/slurm-wlm/select_cons_tres.so", R_OK) = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/select_cons_tres.so", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\\\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=151024, ...}) = 0
mmap(NULL, 155344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2edda3f000
mprotect(0x7f2edda44000, 126976, PROT_NONE) = 0
mmap(0x7f2edda44000, 102400, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5000) = 0x7f2edda44000
mmap(0x7f2edda5d000, 20480, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e000) = 0x7f2edda5d000
mmap(0x7f2edda63000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x23000) = 0x7f2edda63000
close(3)                                = 0
mprotect(0x7f2edda63000, 4096, PROT_READ) = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
fstat(3, {st_mode=S_IFDIR|0755, st_size=20480, ...}) = 0
getdents64(3, /* 102 entries */, 32768) = 4320
getdents64(3, /* 0 entries */, 32768)   = 0
close(3)                                = 0
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm/switch_none.so", {st_mode=S_IFREG|0644, st_size=18400, ...}) = 0
access("/usr/lib/x86_64-linux-gnu/slurm-wlm/switch_none.so", R_OK) = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/switch_none.so", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200 \0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=18400, ...}) = 0
mmap(NULL, 20552, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2eddeef000
mmap(0x7f2eddef1000, 4096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f2eddef1000
mmap(0x7f2eddef2000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f2eddef2000
mmap(0x7f2eddef3000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f2eddef3000
close(3)                                = 0
mprotect(0x7f2eddef3000, 4096, PROT_READ) = 0
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm/switch_generic.so", {st_mode=S_IFREG|0644, st_size=31000, ...}) = 0
access("/usr/lib/x86_64-linux-gnu/slurm-wlm/switch_generic.so", R_OK) = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/switch_generic.so", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\"\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=31000, ...}) = 0
mmap(NULL, 33264, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2edda36000
mmap(0x7f2edda38000, 12288, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f2edda38000
mmap(0x7f2edda3b000, 8192, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5000) = 0x7f2edda3b000
mmap(0x7f2edda3d000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f2edda3d000
close(3)                                = 0
mprotect(0x7f2edda3d000, 4096, PROT_READ) = 0
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm/switch_cray_aries.so", {st_mode=S_IFREG|0644, st_size=18552, ...}) = 0
access("/usr/lib/x86_64-linux-gnu/slurm-wlm/switch_cray_aries.so", R_OK) = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/switch_cray_aries.so", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260!\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=18552, ...}) = 0
mmap(NULL, 20712, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2edda30000
mmap(0x7f2edda32000, 4096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f2edda32000
mmap(0x7f2edda33000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f2edda33000
mmap(0x7f2edda34000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f2edda34000
close(3)                                = 0
mprotect(0x7f2edda34000, 4096, PROT_READ) = 0
rt_sigprocmask(SIG_BLOCK, [HUP INT QUIT USR1 USR2 PIPE ALRM TERM CONT], NULL, 8) = 0
rt_sigprocmask(SIG_BLOCK, [WINCH], NULL, 8) = 0
openat(AT_FDCWD, "/etc/slurm-llnl/plugstack.conf", O_RDONLY|O_CLOEXEC) = 3
fcntl(3, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(3, {st_mode=S_IFREG|0644, st_size=48, ...}) = 0
read(3, "include /etc/slurm-llnl/plugstac"..., 4096) = 48
openat(AT_FDCWD, "/etc/slurm-llnl/plugstack.conf.d", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 4
fstat(4, {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
getdents64(4, /* 2 entries */, 32768)   = 48
getdents64(4, /* 0 entries */, 32768)   = 0
close(4)                                = 0
read(3, "", 4096)                       = 0
close(3)                                = 0
getcwd("/usr/lib/x86_64-linux-gnu/pmix/lib", 4096) = 35
getgid()                                = 0
getuid()                                = 0
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm/launch_slurm.so", {st_mode=S_IFREG|0644, st_size=35520, ...}) = 0
access("/usr/lib/x86_64-linux-gnu/slurm-wlm/launch_slurm.so", R_OK) = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/launch_slurm.so", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\00006\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=35520, ...}) = 0
mmap(NULL, 38032, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2edda26000
mprotect(0x7f2edda29000, 20480, PROT_NONE) = 0
mmap(0x7f2edda29000, 12288, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f2edda29000
mmap(0x7f2edda2c000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f2edda2c000
mmap(0x7f2edda2e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7f2edda2e000
close(3)                                = 0
mprotect(0x7f2edda2e000, 4096, PROT_READ) = 0
mmap(NULL, 266240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f2edd9e5000
munmap(0x7f2edd9e5000, 266240)          = 0
stat("/usr/lib/x86_64-linux-gnu/slurm-wlm/mpi_pmix_v3.so", {st_mode=S_IFREG|0644, st_size=185160, ...}) = 0
access("/usr/lib/x86_64-linux-gnu/slurm-wlm/mpi_pmix_v3.so", R_OK) = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/slurm-wlm/mpi_pmix_v3.so", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260q\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=185160, ...}) = 0
mmap(NULL, 188752, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2edd9f7000
mmap(0x7f2edd9fd000, 110592, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f2edd9fd000
mmap(0x7f2edda18000, 45056, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21000) = 0x7f2edda18000
mmap(0x7f2edda23000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2b000) = 0x7f2edda23000
mmap(0x7f2edda25000, 336, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f2edda25000
close(3)                                = 0
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=34055, ...}) = 0
mmap(NULL, 34055, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f2edd9ee000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libhwloc.so.15", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0l\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=331696, ...}) = 0
mmap(NULL, 334008, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2edd99c000
mprotect(0x7f2edd9a2000, 303104, PROT_NONE) = 0
mmap(0x7f2edd9a2000, 229376, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f2edd9a2000
mmap(0x7f2edd9da000, 69632, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3e000) = 0x7f2edd9da000
mmap(0x7f2edd9ec000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4f000) = 0x7f2edd9ec000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\361\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=1325424, ...}) = 0
mmap(NULL, 1327360, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2edd857000
mmap(0x7f2edd866000, 634880, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xf000) = 0x7f2edd866000
mmap(0x7f2edd901000, 626688, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xaa000) = 0x7f2edd901000
mmap(0x7f2edd99a000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x142000) = 0x7f2edd99a000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libudev.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@X\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=174272, ...}) = 0
mmap(NULL, 178440, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f2edd82b000
mmap(0x7f2edd830000, 110592, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5000) = 0x7f2edd830000
mmap(0x7f2edd84b000, 40960, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x20000) = 0x7f2edd84b000
mmap(0x7f2edd855000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x29000) = 0x7f2edd855000
close(3)                                = 0
mprotect(0x7f2edd855000, 4096, PROT_READ) = 0
mprotect(0x7f2edd99a000, 4096, PROT_READ) = 0
mprotect(0x7f2edd9ec000, 4096, PROT_READ) = 0
mprotect(0x7f2edda23000, 4096, PROT_READ) = 0
munmap(0x7f2edd9ee000, 34055)           = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/pmix/lib/libpmix.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
poll([{fd=2, events=POLLOUT}], 1, 5000) = 1 ([{fd=2, revents=POLLOUT}])
fstat(2, {st_mode=S_IFCHR|0620, st_rdev=makedev(0x88, 0x3), ...}) = 0
write(2, "srun: error: (null) [0] /mpi_pmi"..., 100srun: error: (null) [0] /mpi_pmix.c:133 [init] mpi/pmix: ERROR: pmi/pmix: can not load PMIx library
) = 100
munmap(0x7f2edd9f7000, 188752)          = 0
munmap(0x7f2edd99c000, 334008)          = 0
munmap(0x7f2edd857000, 1327360)         = 0
munmap(0x7f2edd82b000, 178440)          = 0
poll([{fd=2, events=POLLOUT}], 1, 5000) = 1 ([{fd=2, revents=POLLOUT}])
fstat(2, {st_mode=S_IFCHR|0620, st_rdev=makedev(0x88, 0x3), ...}) = 0
write(2, "srun: error: Couldn't load speci"..., 96srun: error: Couldn't load specified plugin name for mpi/pmix_v3: Plugin init() callback failed
) = 96
poll([{fd=2, events=POLLOUT}], 1, 5000) = 1 ([{fd=2, revents=POLLOUT}])
fstat(2, {st_mode=S_IFCHR|0620, st_rdev=makedev(0x88, 0x3), ...}) = 0
write(2, "srun: error: cannot create mpi c"..., 55srun: error: cannot create mpi context for mpi/pmix_v3
) = 55
poll([{fd=2, events=POLLOUT}], 1, 5000) = 1 ([{fd=2, revents=POLLOUT}])
fstat(2, {st_mode=S_IFCHR|0620, st_rdev=makedev(0x88, 0x3), ...}) = 0
write(2, "srun: error: invalid MPI type 'p"..., 73srun: error: invalid MPI type 'pmix_v3', --mpi=list for acceptable types
) = 73
exit_group(1)                           = ?
+++ exited with 1 +++

Attachment: signature.asc
Description: OpenPGP digital signature

Reply to: