sshd dying mysteriously on Woody (strace attached)
I am running Woody (with all security updates) on a Dell PowerEdge 2650.
After running fine since being setup several weeks ago, today sshd suddenly
died mid-connection.
I was able to run sshd manually from the console in debug mode, but attempts
to start sshd normally failed. This message appears in auth.log:
sshd[17803]: fatal: daemon() failed: Success
Attached is the output from trying to start sshd under strace.
Adam
execve("/usr/sbin/sshd", ["/usr/sbin/sshd"], [/* 19 vars */]) = 0
uname({sys="Linux", node="ds1.owentree.com", ...}) = 0
brk(0) = 0x8092680
open("/etc/ld.so.preload", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=7941, ...}) = 0
old_mmap(NULL, 7941, PROT_READ, MAP_PRIVATE, 3, 0) = 0x40014000
close(3) = 0
open("/lib/libwrap.so.0", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0@ \0\000"..., 1024)
= 1024
fstat64(3, {st_mode=S_IFREG|0644, st_size=24328, ...}) = 0
old_mmap(NULL, 29092, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x40016000
mprotect(0x4001c000, 4516, PROT_NONE) = 0
old_mmap(0x4001c000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3,
0x5000) = 0x4001c000
old_mmap(0x4001d000, 420, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED
MAP_ANONYMOUS, -1, 0) = 0x4001d000
close(3) = 0
open("/lib/libpam.so.0", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\24"..., 1024) =
1024
fstat64(3, {st_mode=S_IFREG|0644, st_size=29420, ...}) = 0
old_mmap(NULL, 32428, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x4001e000
mprotect(0x40025000, 3756, PROT_NONE) = 0
old_mmap(0x40025000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3,
0x6000) = 0x40025000
close(3) = 0
open("/lib/libdl.so.2", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0T\27\0\000"...,
1024) = 1024
fstat64(3, {st_mode=S_IFREG|0644, st_size=8008, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x40026000
old_mmap(NULL, 11004, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x40027000
mprotect(0x40029000, 2812, PROT_NONE) = 0
old_mmap(0x40029000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3,
0x1000) = 0x40029000
close(3) = 0
open("/lib/libutil.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200\16"..., 1024) =
1024
fstat64(3, {st_mode=S_IFREG|0644, st_size=7600, ...}) = 0
old_mmap(NULL, 10568, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x4002a000
mprotect(0x4002c000, 2376, PROT_NONE) = 0
old_mmap(0x4002c000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3,
0x1000) = 0x4002c000
close(3) = 0
open("/usr/lib/libz.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200\30"..., 1024) =
1024
fstat64(3, {st_mode=S_IFREG|0644, st_size=55432, ...}) = 0
old_mmap(NULL, 54460, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x4002d000
mprotect(0x40039000, 5308, PROT_NONE) = 0
old_mmap(0x40039000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3,
0xc000) = 0x40039000
close(3) = 0
open("/lib/libnsl.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 ;\0\000"..., 1024)
= 1024
fstat64(3, {st_mode=S_IFREG|0644, st_size=69472, ...}) = 0
old_mmap(NULL, 80988, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x4003b000
mprotect(0x4004c000, 11356, PROT_NONE) = 0
old_mmap(0x4004c000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3,
0x10000) = 0x4004c000
old_mmap(0x4004d000, 7260, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED
MAP_ANONYMOUS, -1, 0) = 0x4004d000
close(3) = 0
open("/usr/lib/libcrypto.so.0.9.6", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\240\36"..., 1024) =
1024
fstat64(3, {st_mode=S_IFREG|0644, st_size=771908, ...}) = 0
old_mmap(NULL, 783840, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x4004f000
mprotect(0x40101000, 54752, PROT_NONE) = 0
old_mmap(0x40101000, 45056, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3,
0xb2000) = 0x40101000
old_mmap(0x4010c000, 9696, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED
MAP_ANONYMOUS, -1, 0) = 0x4010c000
close(3) = 0
open("/lib/libc.so.6", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\30\222"..., 1024) =
1024
fstat64(3, {st_mode=S_IFREG|0755, st_size=1153784, ...}) = 0
old_mmap(NULL, 1166560, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x4010f000
mprotect(0x40222000, 40160, PROT_NONE) = 0
old_mmap(0x40222000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3,
0x113000) = 0x40222000
old_mmap(0x40228000, 15584, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED
MAP_ANONYMOUS, -1, 0) = 0x40228000
close(3) = 0
open("/lib/libcrypt.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\t\0"..., 1024)
= 1024
fstat64(3, {st_mode=S_IFREG|0644, st_size=19136, ...}) = 0
old_mmap(NULL, 182044, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x4022c000
mprotect(0x40231000, 161564, PROT_NONE) = 0
old_mmap(0x40231000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3,
0x4000) = 0x40231000
old_mmap(0x40232000, 157468, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED
MAP_ANONYMOUS, -1, 0) = 0x40232000
close(3) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x40259000
munmap(0x40014000, 7941) = 0
brk(0) = 0x8092680
brk(0x80926f8) = 0x80926f8
brk(0x8093000) = 0x8093000
brk(0x8094000) = 0x8094000
getpid() = 17776
getpid() = 17776
open("/dev/urandom", O_RDONLY) = 3
read(3, "N\313\304\376\304/\262=\314rnd\263\326\5\245$\345\216J"..., 20) =
20
close(3) = 0
getpid() = 17776
getpid() = 17776
getuid32() = 0
getpid() = 17776
time(NULL) = 1083267492
getpid() = 17776
open("/etc/ssh/sshd_config", O_RDONLY|O_LARGEFILE) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=2044, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x40014000
read(3, "# Package generated configuratio"..., 4096) = 2044
getuid32() = 0
getuid32() = 0
read(3, "", 4096) = 0
close(3) = 0
munmap(0x40014000, 4096) = 0
gettimeofday({1083267492, 386911}, NULL) = 0
getpid() = 17776
open("/etc/resolv.conf", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=66, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x40014000
read(3, "domain owentree.com\n#nameserver "..., 4096) = 66
read(3, "", 4096) = 0
close(3) = 0
munmap(0x40014000, 4096) = 0
open("/etc/ssh/ssh_host_rsa_key", O_RDONLY|O_LARGEFILE) = 3
fstat64(3, {st_mode=S_IFREG|0600, st_size=883, ...}) = 0
getuid32() = 0
_llseek(3, 0, [883], SEEK_END) = 0
_llseek(3, 0, [0], SEEK_SET) = 0
brk(0x8096000) = 0x8096000
read(3, "-----BEGIN RSA PRIVATE KEY-----\n"..., 883) = 883
_llseek(3, 0, [0], SEEK_SET) = 0
fcntl64(3, F_GETFL) = 0x8000 (flags O_RDONLY
O_LARGEFILE)
fstat64(3, {st_mode=S_IFREG|0600, st_size=883, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x40014000
_llseek(3, 0, [0], SEEK_CUR) = 0
read(3, "-----BEGIN RSA PRIVATE KEY-----\n"..., 4096) = 883
close(3) = 0
munmap(0x40014000, 4096) = 0
open("/etc/ssh/ssh_host_dsa_key", O_RDONLY|O_LARGEFILE) = 3
fstat64(3, {st_mode=S_IFREG|0600, st_size=664, ...}) = 0
getuid32() = 0
_llseek(3, 0, [664], SEEK_END) = 0
_llseek(3, 0, [0], SEEK_SET) = 0
read(3, "-----BEGIN DSA PRIVATE KEY-----\n"..., 664) = 664
_llseek(3, 0, [0], SEEK_SET) = 0
fcntl64(3, F_GETFL) = 0x8000 (flags O_RDONLY
O_LARGEFILE)
fstat64(3, {st_mode=S_IFREG|0600, st_size=664, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x40014000
_llseek(3, 0, [0], SEEK_CUR) = 0
read(3, "-----BEGIN DSA PRIVATE KEY-----\n"..., 4096) = 664
close(3) = 0
munmap(0x40014000, 4096) = 0
socket(PF_UNIX, SOCK_STREAM, 0) = 3
connect(3, {sin_family=AF_UNIX, path="/var/run/.nscd_socket"}, 110) = -1
ENOENT (No such file or directory)
close(3) = 0
open("/etc/nsswitch.conf", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=465, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x40014000
read(3, "# /etc/nsswitch.conf\n#\n# Example"..., 4096) = 465
read(3, "", 4096) = 0
close(3) = 0
munmap(0x40014000, 4096) = 0
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=7941, ...}) = 0
old_mmap(NULL, 7941, PROT_READ, MAP_PRIVATE, 3, 0) = 0x40014000
close(3) = 0
open("/lib/libnss_compat.so.2", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\25"..., 1024) =
1024
fstat64(3, {st_mode=S_IFREG|0644, st_size=40152, ...}) = 0
old_mmap(NULL, 43256, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x4025a000
mprotect(0x40264000, 2296, PROT_NONE) = 0
old_mmap(0x40264000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3,
0x9000) = 0x40264000
close(3) = 0
munmap(0x40014000, 7941) = 0
uname({sys="Linux", node="ds1.owentree.com", ...}) = 0
open("/etc/passwd", O_RDONLY) = 3
fcntl64(3, F_GETFD) = 0
fcntl64(3, F_SETFD, FD_CLOEXEC) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=1377, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x40014000
_llseek(3, 0, [0], SEEK_CUR) = 0
read(3, "root:x:0:0:root:/root:/bin/bash\n"..., 4096) = 1377
close(3) = 0
munmap(0x40014000, 4096) = 0
stat64("/var/run/sshd", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
setgroups32(0, 0) = 0
fork() = 17777
_exit(0) = ?
Reply to: