Bug#708750: libc-bin: robust mutex deadlocks instead of returning EOWNERDEAD

[Andrey Semashev <andysem@mail.ru>]

Package: libc-bin
Version: 2.11.3-4
Severity: normal


This bug has been reported as: http://sourceware.org/bugzilla/show_bug.cgi?id=13002 and http://bugzilla.redhat.com/show_bug.cgi?id=628608
At least, Debian Squeeze and Wheezy are affected. Kubuntu 13.04 seem to be not affected by this.

If a parent process and child process share a robust mutex and the child exits
while holding the mutex lock, when the parent tries to acquire the lock it will
hang instead of being notified of the state by EOWNERDEAD.

Here's a testcase which exits successfully on Solaris but deadlocks with NPTL

#include <sys/types.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <pthread.h>
#include <unistd.h>
#include <stdlib.h>
#include <errno.h>
#include <stdio.h>
#include <fcntl.h>

void initialize(pthread_mutex_t* mtx)
{
    pthread_mutexattr_t mtxa;
    if(pthread_mutexattr_init(&mtxa))
        abort();
    if(pthread_mutexattr_setpshared(&mtxa, PTHREAD_PROCESS_SHARED))
        abort();
    if(pthread_mutexattr_setrobust_np(&mtxa, PTHREAD_MUTEX_ROBUST_NP))
        abort();
    if(pthread_mutex_init(mtx, &mtxa))
        abort();
    pthread_mutexattr_destroy(&mtxa);
}

void set_consistent(pthread_mutex_t* mtx)
{
    if(pthread_mutex_consistent_np(mtx))
        abort();
}

void lock(pthread_mutex_t* mtx)
{
    int err;
    if((err = pthread_mutex_lock(mtx))) {
        if(EOWNERDEAD == err) { // handle abandoned mutex
        if(pthread_mutex_consistent_np(mtx))
        abort();
    }
        else
            abort();
    }
}

void unlock(pthread_mutex_t* mtx)
{
    if(pthread_mutex_unlock(mtx))
        abort();
}

pid_t spawn(int(*fn)())
{
    // fork a child process
    pid_t pid = fork();
    switch(pid) {
    case 0:
        exit(fn());
    case -1:
        abort();
    default:
        return pid;
    }
}

char const shared_file[] = "shared_file";


void* open_shared_file()
{
    int fd = open(shared_file, O_CREAT | O_RDWR, (mode_t)0666);
    if(fd < 0)
    abort();
    struct stat st;
    if(fstat(fd, &st))
    abort();
    int new_file = !st.st_size;
    if (new_file)
        if(ftruncate(fd, sizeof(pthread_mutex_t)))
            abort();
    void* mem = mmap(NULL, sizeof(pthread_mutex_t), PROT_READ | PROT_WRITE,
MAP_SHARED, fd, 0);
    close(fd);
    if(MAP_FAILED == mem)
    abort();
    if (new_file)
    initialize((pthread_mutex_t*)mem);
    return mem;
}

int process_1()
{
    unsigned pid = getpid();
    printf("%u: process 1\n", pid);
    pthread_mutex_t* m = (pthread_mutex_t*)open_shared_file();
    printf("%u: locking mutex...\n", pid);
    lock(m);
    printf("%u: exiting\n", pid);

    return 0;
}

int process_2()
{
    unsigned pid = getpid();
    printf("%u: process 2\n", pid);
    pthread_mutex_t* m = (pthread_mutex_t*)open_shared_file();
    printf("%u: locking mutex...\n", pid);
    lock(m);
    printf("%u: mutex locked\n", pid);
    unlock(m);
    return 0;
}

int main(int ac, char** av)
{
    // fork process_1 and wait till it terminates
    pid_t child;
    unlink(shared_file);
    int child_status;
    child = spawn(process_1);
    if(-1 == waitpid(child, &child_status, 0))
    abort();

    // now do process_2
    return process_2();
}

-- System Information:
Debian Release: 6.0.7
  APT prefers oldstable-updates
  APT policy: (500, 'oldstable-updates'), (500, 'oldstable')
Architecture: amd64 (x86_64)

Kernel: Linux 2.6.39-bpo.2-amd64 (SMP w/24 CPU cores)
Locale: LANG=en_US.UTF-8, LC_CTYPE=en_US.UTF-8 (charmap=UTF-8)
Shell: /bin/sh linked to /bin/dash

-- no debconf information