Bug#561611: gcc-4.3: generated 32 bit x86 code crashes with FPE when using union with double member and optimization

[Csaba Halász <csaba.halasz@gmail.com>]

Package: gcc-4.3
Version: 4.3.2-1.1
Severity: normal


Test program:

/* compile with gcc -m32 -O2 -lm */
extern int feenableexcept(int);
typedef union { double num; int tag[2]; } naRef;

int naStr_len(naRef r)  __attribute__((noinline));
int naStr_len(naRef r) { return 0; }
volatile int use_num;

static int equal(naRef a, naRef b)
{
    if (use_num) return a.num == b.num;
    if (naStr_len(a) != naStr_len(b)) return 0;
    return 1;
}

int findcell(naRef *hr, naRef key, int len)
{
    int i;
    for (i = 0; i < len; i++)
        if (equal(key, hr[i]))
            return i;
    return 0;
}

int main(int argc, char* argv[])
{
    naRef hr;
    naRef key;
    feenableexcept(13);
    key.tag[0] = 0;
    key.tag[1] = 0x7ff56789;
    hr = key;
    use_num = 0;
    return findcell(&hr, key, use_num + 1);
}
/* end of test program */

Compiling the above program either with native 32 bit gcc or the 32
bit mode of the 64 bit compiler with -O2 optimization generates code
that will crash with a floating point exception.
Studying the assembly code, apparently gcc uses fld/fstp to copy the
"key" argument in the "findcell" function. It does that before
consulting the use_num variable. The value set in the "tag" member
unfortunately corresponds to a NaN in the "num" member, hence the
exception.

findcell:
        pushl   %ebp
        movl    %esp, %ebp
        pushl   %edi
        pushl   %esi
        pushl   %ebx
        subl    $28, %esp
        movl    20(%ebp), %edi
        testl   %edi, %edi
        jle     .L4
        fldl    12(%ebp)
        xorl    %esi, %esi
        fstpl   -32(%ebp)        # <---- FAULT

Version 4.2.4 not affected, 4.4 series and trunk reported by others as affected.


-- System Information:
Debian Release: 5.0.3
  APT prefers stable
  APT policy: (500, 'stable')
Architecture: amd64 (x86_64)

Kernel: Linux 2.6.28.9 (SMP w/2 CPU cores)
Locale: LANG=C, LC_CTYPE=C (charmap=ANSI_X3.4-1968)
Shell: /bin/sh linked to /bin/bash

Versions of packages gcc-4.3 depends on:
ii  binutils            2.18.1~cvs20080103-7 The GNU assembler, linker and bina
ii  cpp-4.3             4.3.2-1.1            The GNU C preprocessor
ii  gcc-4.3-base        4.3.2-1.1            The GNU Compiler Collection (base
ii  libc6               2.7-18               GNU C Library: Shared libraries
ii  libgcc1             1:4.3.2-1.1          GCC support library
ii  libgomp1            4.3.2-1.1            GCC OpenMP (GOMP) support library

Versions of packages gcc-4.3 recommends:
ii  libc6-dev                     2.7-18     GNU C Library: Development Librari

Versions of packages gcc-4.3 suggests:
pn  gcc-4.3-doc                   <none>     (no description available)
pn  gcc-4.3-locales               <none>     (no description available)
ii  gcc-4.3-multilib              4.3.2-1.1  The GNU C compiler (multilib files
pn  libgcc1-dbg                   <none>     (no description available)
pn  libgomp1-dbg                  <none>     (no description available)
pn  libmudflap0-4.3-dev           <none>     (no description available)
pn  libmudflap0-dbg               <none>     (no description available)

-- no debconf information