Bug#646549: libc6: "Invalid instruction" in _dl_x86_64_save_sse on processor that supports AVX

[Anton Tolchanov <debian@ryzh.kiev.ua>]

Package: libc6
Version: 2.11.2-10
Severity: important
Tags: squeeze patch

I've got a server with Intel Xeon E31270 CPU that runs Squeeze and I
constantly get Chef (a configuration management toolkit written in Ruby)
killed by SIGILL:

  Program received signal SIGILL, Illegal instruction.
  _dl_x86_64_save_sse () at ../sysdeps/x86_64/dl-trampoline.S:189

The attached patch (backported from glibc trunk) seems to fix the issue.

-- System Information:
Debian Release: 6.0.3
  APT prefers stable
  APT policy: (500, 'stable')
Architecture: amd64 (x86_64)

Kernel: Linux 2.6.32-5-xen-amd64 (SMP w/8 CPU cores)
Locale: LANG=en_US.UTF-8, LC_CTYPE=en_US.UTF-8 (charmap=UTF-8)
Shell: /bin/sh linked to /bin/dash

Versions of packages libc6 depends on:
ii  libc-bin                      2.11.2-10  Embedded GNU C Library: Binaries
ii  libgcc1                       1:4.4.5-8  GCC support library

libc6 recommends no packages.

Versions of packages libc6 suggests:
ii  debconf [debconf-2.0]         1.5.36.1   Debian configuration management sy
pn  glibc-doc                     <none>     (no description available)
ii  locales                       2.11.2-10  Embedded GNU C Library: National L

-- debconf-show failed

diff -ur eglibc-2.11.2//sysdeps/x86_64/dl-trampoline.S eglibc-2.11.2.changed//sysdeps/x86_64/dl-trampoline.S
--- eglibc-2.11.2//sysdeps/x86_64/dl-trampoline.S	2009-08-26 05:17:41.000000000 +0300
+++ eglibc-2.11.2.changed//sysdeps/x86_64/dl-trampoline.S	2011-10-25 01:10:36.153062354 +0300
@@ -139,16 +139,26 @@
 	movl	$1, %eax
 	cpuid
 	movq	%r11,%rbx		# Restore rbx
-	movl	$1, %eax
-	testl	$(1 << 28), %ecx
+	xorl	%eax, %eax
+	// AVX and XSAVE supported?
+	andl	$((1 << 28) | (1 << 27)), %ecx
+	cmpl	$((1 << 28) | (1 << 27)), %ecx
 	jne	2f
-	negl	%eax
-2:	movl	%eax, L(have_avx)(%rip)
+	xorl	%ecx, %ecx
+	// Get XFEATURE_ENABLED_MASK
+	xgetbv
+	andl	$0x6, %eax
+	cmpl	$0x6, %eax
+	// Nonzero if SSE and AVX state saving is enabled.
+	sete	%al
+2:	leal	-1(%eax,%eax), %eax
+	movl	%eax, L(have_avx)(%rip)
 	cmpl	$0, %eax
 
 1:	js	L(no_avx)
 
 #  define RESTORE_AVX
+#  define MORE_CODE
 #  include "dl-trampoline.h"
 
 	.align 16
@@ -176,11 +186,20 @@
 	movl	$1, %eax
 	cpuid
 	movq	%r11,%rbx		# Restore rbx
-	movl	$1, %eax
-	testl	$(1 << 28), %ecx
+	xorl	%eax, %eax
+	// AVX and XSAVE supported?
+	andl	$((1 << 28) | (1 << 27)), %ecx
+	cmpl	$((1 << 28) | (1 << 27)), %ecx
 	jne	2f
-	negl	%eax
-2:	movl	%eax, L(have_avx)(%rip)
+	xorl	%ecx, %ecx
+	// Get XFEATURE_ENABLED_MASK
+	xgetbv
+	andl	$0x6, %eax
+	cmpl	$0x6, %eax
+	// Nonzero if SSE and AVX state saving is enabled.
+	sete	%al
+2:	leal	-1(%eax,%eax), %eax
+	movl	%eax, L(have_avx)(%rip)
 	cmpl	$0, %eax
 
 1:	js	L(no_avx5)
diff -ur eglibc-2.11.2//sysdeps/x86_64/dl-trampoline.h eglibc-2.11.2.changed//sysdeps/x86_64/dl-trampoline.h
--- eglibc-2.11.2//sysdeps/x86_64/dl-trampoline.h	2009-08-09 17:33:15.000000000 +0300
+++ eglibc-2.11.2.changed//sysdeps/x86_64/dl-trampoline.h	2011-10-25 01:15:09.501060129 +0300
@@ -267,3 +267,10 @@
 				# (eats the reloc index and link_map)
 	cfi_adjust_cfa_offset(-48)
 	retq
+
+#ifdef MORE_CODE
+	cfi_adjust_cfa_offset(48)
+	cfi_rel_offset(%rbx, 0)
+	cfi_def_cfa_register(%rbx)
+# undef MORE_CODE
+#endif