Bug#932491: python3-apt: segfault reading from lzma stream

[Cyril Brulebois <kibi@debian.org>]

Cyril Brulebois <kibi@debian.org> (2023-11-02):
> Today I had a few more minutes to spend on this, so here's a little
> debugging session. My main system is still bullseye, but the same tests
> in a bookworm chroots fail the same way.

“But maybe it's a bug in the lzma library?” one might ask.

Adding a bzip2 test between gzip and lzma leads to the following, again
on both bullseye and bookworm (after creating a Test.bz2/Packages.bz2
from one of the other files):

With bug-932491-aa.py (bug-932491-a.py + bzip2):

    $ ./bug-932491-aa.py Test
    gz == bz: True
    gz == xz: True
    gz: section 1 size: 29
    gz: section 1 keys: ['Package', 'Desc']
    gz: section 2 size: 47
    gz: section 2 keys: ['Package', 'Desc']
    Traceback (most recent call last):
      File "/home/kibi/tmp/./bug-932491-c.py", line 37, in <module>
        tf_bz.step()
    apt_pkg.Error: E:Unable to parse package file  (1)

    $ ./bug-932491-aa.py Packages
    gz == bz: True
    gz == xz: True
    gz: section 1 size: 1281
    gz: section 1 keys: ['Package', 'Version', 'Installed-Size', 'Maintainer', 'Architecture', 'Depends', 'Pre-Depends', 'Description', 'Homepage', 'Description-md5', 'Tag', 'Section', 'Priority', 'Filename', 'Size', 'MD5sum', 'SHA256']
    gz: section 2 size: 585
    gz: section 2 keys: ['Package', 'Version', 'Installed-Size', 'Maintainer', 'Architecture', 'Pre-Depends', 'Suggests', 'Description', 'Homepage', 'Description-md5', 'Tag', 'Section', 'Priority', 'Filename', 'Size', 'MD5sum', 'SHA256']
    bz: section 1 size: 1410
    Segmentation fault

With bug-932491-bb.py (bug-932491-b.py + bzip2):

    $ ./bug-932491-bb.py Test
    gz packages: 2
    Traceback (most recent call last):
      File "/home/kibi/tmp/./bug-932491-bb.py", line 26, in <module>
        for stanza in tf_bz:
    apt_pkg.Error: E:Unable to parse package file  (1)

    $ ./bug-932491-bb.py Packages
    gz packages: 50771
    Traceback (most recent call last):
      File "/home/kibi/tmp/./bug-932491-bb.py", line 27, in <module>
        bz_packages.append(stanza['Package'])
                           ~~~~~~^^^^^^^^^^^
    KeyError: 'Package'


It looks like we might be getting chunks of different sizes depending on
the underlying file objects, and some buffering/seeking code is buggy on
the apt_pkg side?


Cheers,
-- 
Cyril Brulebois (kibi@debian.org)            <https://debamax.com/>
D-I release manager -- Release team member -- Freelance Consultant

#!/usr/bin/python3
"""
Test case for #932491, version a+bz2
"""
import bz2
import gzip
import lzma
import sys

import apt_pkg

root = sys.argv[1]

# Check data decompression works fine:
with gzip.open(f'{root}.gz') as gz:
    gz_text = gz.read()
with bz2.open(f'{root}.bz2') as bz:
    bz_text = bz.read()
with lzma.open(f'{root}.xz') as xz:
    xz_text = xz.read()
print(f'gz == bz: {gz_text == bz_text}')
print(f'gz == xz: {gz_text == xz_text}')

# Perform 2 manual steps with gz:
with gzip.open(f'{root}.gz') as gz:
    tf_gz = apt_pkg.TagFile(gz)
    tf_gz.step()
    print(f'gz: section 1 size: {tf_gz.section.bytes()}')
    print(f'gz: section 1 keys: {tf_gz.section.keys()}')
    tf_gz.step()
    print(f'gz: section 2 size: {tf_gz.section.bytes()}')
    print(f'gz: section 2 keys: {tf_gz.section.keys()}')

# Perform 2 manual steps with bz:
with bz2.open(f'{root}.bz2') as bz:
    tf_bz = apt_pkg.TagFile(bz)
    tf_bz.step()
    print(f'bz: section 1 size: {tf_bz.section.bytes()}')
    print(f'bz: section 1 keys: {tf_bz.section.keys()}')
    tf_bz.step()
    print(f'bz: section 2 size: {tf_bz.section.bytes()}')
    print(f'bz: section 2 keys: {tf_bz.section.keys()}')

# Perform 2 manual steps with xz:
with lzma.open(f'{root}.xz') as xz:
    tf_xz = apt_pkg.TagFile(xz)
    tf_xz.step()
    print(f'xz: section 1 size: {tf_xz.section.bytes()}')
    print(f'xz: section 1 keys: {tf_xz.section.keys()}')
    tf_xz.step()
    print(f'xz: section 2 size: {tf_xz.section.bytes()}')
    print(f'xz: section 2 keys: {tf_xz.section.keys()}')

#!/usr/bin/python3
"""
Test case for #932491: version b+bz2
"""
import bz2
import gzip
import lzma
import sys

import apt_pkg

root = sys.argv[1]

# Start a loop:
gz_packages = []
with gzip.open(f'{root}.gz') as gz:
    tf_gz = apt_pkg.TagFile(gz)
    for stanza in tf_gz:
        gz_packages.append(stanza['Package'])
print(f'gz packages: {len(gz_packages)}')

# Start a loop:
bz_packages = []
with bz2.open(f'{root}.bz2') as bz:
    tf_bz = apt_pkg.TagFile(bz)
    for stanza in tf_bz:
        bz_packages.append(stanza['Package'])
print(f'bz packages: {len(bz_packages)}')

# Start a loop:
xz_packages = []
with lzma.open(f'{root}.xz') as xz:
    tf_xz = apt_pkg.TagFile(xz)
    for stanza in tf_xz:
        print('.', end='')
        xz_packages.append(stanza['Package'])
print()
print(f'xz packages: {len(xz_packages)}')

Attachment: signature.asc
Description: PGP signature