Commits:
-
3ab76708
by Sebastian Andrzej Siewior at 2023-09-24T10:20:32+00:00
dak: Use threaded compression/ decompression.
Use threaded compression/ decompression while compressing or
decompressing files. The -T option is understood by xz and if the
threaded support is missing then xz continues single threaded.
The threaded compression option creates multiple blocks and this format
is understood by xz for a long time (since the early beginning).
The -e option for compression creates slightly smaller files by spending
a few extra cycles.
To add some numbers, the Packages.xz in amd64/sid os of now:
+---------+------+---------------+-----------------------+
| switch | time | Size in bytes | difference to default |
+---------+------+---------------+-----------------------+
| default |0:16 | 9513308 | - |
| -e |0:23 | 9329808 | -1.9% / -179 KiB |
| -T0 |0:07 | 9579240 | +0.6% / + 64 KiB |
| -T0e |0:10 | 9404052 | -1.1% / -106 KiB |
+---------+------+---------------+-----------------------+
One same hardware decompression of the original files takes 0m0,571s
and the threaded decompression takes 0m0,297s.
Signed-off-by: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
-
1e13e086
by Joerg Jaspert at 2023-09-24T10:21:23+00:00
Merge branch 'block_compress_packages' into 'master'
dak: Use threaded compression/ decompression.
See merge request ftp-team/dak!273
3 changed files:
Changes:
dak/generate_index_diffs.py
| ... |
... |
@@ -102,7 +102,7 @@ async def smartlink(f, t): |
|
102
|
102
|
elif os.path.isfile("%s.bz2" % (f)):
|
|
103
|
103
|
await call_decompressor(['bzip2', '-d'], '{}.bz2'.format(f), t)
|
|
104
|
104
|
elif os.path.isfile("%s.xz" % (f)):
|
|
105
|
|
- await call_decompressor(['xz', '-d'], '{}.xz'.format(f), t)
|
|
|
105
|
+ await call_decompressor(['xz', '-d', '-T0'], '{}.xz'.format(f), t)
|
|
106
|
106
|
elif os.path.isfile(f"{f}.zst"):
|
|
107
|
107
|
await call_decompressor(['zstd', '--decompress'], f'{f}.zst', t)
|
|
108
|
108
|
else:
|
daklib/compress.py
| ... |
... |
@@ -29,7 +29,7 @@ def decompress_zstd(input: IO, output: IO) -> None: |
|
29
|
29
|
|
|
30
|
30
|
|
|
31
|
31
|
def decompress_xz(input: IO, output: IO) -> None:
|
|
32
|
|
- subprocess.check_call(["xz", "--decompress"], stdin=input, stdout=output)
|
|
|
32
|
+ subprocess.check_call(["xz", "--decompress", "-T0"], stdin=input, stdout=output)
|
|
33
|
33
|
|
|
34
|
34
|
|
|
35
|
35
|
def decompress_bz2(input: IO, output: IO) -> None:
|
daklib/filewriter.py
| ... |
... |
@@ -42,7 +42,7 @@ class CompressionMethod: |
|
42
|
42
|
_compression_methods = (
|
|
43
|
43
|
CompressionMethod('bzip2', '.bz2', ['bzip2', '-9']),
|
|
44
|
44
|
CompressionMethod('gzip', '.gz', ['gzip', '-9cn', '--rsyncable', '--no-name']),
|
|
45
|
|
- CompressionMethod('xz', '.xz', ['xz', '-c']),
|
|
|
45
|
+ CompressionMethod('xz', '.xz', ['xz', '-c', '-e', '-T0']),
|
|
46
|
46
|
CompressionMethod('zstd', '.zst', ['zstd', '--compress']),
|
|
47
|
47
|
# 'none' must be the last compression method as BaseFileWriter
|
|
48
|
48
|
# handling it will remove the input file for other compressions
|
|