HALF ECHO LEARN / REFERENCE COLLECTION
v262.2src/basic/compress.c
Git blob: 58fbbe3ce54f45bd88ce88d1ea5f7b4d345a0a9e
Lines 1–1000 of 2,729
/* SPDX-License-Identifier: LGPL-2.1-or-later */
#include <fcntl.h>
#include <stdio.h>
#include <sys/stat.h>
#include <unistd.h>
#if HAVE_XZ
#ifndef SYSTEMD_CFLAGS_MARKER_LIBXZ
# error "missing libxz_cflags in meson dependency."
#endif
#include <lzma.h>
#endif
#if HAVE_LZ4
#ifndef SYSTEMD_CFLAGS_MARKER_LIBLZ4
# error "missing liblz4_cflags in meson dependency."
#endif
#include <lz4.h>
#include <lz4frame.h>
#include <lz4hc.h>
#endif
#if HAVE_ZSTD
#ifndef SYSTEMD_CFLAGS_MARKER_LIBZSTD
# error "missing libzstd_cflags in meson dependency."
#endif
#include <zstd.h>
#include <zstd_errors.h>
#endif
#if HAVE_ZLIB
#ifndef SYSTEMD_CFLAGS_MARKER_LIBZ
# error "missing libz_cflags in meson dependency."
#endif
#include <zlib.h>
#endif
#if HAVE_BZIP2
#ifndef SYSTEMD_CFLAGS_MARKER_LIBBZIP2
# error "missing libbzip2_cflags in meson dependency."
#endif
#include <bzlib.h>
#endif
#include "alloc-util.h"
#include "bitfield.h"
#include "compress.h"
#include "dlfcn-util.h"
#include "io-util.h"
#include "log.h"
#include "string-table.h"
#include "unaligned.h"
#if HAVE_XZ
static DLSYM_PROTOTYPE(lzma_code) = NULL;
static DLSYM_PROTOTYPE(lzma_easy_encoder) = NULL;
static DLSYM_PROTOTYPE(lzma_end) = NULL;
static DLSYM_PROTOTYPE(lzma_stream_buffer_encode) = NULL;
static DLSYM_PROTOTYPE(lzma_stream_decoder) = NULL;
static DLSYM_PROTOTYPE(lzma_lzma_preset) = NULL;
/* We can’t just do _cleanup_(sym_lzma_end) because a compiler bug makes
* this fail with:
* ../src/basic/compress.c: In function ‘decompress_blob_xz’:
* ../src/basic/compress.c:304:9: error: cleanup argument not a function
* 304 | _cleanup_(sym_lzma_end) lzma_stream s = LZMA_STREAM_INIT;
* | ^~~~~~~~~
*/
static inline void lzma_end_wrapper(lzma_stream *ls) {
sym_lzma_end(ls);
}
#endif
#if HAVE_LZ4
static DLSYM_PROTOTYPE(LZ4F_compressBegin) = NULL;
static DLSYM_PROTOTYPE(LZ4F_compressBound) = NULL;
static DLSYM_PROTOTYPE(LZ4F_compressEnd) = NULL;
static DLSYM_PROTOTYPE(LZ4F_compressUpdate) = NULL;
static DLSYM_PROTOTYPE(LZ4F_createCompressionContext) = NULL;
static DLSYM_PROTOTYPE(LZ4F_createDecompressionContext) = NULL;
static DLSYM_PROTOTYPE(LZ4F_decompress) = NULL;
static DLSYM_PROTOTYPE(LZ4F_freeCompressionContext) = NULL;
static DLSYM_PROTOTYPE(LZ4F_freeDecompressionContext) = NULL;
static DLSYM_PROTOTYPE(LZ4F_isError) = NULL;
static DLSYM_PROTOTYPE(LZ4_compress_HC) = NULL;
static DLSYM_PROTOTYPE(LZ4_compress_default) = NULL;
static DLSYM_PROTOTYPE(LZ4_decompress_safe) = NULL;
static DLSYM_PROTOTYPE(LZ4_decompress_safe_partial) = NULL;
static DLSYM_PROTOTYPE(LZ4_versionNumber) = NULL;
static const LZ4F_preferences_t lz4_preferences = {
.frameInfo.blockSizeID = 5,
};
#endif
#if HAVE_ZSTD
static DLSYM_PROTOTYPE(ZSTD_CCtx_setParameter) = NULL;
static DLSYM_PROTOTYPE(ZSTD_compress) = NULL;
static DLSYM_PROTOTYPE(ZSTD_compressStream2) = NULL;
static DLSYM_PROTOTYPE(ZSTD_createCCtx) = NULL;
static DLSYM_PROTOTYPE(ZSTD_createDCtx) = NULL;
static DLSYM_PROTOTYPE(ZSTD_CStreamInSize) = NULL;
static DLSYM_PROTOTYPE(ZSTD_CStreamOutSize) = NULL;
static DLSYM_PROTOTYPE(ZSTD_decompressStream) = NULL;
static DLSYM_PROTOTYPE(ZSTD_DStreamInSize) = NULL;
static DLSYM_PROTOTYPE(ZSTD_DStreamOutSize) = NULL;
static DLSYM_PROTOTYPE(ZSTD_freeCCtx) = NULL;
static DLSYM_PROTOTYPE(ZSTD_freeDCtx) = NULL;
static DLSYM_PROTOTYPE(ZSTD_getErrorCode) = NULL;
static DLSYM_PROTOTYPE(ZSTD_getErrorName) = NULL;
static DLSYM_PROTOTYPE(ZSTD_getFrameContentSize) = NULL;
static DLSYM_PROTOTYPE(ZSTD_isError) = NULL;
DEFINE_TRIVIAL_CLEANUP_FUNC_FULL_RENAME(ZSTD_DCtx*, sym_ZSTD_freeDCtx, ZSTD_freeDCtxp, NULL);
static int zstd_ret_to_errno(size_t ret) {
switch (sym_ZSTD_getErrorCode(ret)) {
case ZSTD_error_dstSize_tooSmall:
return -ENOBUFS;
case ZSTD_error_memory_allocation:
return -ENOMEM;
default:
return -EBADMSG;
}
}
#endif
#if HAVE_ZLIB
static DLSYM_PROTOTYPE(deflateInit2_) = NULL;
static DLSYM_PROTOTYPE(deflate) = NULL;
static DLSYM_PROTOTYPE(deflateEnd) = NULL;
static DLSYM_PROTOTYPE(inflateInit2_) = NULL;
static DLSYM_PROTOTYPE(inflate) = NULL;
static DLSYM_PROTOTYPE(inflateEnd) = NULL;
static DLSYM_PROTOTYPE(inflateReset) = NULL;
static inline void deflateEnd_wrapper(z_stream *s) {
sym_deflateEnd(s);
}
static inline void inflateEnd_wrapper(z_stream *s) {
sym_inflateEnd(s);
}
#endif
#if HAVE_BZIP2
static DLSYM_PROTOTYPE(BZ2_bzCompressInit) = NULL;
static DLSYM_PROTOTYPE(BZ2_bzCompress) = NULL;
static DLSYM_PROTOTYPE(BZ2_bzCompressEnd) = NULL;
static DLSYM_PROTOTYPE(BZ2_bzDecompressInit) = NULL;
static DLSYM_PROTOTYPE(BZ2_bzDecompress) = NULL;
static DLSYM_PROTOTYPE(BZ2_bzDecompressEnd) = NULL;
static inline void BZ2_bzCompressEnd_wrapper(bz_stream *s) {
sym_BZ2_bzCompressEnd(s);
}
static inline void BZ2_bzDecompressEnd_wrapper(bz_stream *s) {
sym_BZ2_bzDecompressEnd(s);
}
#endif
/* Opaque Compressor/Decompressor struct definition */
struct Compressor {
Compression type;
bool encoding;
bool finished;
union {
#if HAVE_XZ
lzma_stream xz;
#endif
#if HAVE_LZ4
struct {
LZ4F_compressionContext_t c_lz4;
void *lz4_header; /* stashed frame header from LZ4F_compressBegin */
size_t lz4_header_size;
};
LZ4F_decompressionContext_t d_lz4;
#endif
#if HAVE_ZSTD
ZSTD_CCtx *c_zstd;
ZSTD_DCtx *d_zstd;
#endif
#if HAVE_ZLIB
z_stream gzip;
#endif
#if HAVE_BZIP2
bz_stream bzip2;
#endif
};
};
#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
/* zlib windowBits value for gzip format: MAX_WBITS (15) + 16 to enable gzip header detection/generation */
#define ZLIB_WBITS_GZIP (15 + 16)
static const char* const compression_table[_COMPRESSION_MAX] = {
[COMPRESSION_NONE] = "uncompressed", /* backwards compatibility with importd */
[COMPRESSION_XZ] = "xz",
[COMPRESSION_LZ4] = "lz4",
[COMPRESSION_ZSTD] = "zstd",
[COMPRESSION_GZIP] = "gzip",
[COMPRESSION_BZIP2] = "bzip2",
};
static const char* const compression_uppercase_table[_COMPRESSION_MAX] = {
[COMPRESSION_NONE] = "NONE", /* backwards compatibility with SYSTEMD_JOURNAL_COMPRESS=NONE */
[COMPRESSION_XZ] = "XZ",
[COMPRESSION_LZ4] = "LZ4",
[COMPRESSION_ZSTD] = "ZSTD",
[COMPRESSION_GZIP] = "GZIP",
[COMPRESSION_BZIP2] = "BZIP2",
};
static const char* const compression_extension_table[_COMPRESSION_MAX] = {
[COMPRESSION_NONE] = "",
[COMPRESSION_XZ] = ".xz",
[COMPRESSION_LZ4] = ".lz4",
[COMPRESSION_ZSTD] = ".zst",
[COMPRESSION_GZIP] = ".gz",
[COMPRESSION_BZIP2] = ".bz2",
};
DEFINE_STRING_TABLE_LOOKUP(compression, Compression);
DEFINE_STRING_TABLE_LOOKUP(compression_uppercase, Compression);
DEFINE_STRING_TABLE_LOOKUP(compression_extension, Compression);
Compression compression_from_string_harder(const char *s) {
Compression c;
assert(s);
c = compression_from_string(s);
if (c >= 0)
return c;
return compression_uppercase_from_string(s);
}
Compression compression_from_filename(const char *filename) {
Compression c;
const char *e;
assert(filename);
e = strrchr(filename, '.');
if (!e)
return COMPRESSION_NONE;
c = compression_extension_from_string(e);
if (c < 0)
return COMPRESSION_NONE;
return c;
}
bool compression_supported_journal(Compression c) {
static const unsigned supported =
(1U << COMPRESSION_NONE) |
(1U << COMPRESSION_XZ) * HAVE_XZ |
(1U << COMPRESSION_LZ4) * HAVE_LZ4 |
(1U << COMPRESSION_ZSTD) * HAVE_ZSTD;
assert(c >= 0);
assert(c < _COMPRESSION_MAX);
return BIT_SET(supported, c);
}
bool compression_supported(Compression c) {
static const unsigned supported =
(1U << COMPRESSION_NONE) |
(1U << COMPRESSION_XZ) * HAVE_XZ |
(1U << COMPRESSION_LZ4) * HAVE_LZ4 |
(1U << COMPRESSION_ZSTD) * HAVE_ZSTD |
(1U << COMPRESSION_GZIP) * HAVE_ZLIB |
(1U << COMPRESSION_BZIP2) * HAVE_BZIP2;
assert(c >= 0);
assert(c < _COMPRESSION_MAX);
return BIT_SET(supported, c);
}
Compression compression_detect_from_magic(const uint8_t data[static COMPRESSION_MAGIC_BYTES_MAX]) {
/* Magic signatures per RFC 1952 (gzip), tukaani.org/xz/xz-file-format.txt (xz),
* RFC 8878 (zstd), lz4/doc/lz4_Frame_format.md (lz4), and the bzip2 file format.
* Make sure to update COMPRESSION_MAGIC_BYTES_MAX if needed when adding a new magic. */
if (memcmp(data, (const uint8_t[]) { 0x1f, 0x8b }, 2) == 0)
return COMPRESSION_GZIP;
if (memcmp(data, (const uint8_t[]) { 0xfd, '7', 'z', 'X', 'Z', 0x00 }, 6) == 0)
return COMPRESSION_XZ;
if (memcmp(data, (const uint8_t[]) { 0x28, 0xb5, 0x2f, 0xfd }, 4) == 0)
return COMPRESSION_ZSTD;
if (memcmp(data, (const uint8_t[]) { 0x04, 0x22, 0x4d, 0x18 }, 4) == 0)
return COMPRESSION_LZ4;
if (memcmp(data, (const uint8_t[]) { 'B', 'Z', 'h' }, 3) == 0)
return COMPRESSION_BZIP2;
return _COMPRESSION_INVALID;
}
int dlopen_xz(int log_level) {
#if HAVE_XZ
static void *lzma_dl = NULL;
LIBLZMA_NOTE(suggested);
return dlopen_many_sym_or_warn(
&lzma_dl,
"liblzma.so.5", log_level,
DLSYM_ARG(lzma_code),
DLSYM_ARG(lzma_easy_encoder),
DLSYM_ARG(lzma_end),
DLSYM_ARG(lzma_stream_buffer_encode),
DLSYM_ARG(lzma_lzma_preset),
DLSYM_ARG(lzma_stream_decoder));
#else
return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP),
"lzma support is not compiled in.");
#endif
}
int dlopen_lz4(int log_level) {
#if HAVE_LZ4
static void *lz4_dl = NULL;
LIBLZ4_NOTE(suggested);
return dlopen_many_sym_or_warn(
&lz4_dl,
"liblz4.so.1", log_level,
DLSYM_ARG(LZ4F_compressBegin),
DLSYM_ARG(LZ4F_compressBound),
DLSYM_ARG(LZ4F_compressEnd),
DLSYM_ARG(LZ4F_compressUpdate),
DLSYM_ARG(LZ4F_createCompressionContext),
DLSYM_ARG(LZ4F_createDecompressionContext),
DLSYM_ARG(LZ4F_decompress),
DLSYM_ARG(LZ4F_freeCompressionContext),
DLSYM_ARG(LZ4F_freeDecompressionContext),
DLSYM_ARG(LZ4F_isError),
DLSYM_ARG(LZ4_compress_default),
DLSYM_ARG(LZ4_compress_HC),
DLSYM_ARG(LZ4_decompress_safe),
DLSYM_ARG(LZ4_decompress_safe_partial),
DLSYM_ARG(LZ4_versionNumber));
#else
return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP),
"lz4 support is not compiled in.");
#endif
}
int dlopen_zstd(int log_level) {
#if HAVE_ZSTD
static void *zstd_dl = NULL;
LIBZSTD_NOTE(suggested);
return dlopen_many_sym_or_warn(
&zstd_dl,
"libzstd.so.1", log_level,
DLSYM_ARG(ZSTD_getErrorCode),
DLSYM_ARG(ZSTD_compress),
DLSYM_ARG(ZSTD_getFrameContentSize),
DLSYM_ARG(ZSTD_decompressStream),
DLSYM_ARG(ZSTD_getErrorName),
DLSYM_ARG(ZSTD_DStreamOutSize),
DLSYM_ARG(ZSTD_CStreamInSize),
DLSYM_ARG(ZSTD_CStreamOutSize),
DLSYM_ARG(ZSTD_CCtx_setParameter),
DLSYM_ARG(ZSTD_compressStream2),
DLSYM_ARG(ZSTD_DStreamInSize),
DLSYM_ARG(ZSTD_freeCCtx),
DLSYM_ARG(ZSTD_freeDCtx),
DLSYM_ARG(ZSTD_isError),
DLSYM_ARG(ZSTD_createDCtx),
DLSYM_ARG(ZSTD_createCCtx));
#else
return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP),
"zstd support is not compiled in.");
#endif
}
int dlopen_zlib(int log_level) {
#if HAVE_ZLIB
static void *zlib_dl = NULL;
LIBZ_NOTE(suggested);
return dlopen_many_sym_or_warn(
&zlib_dl,
"libz.so.1", log_level,
DLSYM_ARG(deflateInit2_),
DLSYM_ARG(deflate),
DLSYM_ARG(deflateEnd),
DLSYM_ARG(inflateInit2_),
DLSYM_ARG(inflate),
DLSYM_ARG(inflateEnd),
DLSYM_ARG(inflateReset));
#else
return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP),
"zlib support is not compiled in.");
#endif
}
int dlopen_bzip2(int log_level) {
#if HAVE_BZIP2
static void *bzip2_dl = NULL;
LIBBZ2_NOTE(suggested);
return dlopen_many_sym_or_warn(
&bzip2_dl,
"libbz2.so.1", log_level,
DLSYM_ARG(BZ2_bzCompressInit),
DLSYM_ARG(BZ2_bzCompress),
DLSYM_ARG(BZ2_bzCompressEnd),
DLSYM_ARG(BZ2_bzDecompressInit),
DLSYM_ARG(BZ2_bzDecompress),
DLSYM_ARG(BZ2_bzDecompressEnd));
#else
return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP),
"bzip2 support is not compiled in.");
#endif
}
int dlopen_compress_journal(Compression c, int log_level) {
switch (c) {
case COMPRESSION_NONE:
return 0;
case COMPRESSION_XZ:
return dlopen_xz(log_level);
case COMPRESSION_LZ4:
return dlopen_lz4(log_level);
case COMPRESSION_ZSTD:
return dlopen_zstd(log_level);
default:
return -EOPNOTSUPP;
}
}
int dlopen_compress(Compression c, int log_level) {
switch (c) {
case COMPRESSION_GZIP:
return dlopen_zlib(log_level);
case COMPRESSION_BZIP2:
return dlopen_bzip2(log_level);
default:
return dlopen_compress_journal(c, log_level);
}
}
static int compress_blob_xz(
const void *src,
uint64_t src_size,
void *dst,
size_t dst_alloc_size,
size_t *dst_size,
int level) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_alloc_size > 0);
assert(dst_size);
#if HAVE_XZ
lzma_options_lzma opt = {
1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4
};
lzma_filter filters[] = {
{ LZMA_FILTER_LZMA2, &opt },
{ LZMA_VLI_UNKNOWN, NULL }
};
lzma_ret ret;
size_t out_pos = 0;
int r;
r = dlopen_xz(LOG_DEBUG);
if (r < 0)
return r;
if (level >= 0) {
/* This returns lzma_bool, and true means failure, i.e. the specified level is not supported. */
if (sym_lzma_lzma_preset(&opt, (uint32_t) level))
return -EINVAL;
}
/* Returns < 0 if we couldn't compress the data or the
* compressed result is longer than the original */
if (src_size < 80)
return -ENOBUFS;
ret = sym_lzma_stream_buffer_encode(filters, LZMA_CHECK_NONE, /* allocator= */ NULL,
src, src_size, dst, &out_pos, dst_alloc_size);
if (ret != LZMA_OK)
return -ENOBUFS;
*dst_size = out_pos;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
static int compress_blob_lz4(
const void *src,
uint64_t src_size,
void *dst,
size_t dst_alloc_size,
size_t *dst_size,
int level) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_alloc_size > 0);
assert(dst_size);
#if HAVE_LZ4
int r;
r = dlopen_lz4(LOG_DEBUG);
if (r < 0)
return r;
/* Returns < 0 if we couldn't compress the data or the
* compressed result is longer than the original */
if (src_size < 9)
return -ENOBUFS;
if (src_size > INT_MAX)
return -EFBIG;
if (dst_alloc_size > INT_MAX)
dst_alloc_size = INT_MAX;
if (level <= 0)
r = sym_LZ4_compress_default(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8);
else
r = sym_LZ4_compress_HC(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8, level);
if (r <= 0)
return -ENOBUFS;
unaligned_write_le64(dst, src_size);
*dst_size = r + 8;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
static int compress_blob_zstd(
const void *src,
uint64_t src_size,
void *dst,
size_t dst_alloc_size,
size_t *dst_size,
int level) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_alloc_size > 0);
assert(dst_size);
#if HAVE_ZSTD
size_t k;
int r;
r = dlopen_zstd(LOG_DEBUG);
if (r < 0)
return r;
k = sym_ZSTD_compress(dst, dst_alloc_size, src, src_size, level < 0 ? 0 : level);
if (sym_ZSTD_isError(k))
return zstd_ret_to_errno(k);
*dst_size = k;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
static int compress_blob_gzip(const void *src, uint64_t src_size,
void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_alloc_size > 0);
assert(dst_size);
#if HAVE_ZLIB
int r;
r = dlopen_zlib(LOG_DEBUG);
if (r < 0)
return r;
if (src_size > UINT_MAX)
return -EFBIG;
if (dst_alloc_size > UINT_MAX)
dst_alloc_size = UINT_MAX;
_cleanup_(deflateEnd_wrapper) z_stream s = {};
r = sym_deflateInit2_(&s, level < 0 ? Z_DEFAULT_COMPRESSION : level,
/* method= */ Z_DEFLATED,
/* windowBits= */ ZLIB_WBITS_GZIP,
/* memLevel= */ 8,
/* strategy= */ Z_DEFAULT_STRATEGY,
ZLIB_VERSION, (int) sizeof(s));
if (r != Z_OK)
return -ENOMEM;
s.next_in = (void*) src;
s.avail_in = src_size;
s.next_out = dst;
s.avail_out = dst_alloc_size;
r = sym_deflate(&s, Z_FINISH);
if (r != Z_STREAM_END)
return -ENOBUFS;
*dst_size = dst_alloc_size - s.avail_out;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
static int compress_blob_bzip2(
const void *src, uint64_t src_size,
void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_alloc_size > 0);
assert(dst_size);
#if HAVE_BZIP2
int r;
r = dlopen_bzip2(LOG_DEBUG);
if (r < 0)
return r;
if (src_size > UINT_MAX)
return -EFBIG;
if (dst_alloc_size > UINT_MAX)
dst_alloc_size = UINT_MAX;
_cleanup_(BZ2_bzCompressEnd_wrapper) bz_stream s = {};
r = sym_BZ2_bzCompressInit(&s, level < 0 ? 9 : level, /* verbosity= */ 0, /* workFactor= */ 0);
if (r != BZ_OK)
return -ENOMEM;
s.next_in = (char*) src;
s.avail_in = src_size;
s.next_out = (char*) dst;
s.avail_out = dst_alloc_size;
r = sym_BZ2_bzCompress(&s, BZ_FINISH);
if (r != BZ_STREAM_END)
return -ENOBUFS;
*dst_size = dst_alloc_size - s.avail_out;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
int compress_blob_journal(
Compression compression,
const void *src, uint64_t src_size,
void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
switch (compression) {
case COMPRESSION_XZ:
return compress_blob_xz(src, src_size, dst, dst_alloc_size, dst_size, level);
case COMPRESSION_LZ4:
return compress_blob_lz4(src, src_size, dst, dst_alloc_size, dst_size, level);
case COMPRESSION_ZSTD:
return compress_blob_zstd(src, src_size, dst, dst_alloc_size, dst_size, level);
default:
return -EOPNOTSUPP;
}
}
int compress_blob(
Compression compression,
const void *src, uint64_t src_size,
void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
switch (compression) {
case COMPRESSION_GZIP:
return compress_blob_gzip(src, src_size, dst, dst_alloc_size, dst_size, level);
case COMPRESSION_BZIP2:
return compress_blob_bzip2(src, src_size, dst, dst_alloc_size, dst_size, level);
default:
return compress_blob_journal(compression, src, src_size, dst, dst_alloc_size, dst_size, level);
}
}
static int decompress_blob_xz(
const void *src,
uint64_t src_size,
void **dst,
size_t *dst_size,
size_t dst_max) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_size);
#if HAVE_XZ
int r;
r = dlopen_xz(LOG_DEBUG);
if (r < 0)
return r;
_cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, /* flags= */ 0);
if (ret != LZMA_OK)
return -ENOMEM;
size_t space = MIN(src_size * 2, dst_max ?: SIZE_MAX);
if (!greedy_realloc(dst, space, 1))
return -ENOMEM;
s.next_in = src;
s.avail_in = src_size;
s.next_out = *dst;
s.avail_out = space;
for (;;) {
size_t used;
ret = sym_lzma_code(&s, LZMA_FINISH);
if (ret == LZMA_STREAM_END)
break;
if (ret != LZMA_OK)
return -ENOMEM;
if (dst_max > 0 && (space - s.avail_out) >= dst_max)
break;
if (dst_max > 0 && space == dst_max)
return -ENOBUFS;
used = space - s.avail_out;
/* Silence static analyzers, space is bounded by allocation size */
assert(space <= SIZE_MAX / 2);
space = MIN(2 * space, dst_max ?: SIZE_MAX);
if (!greedy_realloc(dst, space, 1))
return -ENOMEM;
s.avail_out = space - used;
s.next_out = *(uint8_t**)dst + used;
}
*dst_size = space - s.avail_out;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
static int decompress_blob_lz4(
const void *src,
uint64_t src_size,
void **dst,
size_t *dst_size,
size_t dst_max) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_size);
#if HAVE_LZ4
char* out;
int r, size; /* LZ4 uses int for size */
r = dlopen_lz4(LOG_DEBUG);
if (r < 0)
return r;
if (src_size <= 8)
return -EBADMSG;
if (src_size - 8 > INT_MAX)
return -EFBIG;
size = unaligned_read_le64(src);
if (size < 0 || (unsigned) size != unaligned_read_le64(src))
return -EFBIG;
if (dst_max > 0 && (size_t) size > dst_max)
return -ENOBUFS;
out = greedy_realloc(dst, size, 1);
if (!out)
return -ENOMEM;
r = sym_LZ4_decompress_safe((char*)src + 8, out, src_size - 8, size);
if (r < 0 || r != size)
return -EBADMSG;
*dst_size = size;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
size_t zstd_dstream_out_size(void) {
#if HAVE_ZSTD
if (dlopen_zstd(LOG_DEBUG) < 0)
return 0;
return sym_ZSTD_DStreamOutSize();
#else
return 0;
#endif
}
static int decompress_blob_zstd(
const void *src,
uint64_t src_size,
void **dst,
size_t *dst_size,
size_t dst_max) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_size);
#if HAVE_ZSTD
int r;
r = dlopen_zstd(LOG_DEBUG);
if (r < 0)
return r;
/* ZSTD_getFrameContentSize() returns unsigned long long, which is not necessarily size_t, so
* keep the value in that type until we've clamped it down to something that fits size_t. */
unsigned long long size = sym_ZSTD_getFrameContentSize(src, src_size);
if (size == ZSTD_CONTENTSIZE_ERROR)
return -EBADMSG;
/* ZSTD_CONTENTSIZE_UNKNOWN is returned when the frame header doesn't record the decompressed
* size, which is the case e.g. for kernel images compressed with 'zstd'. Per the zstd
* documentation this is not an error, it just means we don't know the output size upfront and
* have to grow the output buffer as we decompress. */
/* dst_max == 0 means "no limit"; fold that into a single ceiling up front so the rest of the
* function can treat the unlimited case like any other rather than repeating the sentinel. */
size_t cap = dst_max ?: SIZE_MAX;
size_t space;
if (size != ZSTD_CONTENTSIZE_UNKNOWN)
/* The MIN clamps the (possibly 64-bit) content size to the ceiling, which is <= SIZE_MAX,
* so the result always fits size_t. */
space = MAX(sym_ZSTD_DStreamOutSize(), (size_t) MIN(size, (unsigned long long) cap));
else {
/* Start from twice the compressed size, guarding the doubling against overflow, then
* clamp to the ceiling. */
uint64_t twice = src_size <= UINT64_MAX / 2 ? src_size * 2 : UINT64_MAX;
space = MAX(sym_ZSTD_DStreamOutSize(), (size_t) MIN(twice, (uint64_t) cap));
}
if (!greedy_realloc(dst, space, 1))
return -ENOMEM;
space = MALLOC_SIZEOF_SAFE(*dst);
_cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = sym_ZSTD_createDCtx();
if (!dctx)
return -ENOMEM;
ZSTD_inBuffer input = {
.src = src,
.size = src_size,
};
ZSTD_outBuffer output = {
.dst = *dst,
/* Never offer the decoder more room than the cap: greedy_realloc() over-allocates, so
* without this clamp a single ZSTD_decompressStream() call could write past dst_max
* before the output.pos >= cap check below fires. */
.size = MIN(space, cap),
};
for (;;) {
size_t k = sym_ZSTD_decompressStream(dctx, &output, &input);
if (sym_ZSTD_isError(k))
return log_debug_errno(zstd_ret_to_errno(k), "ZSTD decoder failed: %s", sym_ZSTD_getErrorName(k));
/* output.size is clamped to MIN(space, cap) here and after every grow, and zstd keeps
* output.pos <= output.size, so output.pos never exceeds cap: this fires exactly when the
* caller's limit has been filled. */
if (output.pos >= cap)
break;
/* k == 0 means the current frame was fully decoded and flushed. If no input is left
* either, the whole stream is done. This must be checked before the grow branch below:
* the frame can finish exactly as the output buffer fills, and growing then would call
* the decoder again on already-exhausted input, which it'd misread as a truncated next
* frame. */
if (k == 0 && input.pos >= input.size)
break;
/* Otherwise there is still work to do: either decoded data buffered inside the decoder
* that didn't fit (k != 0, e.g. zstd's buffered-flush mode), or another concatenated
* frame to decode (k == 0 with input left). If the output buffer is full, grow it and
* call again. Guard against the doubling overflowing size_t (only reachable with an
* uncapped dst_max once the allocation has already grown past SIZE_MAX/2). */
if (output.pos == output.size) {
if (space > SIZE_MAX / 2)
return -E2BIG;
space = MIN(2 * space, cap);
if (!greedy_realloc(dst, space, 1))
return -ENOMEM;
space = MALLOC_SIZEOF_SAFE(*dst);
output.dst = *dst;
output.size = MIN(space, cap);
continue;
}
/* The output buffer still has room but the decoder stopped. If input is exhausted the
* decoder wanted more (k != 0, since k == 0 + no input broke above), i.e. the stream was
* truncated mid-frame. Otherwise input remains (a concatenated frame) and we loop. */
if (input.pos >= input.size)
return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "ZSTD stream ended unexpectedly, probably corrupted.");
}
*dst_size = output.pos;
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
static int decompress_blob_gzip(
const void *src,
uint64_t src_size,
void **dst,
size_t *dst_size,
size_t dst_max) {
assert(src);
assert(src_size > 0);
assert(dst);
assert(dst_size);
#if HAVE_ZLIB
int r;
r = dlopen_zlib(LOG_DEBUG);
if (r < 0)
return r;
if (src_size > UINT_MAX)
return -EFBIG;
_cleanup_(inflateEnd_wrapper) z_stream s = {};
r = sym_inflateInit2_(&s, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(s));
if (r != Z_OK)
return -ENOMEM;
size_t space = MIN3(src_size * 2, dst_max ?: SIZE_MAX, (size_t) UINT_MAX);
if (!greedy_realloc(dst, space, 1))
return -ENOMEM;
s.next_in = (void*) src;
s.avail_in = src_size;
s.next_out = *dst;
s.avail_out = space;
for (;;) {
size_t used;
r = sym_inflate(&s, Z_NO_FLUSH);
if (r == Z_STREAM_END)
break;
if (!IN_SET(r, Z_OK, Z_BUF_ERROR))
return -EBADMSG;