HALF ECHO LEARN / REFERENCE COLLECTION

v262.2

src/basic/compress.c

88,255 bytesGit mode 100644Download original fileLGPL-2.1-or-later

Git blob: 58fbbe3ce54f45bd88ce88d1ea5f7b4d345a0a9e

Lines 1001–2000 of 2,729

                if (dst_max > 0 && (space - s.avail_out) >= dst_max)
                        break;
                if (dst_max > 0 && space == dst_max)
                        return -ENOBUFS;
 
                used = space - s.avail_out;
                space = MIN3(2 * space, dst_max ?: SIZE_MAX, UINT_MAX);
                if (!greedy_realloc(dst, space, 1))
                        return -ENOMEM;
 
                s.avail_out = space - used;
                s.next_out = *(uint8_t**)dst + used;
        }
 
        *dst_size = space - s.avail_out;
        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}
 
static int decompress_blob_bzip2(
                const void *src,
                uint64_t src_size,
                void **dst,
                size_t *dst_size,
                size_t dst_max) {
 
        assert(src);
        assert(src_size > 0);
        assert(dst);
        assert(dst_size);
 
#if HAVE_BZIP2
        int r;
 
        r = dlopen_bzip2(LOG_DEBUG);
        if (r < 0)
                return r;
 
        if (src_size > UINT_MAX)
                return -EFBIG;
 
        _cleanup_(BZ2_bzDecompressEnd_wrapper) bz_stream s = {};
 
        if (sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0) != BZ_OK)
                return -EIO;
 
        size_t space = MIN3(src_size * 2, dst_max ?: SIZE_MAX, (size_t) UINT_MAX);
        if (!greedy_realloc(dst, space, 1))
                return -ENOMEM;
 
        s.next_in = (char*) src;
        s.avail_in = src_size;
        s.next_out = (char*) *dst;
        s.avail_out = space;
 
        for (;;) {
                size_t used;
 
                r = sym_BZ2_bzDecompress(&s);
                if (r == BZ_STREAM_END)
                        break;
                if (r != BZ_OK)
                        return -EBADMSG;
 
                if (dst_max > 0 && (space - s.avail_out) >= dst_max)
                        break;
                if (dst_max > 0 && space == dst_max)
                        return -ENOBUFS;
 
                used = space - s.avail_out;
                space = MIN3(2 * space, dst_max ?: SIZE_MAX, (size_t) UINT_MAX);
                if (!greedy_realloc(dst, space, 1))
                        return -ENOMEM;
 
                s.avail_out = space - used;
                s.next_out = (char*) *dst + used;
        }
 
        *dst_size = space - s.avail_out;
        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}
 
int decompress_blob_journal(
                Compression compression,
                const void *src,
                uint64_t src_size,
                void **dst,
                size_t *dst_size,
                size_t dst_max) {
 
        switch (compression) {
        case COMPRESSION_XZ:
                return decompress_blob_xz(
                                src, src_size,
                                dst, dst_size, dst_max);
        case COMPRESSION_LZ4:
                return decompress_blob_lz4(
                                src, src_size,
                                dst, dst_size, dst_max);
        case COMPRESSION_ZSTD:
                return decompress_blob_zstd(
                                src, src_size,
                                dst, dst_size, dst_max);
        default:
                return -EPROTONOSUPPORT;
        }
}
 
int decompress_blob(
                Compression compression,
                const void *src,
                uint64_t src_size,
                void **dst,
                size_t *dst_size,
                size_t dst_max) {
 
        switch (compression) {
        case COMPRESSION_GZIP:
                return decompress_blob_gzip(
                                src, src_size,
                                dst, dst_size, dst_max);
        case COMPRESSION_BZIP2:
                return decompress_blob_bzip2(
                                src, src_size,
                                dst, dst_size, dst_max);
        default:
                return decompress_blob_journal(
                                compression,
                                src, src_size,
                                dst, dst_size, dst_max);
        }
}
 
int decompress_zlib_raw(
                const void *src,
                uint64_t src_size,
                void *dst,
                size_t dst_size,
                int wbits) {
 
#if HAVE_ZLIB
        int r;
 
        r = dlopen_zlib(LOG_DEBUG);
        if (r < 0)
                return r;
 
        if (src_size > UINT_MAX)
                return -EFBIG;
        if (dst_size > UINT_MAX)
                return -EFBIG;
 
        _cleanup_(inflateEnd_wrapper) z_stream s = {
                .next_in = (void*) src,
                .avail_in = src_size,
                .next_out = dst,
                .avail_out = dst_size,
        };
 
        r = sym_inflateInit2_(&s, /* windowBits= */ wbits, ZLIB_VERSION, (int) sizeof(s));
        if (r != Z_OK)
                return -EIO;
 
        r = sym_inflate(&s, Z_FINISH);
        size_t produced = (uint8_t*) s.next_out - (uint8_t*) dst;
 
        if (r != Z_STREAM_END || produced != dst_size)
                return -EBADMSG;
 
        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}
 
static int decompress_startswith_xz(
                const void *src,
                uint64_t src_size,
                void **buffer,
                const void *prefix,
                size_t prefix_len,
                uint8_t extra) {
 
        /* Checks whether the decompressed blob starts with the mentioned prefix. The byte extra needs to
         * follow the prefix */
 
        assert(src);
        assert(src_size > 0);
        assert(buffer);
        assert(prefix);
 
#if HAVE_XZ
        int r;
 
        r = dlopen_xz(LOG_DEBUG);
        if (r < 0)
                return r;
 
        _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
        lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, /* flags= */ 0);
        if (ret != LZMA_OK)
                return -EBADMSG;
 
        if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
                return -ENOMEM;
 
        size_t allocated = MALLOC_SIZEOF_SAFE(*buffer);
 
        s.next_in = src;
        s.avail_in = src_size;
 
        s.next_out = *buffer;
        s.avail_out = allocated;
 
        for (;;) {
                ret = sym_lzma_code(&s, LZMA_FINISH);
 
                if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
                        return -EBADMSG;
 
                if (allocated - s.avail_out >= prefix_len + 1)
                        return memcmp(*buffer, prefix, prefix_len) == 0 &&
                                ((const uint8_t*) *buffer)[prefix_len] == extra;
 
                if (ret == LZMA_STREAM_END)
                        return 0;
 
                s.avail_out += allocated;
 
                if (!(greedy_realloc(buffer, allocated * 2, 1)))
                        return -ENOMEM;
 
                allocated = MALLOC_SIZEOF_SAFE(*buffer);
                s.next_out = *(uint8_t**)buffer + allocated - s.avail_out;
        }
 
#else
        return -EPROTONOSUPPORT;
#endif
}
 
static int decompress_startswith_lz4(
                const void *src,
                uint64_t src_size,
                void **buffer,
                const void *prefix,
                size_t prefix_len,
                uint8_t extra) {
 
        /* Checks whether the decompressed blob starts with the mentioned prefix. The byte extra needs to
         * follow the prefix */
 
        assert(src);
        assert(src_size > 0);
        assert(buffer);
        assert(prefix);
 
#if HAVE_LZ4
        size_t allocated;
        int r;
 
        r = dlopen_lz4(LOG_DEBUG);
        if (r < 0)
                return r;
 
        if (src_size <= 8)
                return -EBADMSG;
 
        if (src_size - 8 > INT_MAX)
                return -EFBIG;
 
        if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
                return -ENOMEM;
        allocated = MALLOC_SIZEOF_SAFE(*buffer);
 
        r = sym_LZ4_decompress_safe_partial(
                        (char*)src + 8,
                        *buffer,
                        src_size - 8,
                        prefix_len + 1,
                        allocated);
 
        /* One lz4 < 1.8.3, we might get "failure" (r < 0), or "success" where just a part of the buffer is
         * decompressed. But if we get a smaller amount of bytes than requested, we don't know whether there
         * isn't enough data to fill the requested size or whether we just got a partial answer.
         */
        if (r < 0 || (size_t) r < prefix_len + 1) {
                size_t size;
 
                if (sym_LZ4_versionNumber() >= 10803)
                        /* We trust that the newer lz4 decompresses the number of bytes we
                         * requested if available in the compressed string. */
                        return 0;
 
                if (r > 0)
                        /* Compare what we have first, in case of mismatch we can
                         * shortcut the full comparison. */
                        if (memcmp(*buffer, prefix, r) != 0)
                                return 0;
 
                /* Before version 1.8.3, lz4 always tries to decode full a "sequence",
                 * so in pathological cases might need to decompress the full field. */
                r = decompress_blob_lz4(src, src_size, buffer, &size, 0);
                if (r < 0)
                        return r;
 
                if (size < prefix_len + 1)
                        return 0;
        }
 
        return memcmp(*buffer, prefix, prefix_len) == 0 &&
                ((const uint8_t*) *buffer)[prefix_len] == extra;
#else
        return -EPROTONOSUPPORT;
#endif
}
 
static int decompress_startswith_zstd(
                const void *src,
                uint64_t src_size,
                void **buffer,
                const void *prefix,
                size_t prefix_len,
                uint8_t extra) {
 
        assert(src);
        assert(src_size > 0);
        assert(buffer);
        assert(prefix);
 
#if HAVE_ZSTD
        int r;
 
        r = dlopen_zstd(LOG_DEBUG);
        if (r < 0)
                return r;
 
        /* ZSTD_getFrameContentSize() returns unsigned long long, which is not necessarily size_t. */
        unsigned long long size = sym_ZSTD_getFrameContentSize(src, src_size);
        if (size == ZSTD_CONTENTSIZE_ERROR)
                return -EBADMSG;
 
        /* ZSTD_CONTENTSIZE_UNKNOWN means the frame doesn't record the decompressed size, so we can't
         * shortcut here. The single decompression below only needs prefix_len + 1 bytes anyway. */
        if (size != ZSTD_CONTENTSIZE_UNKNOWN && size < prefix_len + 1)
                return 0; /* Decompressed text too short to match the prefix and extra */
 
        _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = sym_ZSTD_createDCtx();
        if (!dctx)
                return -ENOMEM;
 
        if (!(greedy_realloc(buffer, MAX(sym_ZSTD_DStreamOutSize(), prefix_len + 1), 1)))
                return -ENOMEM;
 
        ZSTD_inBuffer input = {
                .src = src,
                .size = src_size,
        };
        ZSTD_outBuffer output = {
                .dst = *buffer,
                .size = MALLOC_SIZEOF_SAFE(*buffer),
        };
        size_t k;
 
        k = sym_ZSTD_decompressStream(dctx, &output, &input);
        if (sym_ZSTD_isError(k))
                return log_debug_errno(zstd_ret_to_errno(k), "ZSTD decoder failed: %s", sym_ZSTD_getErrorName(k));
        if (output.pos < prefix_len + 1) {
                /* The output buffer is at least prefix_len + 1 bytes, so if it isn't full the decoder
                 * stopped for another reason: either the frame ended (k == 0), in which case the stream
                 * is simply too short to match the prefix, or it's still expecting input (k != 0), which
                 * means the stream was truncated mid-frame. */
                if (k != 0)
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "ZSTD stream ended unexpectedly, probably corrupted.");
                return 0; /* Decompressed text too short to match the prefix and extra */
        }
 
        return memcmp(*buffer, prefix, prefix_len) == 0 &&
                ((const uint8_t*) *buffer)[prefix_len] == extra;
#else
        return -EPROTONOSUPPORT;
#endif
}
 
static int decompress_startswith_gzip(
                const void *src,
                uint64_t src_size,
                void **buffer,
                const void *prefix,
                size_t prefix_len,
                uint8_t extra) {
 
        assert(src);
        assert(src_size > 0);
        assert(buffer);
        assert(prefix);
 
#if HAVE_ZLIB
        int r;
 
        r = dlopen_zlib(LOG_DEBUG);
        if (r < 0)
                return r;
 
        if (src_size > UINT_MAX)
                return -EFBIG;
 
        _cleanup_(inflateEnd_wrapper) z_stream s = {};
 
        r = sym_inflateInit2_(&s, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(s));
        if (r != Z_OK)
                return -EBADMSG;
 
        if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
                return -ENOMEM;
 
        size_t allocated = MALLOC_SIZEOF_SAFE(*buffer);
 
        s.next_in = (void*) src;
        s.avail_in = src_size;
 
        s.next_out = *buffer;
        s.avail_out = MIN(allocated, (size_t) UINT_MAX);
 
        for (;;) {
                r = sym_inflate(&s, Z_FINISH);
 
                if (!IN_SET(r, Z_OK, Z_STREAM_END, Z_BUF_ERROR))
                        return -EBADMSG;
 
                if (allocated - s.avail_out >= prefix_len + 1)
                        return memcmp(*buffer, prefix, prefix_len) == 0 &&
                                ((const uint8_t*) *buffer)[prefix_len] == extra;
 
                if (r == Z_STREAM_END)
                        return 0;
 
                if (s.avail_out > 0)
                        return -EBADMSG; /* truncated */
 
                size_t used = allocated - s.avail_out;
 
                if (!(greedy_realloc(buffer, allocated * 2, 1)))
                        return -ENOMEM;
 
                allocated = MALLOC_SIZEOF_SAFE(*buffer);
                s.avail_out = MIN(allocated - used, (size_t) UINT_MAX);
                s.next_out = *(uint8_t**)buffer + used;
        }
#else
        return -EPROTONOSUPPORT;
#endif
}
 
static int decompress_startswith_bzip2(
                const void *src,
                uint64_t src_size,
                void **buffer,
                const void *prefix,
                size_t prefix_len,
                uint8_t extra) {
 
        assert(src);
        assert(src_size > 0);
        assert(buffer);
        assert(prefix);
 
#if HAVE_BZIP2
        int r;
 
        r = dlopen_bzip2(LOG_DEBUG);
        if (r < 0)
                return r;
 
        if (src_size > UINT_MAX)
                return -EFBIG;
 
        _cleanup_(BZ2_bzDecompressEnd_wrapper) bz_stream s = {};
 
        if (sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0) != BZ_OK)
                return -EIO;
 
        if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
                return -ENOMEM;
 
        size_t allocated = MALLOC_SIZEOF_SAFE(*buffer);
 
        s.next_in = (char*) src;
        s.avail_in = src_size;
 
        s.next_out = *buffer;
        s.avail_out = MIN(allocated, (size_t) UINT_MAX);
 
        for (;;) {
                r = sym_BZ2_bzDecompress(&s);
 
                if (!IN_SET(r, BZ_OK, BZ_STREAM_END))
                        return -EBADMSG;
 
                if (allocated - s.avail_out >= prefix_len + 1)
                        return memcmp(*buffer, prefix, prefix_len) == 0 &&
                                ((const uint8_t*) *buffer)[prefix_len] == extra;
 
                if (r == BZ_STREAM_END)
                        return 0;
 
                if (s.avail_out > 0)
                        return -EBADMSG; /* truncated */
 
                size_t used = allocated - s.avail_out;
 
                if (!(greedy_realloc(buffer, allocated * 2, 1)))
                        return -ENOMEM;
 
                allocated = MALLOC_SIZEOF_SAFE(*buffer);
                s.avail_out = MIN(allocated - used, (size_t) UINT_MAX);
                s.next_out = (char*) *buffer + used;
        }
#else
        return -EPROTONOSUPPORT;
#endif
}
 
int decompress_startswith_journal(
                Compression compression,
                const void *src, uint64_t src_size,
                void **buffer,
                const void *prefix, size_t prefix_len,
                uint8_t extra) {
 
        switch (compression) {
        case COMPRESSION_XZ:
                return decompress_startswith_xz(src, src_size, buffer, prefix, prefix_len, extra);
        case COMPRESSION_LZ4:
                return decompress_startswith_lz4(src, src_size, buffer, prefix, prefix_len, extra);
        case COMPRESSION_ZSTD:
                return decompress_startswith_zstd(src, src_size, buffer, prefix, prefix_len, extra);
        default:
                return -EOPNOTSUPP;
        }
}
 
int decompress_startswith(
                Compression compression,
                const void *src, uint64_t src_size,
                void **buffer,
                const void *prefix, size_t prefix_len,
                uint8_t extra) {
 
        switch (compression) {
        case COMPRESSION_GZIP:
                return decompress_startswith_gzip(src, src_size, buffer, prefix, prefix_len, extra);
        case COMPRESSION_BZIP2:
                return decompress_startswith_bzip2(src, src_size, buffer, prefix, prefix_len, extra);
        default:
                return decompress_startswith_journal(compression, src, src_size, buffer, prefix, prefix_len, extra);
        }
}
 
int compress_stream(
                Compression type,
                int fdf, int fdt,
                uint64_t max_bytes,
                uint64_t *ret_uncompressed_size) {
 
        _cleanup_(compressor_freep) Compressor *c = NULL;
        _cleanup_free_ void *buf = NULL;
        _cleanup_free_ uint8_t *input = NULL;
        size_t buf_size = 0, buf_alloc = 0;
        uint64_t total_in = 0, total_out = 0;
        int r;
 
        assert(fdf >= 0);
        assert(fdt >= 0);
 
        r = compressor_new(&c, type);
        if (r < 0)
                return r;
 
        input = new(uint8_t, COMPRESS_PIPE_BUFFER_SIZE);
        if (!input)
                return -ENOMEM;
 
        for (;;) {
                size_t m = COMPRESS_PIPE_BUFFER_SIZE;
                ssize_t n;
 
                if (max_bytes != UINT64_MAX && (uint64_t) m > max_bytes)
                        m = (size_t) max_bytes;
 
                n = read(fdf, input, m);
                if (n < 0)
                        return -errno;
 
                if (n == 0) {
                        r = compressor_finish(c, &buf, &buf_size, &buf_alloc);
                        if (r < 0)
                                return r;
 
                        if (buf_size > 0) {
                                r = loop_write(fdt, buf, buf_size);
                                if (r < 0)
                                        return r;
                                total_out += buf_size;
                        }
                        break;
                }
 
                total_in += n;
                if (max_bytes != UINT64_MAX) {
                        assert(max_bytes >= (uint64_t) n);
                        max_bytes -= n;
                }
 
                r = compressor_start(c, input, n, &buf, &buf_size, &buf_alloc);
                if (r < 0)
                        return r;
 
                if (buf_size > 0) {
                        r = loop_write(fdt, buf, buf_size);
                        if (r < 0)
                                return r;
                        total_out += buf_size;
                }
        }
 
        if (ret_uncompressed_size)
                *ret_uncompressed_size = total_in;
 
        if (total_in == 0)
                log_debug("%s compression finished (no input data)", compression_to_string(type));
        else
                log_debug("%s compression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
                          compression_to_string(type), total_in, total_out, (double) total_out / total_in * 100);
 
        return 0;
}
 
static int should_sparse(int fd) {
        struct stat st;
 
        assert(fd >= 0);
 
        /* Determine whether sparse writes should be used for this fd. Sparse writes are only safe on regular
         * files, without O_APPEND (which ignores the lseek position, collapsing any holes we try to create),
         * and only when starting from a position at or beyond the current end of file (otherwise the skipped
         * ranges would overlap pre-existing data instead of being genuinely unwritten). */
 
        if (fstat(fd, &st) < 0)
                return -errno;
 
        if (!S_ISREG(st.st_mode))
                return false;
 
        int flags = fcntl(fd, F_GETFL);
        if (flags < 0)
                return -errno;
 
        if (FLAGS_SET(flags, O_APPEND))
                return false;
 
        off_t pos = lseek(fd, 0, SEEK_CUR);
        if (pos < 0)
                return -errno;
 
        return pos >= st.st_size;
}
 
/* After sparse decompression, set the file size to the current position to account for
 * trailing holes that sparse_write() created via lseek but never extended the file size for. */
static int finalize_sparse(int fd) {
        off_t pos;
 
        assert(fd >= 0);
 
        pos = lseek(fd, 0, SEEK_CUR);
        if (pos < 0)
                return -errno;
 
        if (ftruncate(fd, pos) < 0)
                return -errno;
 
        return 0;
}
 
/* Common helper for decompress_stream_*() wrappers */
 
struct decompress_stream_userdata {
        int fd;
        uint64_t max_bytes;
        uint64_t total_out;
        bool sparse;
};
 
static int decompress_stream_write_callback(const void *data, size_t size, void *userdata) {
        struct decompress_stream_userdata *u = ASSERT_PTR(userdata);
 
        if (u->max_bytes != UINT64_MAX) {
                if (u->max_bytes < size)
                        return -EFBIG;
                u->max_bytes -= size;
        }
 
        u->total_out += size;
 
        if (u->sparse) {
                /* Note: sparse_write() does not retry on EINTR and converts short writes to -EIO.
                 * This is fine here since sparse mode is only used on regular files, where short
                 * writes and EINTR are not expected in practice. */
                ssize_t k = sparse_write(u->fd, data, size, 64);
                if (k < 0)
                        return (int) k;
                return 0;
        }
 
        return loop_write(u->fd, data, size);
}
 
static int decompressor_new(Decompressor **ret, Compression type) {
        assert(ret);
 
        _cleanup_(compressor_freep) Decompressor *c = new0(Decompressor, 1);
        if (!c)
                return -ENOMEM;
 
        c->type = _COMPRESSION_INVALID;
 
        switch (type) {
        case COMPRESSION_NONE:
                break;
 
#if HAVE_XZ
        case COMPRESSION_XZ:
                if (sym_lzma_stream_decoder(&c->xz, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED) != LZMA_OK)
                        return -EIO;
                break;
#endif
 
#if HAVE_LZ4
        case COMPRESSION_LZ4: {
                size_t rc = sym_LZ4F_createDecompressionContext(&c->d_lz4, LZ4F_VERSION);
                if (sym_LZ4F_isError(rc))
                        return -ENOMEM;
 
                break;
        }
#endif
 
#if HAVE_ZSTD
        case COMPRESSION_ZSTD:
                c->d_zstd = sym_ZSTD_createDCtx();
                if (!c->d_zstd)
                        return -ENOMEM;
                break;
#endif
 
#if HAVE_ZLIB
        case COMPRESSION_GZIP:
                if (sym_inflateInit2_(&c->gzip, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(c->gzip)) != Z_OK)
                        return -EIO;
                break;
#endif
 
#if HAVE_BZIP2
        case COMPRESSION_BZIP2:
                if (sym_BZ2_bzDecompressInit(&c->bzip2, /* verbosity= */ 0, /* small= */ 0) != BZ_OK)
                        return -EIO;
                break;
#endif
 
        default:
                assert_not_reached();
        }
 
        c->type = type;
        c->encoding = false;
        *ret = TAKE_PTR(c);
        return 0;
}
 
int decompress_stream(
                Compression type,
                int fdf, int fdt,
                uint64_t max_bytes) {
 
        _cleanup_(compressor_freep) Decompressor *c = NULL;
        _cleanup_free_ uint8_t *buf = NULL;
        uint64_t total_in = 0;
        int r;
 
        assert(fdf >= 0);
        assert(fdt >= 0);
 
        if (type == COMPRESSION_NONE)
                return -EOPNOTSUPP;
 
        r = dlopen_compress(type, LOG_DEBUG);
        if (r < 0)
                return r;
 
        r = decompressor_new(&c, type);
        if (r < 0)
                return r;
 
        struct decompress_stream_userdata userdata = {
                .fd = fdt,
                .max_bytes = max_bytes,
                .sparse = should_sparse(fdt) > 0,
        };
 
        buf = new(uint8_t, COMPRESS_PIPE_BUFFER_SIZE);
        if (!buf)
                return -ENOMEM;
 
        for (;;) {
                ssize_t n;
 
                n = read(fdf, buf, COMPRESS_PIPE_BUFFER_SIZE);
                if (n < 0)
                        return -errno;
 
                total_in += n;
 
                r = decompressor_push(c, buf, n, decompress_stream_write_callback, &userdata);
                if (r < 0)
                        return r;
 
                if (n == 0)
                        break;
        }
 
        if (total_in == 0)
                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "%s decompression failed: no data read",
                                       compression_to_string(type));
 
        if (userdata.sparse) {
                r = finalize_sparse(fdt);
                if (r < 0)
                        return r;
        }
 
        log_debug("%s decompression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
                  compression_to_string(type), total_in, userdata.total_out,
                  (double) userdata.total_out / total_in * 100);
 
        return 0;
}
 
int decompress_stream_by_filename(const char *filename, int fdf, int fdt, uint64_t max_bytes) {
        Compression c = compression_from_filename(filename);
        if (c == COMPRESSION_NONE)
                return -EPROTONOSUPPORT;
 
        return decompress_stream(c, fdf, fdt, max_bytes);
}
 
/* Push-based streaming compression/decompression context API */
 
Compressor* compressor_free(Compressor *c) {
        if (!c)
                return NULL;
 
        switch (c->type) {
 
#if HAVE_XZ
        case COMPRESSION_XZ:
                sym_lzma_end(&c->xz);
                break;
#endif
 
#if HAVE_LZ4
        case COMPRESSION_LZ4:
                if (c->encoding) {
                        sym_LZ4F_freeCompressionContext(c->c_lz4);
                        c->c_lz4 = NULL;
                        c->lz4_header = mfree(c->lz4_header);
                } else {
                        sym_LZ4F_freeDecompressionContext(c->d_lz4);
                        c->d_lz4 = NULL;
                }
                break;
#endif
 
#if HAVE_ZSTD
        case COMPRESSION_ZSTD:
                if (c->encoding) {
                        sym_ZSTD_freeCCtx(c->c_zstd);
                        c->c_zstd = NULL;
                } else {
                        sym_ZSTD_freeDCtx(c->d_zstd);
                        c->d_zstd = NULL;
                }
                break;
#endif
 
#if HAVE_ZLIB
        case COMPRESSION_GZIP:
                if (c->encoding)
                        sym_deflateEnd(&c->gzip);
                else
                        sym_inflateEnd(&c->gzip);
                break;
#endif
 
#if HAVE_BZIP2
        case COMPRESSION_BZIP2:
                if (c->encoding)
                        sym_BZ2_bzCompressEnd(&c->bzip2);
                else
                        sym_BZ2_bzDecompressEnd(&c->bzip2);
                break;
#endif
 
        default:
                break;
        }
 
        return mfree(c);
}
 
Compression compressor_type(const Compressor *c) {
        return c ? c->type : _COMPRESSION_INVALID;
}
 
int decompressor_detect(Decompressor **decompressor, const void *data, size_t size) {
        int r;
 
        assert(decompressor);
 
        if (*decompressor)
                return 1;
 
        if (size < COMPRESSION_MAGIC_BYTES_MAX)
                return 0;
 
        assert(data);
 
        Compression type = compression_detect_from_magic(data);
        if (type == _COMPRESSION_INVALID)
                type = COMPRESSION_NONE;
 
        log_debug("Detected compression type: %s", compression_to_string(type));
 
        r = dlopen_compress(type, LOG_DEBUG);
        if (r < 0)
                return r;
 
        r = decompressor_new(decompressor, type);
        if (r < 0)
                return r;
 
        return 1;
}
 
int decompressor_force_off(Decompressor **decompressor) {
        assert(decompressor);
 
        *decompressor = compressor_free(*decompressor);
        return decompressor_new(decompressor, COMPRESSION_NONE);
}
 
int decompressor_push(Decompressor *c, const void *data, size_t size, DecompressorCallback callback, void *userdata) {
        int r;
 
        assert(c);
        assert(data || size == 0);
        assert(callback);
 
        /* This feeds `size` bytes of compressed data into `c`. Passing size == 0 has special meaning: it
         * signals that no further input will be provided, and the function will attempt to finalize the
         * stream, failing with -EBADMSG if the compressed data was truncated or has trailing garbage after
         * the logical end of stream. */
 
        if (c->encoding)
                return -EINVAL;
 
        if (c->finished && size == 0)
                /* The previous stream was correctly finished, and it was the last one. */
                return 0;
 
#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
        _cleanup_free_ uint8_t *buffer = NULL;
        if (c->type != COMPRESSION_NONE) {
                buffer = new(uint8_t, COMPRESS_PIPE_BUFFER_SIZE);
                if (!buffer)
                        return -ENOMEM;
        }
#endif
 
        switch (c->type) {
 
        case COMPRESSION_NONE:
                if (c->finished)
                        return -EBADMSG;