HALF ECHO LEARN / REFERENCE COLLECTION

v262.2

src/basic/compress.c

88,255 bytesGit mode 100644Download original fileLGPL-2.1-or-later

Git blob: 58fbbe3ce54f45bd88ce88d1ea5f7b4d345a0a9e

Lines 2001–2729 of 2,729

                if (size == 0) {
                        c->finished = true;
                        return 0;
                }
 
                r = callback(data, size, userdata);
                if (r < 0)
                        return r;
 
                return 0;
 
#if HAVE_XZ
        case COMPRESSION_XZ:
                if (c->finished)
                        return -EBADMSG;
 
                c->xz.next_in = data;
                c->xz.avail_in = size;
 
                for (;;) {
                        c->xz.next_out = buffer;
                        c->xz.avail_out = COMPRESS_PIPE_BUFFER_SIZE;
 
                        lzma_ret lzr = sym_lzma_code(&c->xz, size == 0 ? LZMA_FINISH : LZMA_RUN);
                        if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END))
                                return -EBADMSG;
 
                        if (c->xz.avail_out > COMPRESS_PIPE_BUFFER_SIZE)
                                return -EIO;
 
                        size_t produced = COMPRESS_PIPE_BUFFER_SIZE - c->xz.avail_out;
                        if (produced > 0) {
                                r = callback(buffer, produced, userdata);
                                if (r < 0)
                                        return r;
                        }
 
                        if (lzr == LZMA_STREAM_END) {
                                if (c->xz.avail_in > 0)
                                        return -EBADMSG; /* garbage data at the end */
 
                                c->finished = true;
                                return 0;
                        }
 
                        if (c->xz.avail_in == 0 && c->xz.avail_out > 0) {
                                if (size == 0)
                                        return -EBADMSG; /* truncated */
 
                                return 0; /* No progress possible with the current input. */
                        }
                }
#endif
 
#if HAVE_LZ4
        case COMPRESSION_LZ4: {
                const uint8_t *src = data;
                size_t src_remaining = size;
 
                c->finished = false;
 
                for (;;) {
                        size_t produced = COMPRESS_PIPE_BUFFER_SIZE;
                        size_t consumed = src_remaining;
 
                        size_t rc = sym_LZ4F_decompress(c->d_lz4, buffer, &produced, src, &consumed, NULL);
                        if (sym_LZ4F_isError(rc))
                                return -EBADMSG;
 
                        if (consumed > src_remaining ||
                            produced > COMPRESS_PIPE_BUFFER_SIZE)
                                return -EIO;
 
                        if (consumed > 0) {
                                assert(src);
                                src += consumed;
                                src_remaining -= consumed;
                        }
 
                        if (produced > 0) {
                                r = callback(buffer, produced, userdata);
                                if (r < 0)
                                        return r;
                        }
 
                        if (src_remaining == 0) {
                                if (rc == 0) {
                                        /* The current stream has finished. */
                                        c->finished = true;
                                        return 0;
                                }
 
                                if (produced == COMPRESS_PIPE_BUFFER_SIZE)
                                        /* Remaining data exist in the library buffer. We need to read them. */
                                        continue;
 
                                if (size == 0)
                                        return -EBADMSG; /* truncated */
 
                                return 0; /* No progress possible with the current input. */
                        }
                }
        }
#endif
 
#if HAVE_ZSTD
        case COMPRESSION_ZSTD: {
                ZSTD_inBuffer input = {
                        .src =  (void*) data,
                        .size = size,
                };
 
                c->finished = false;
 
                for (;;) {
                        ZSTD_outBuffer output = {
                                .dst = buffer,
                                .size = COMPRESS_PIPE_BUFFER_SIZE,
                        };
 
                        size_t res = sym_ZSTD_decompressStream(c->d_zstd, &output, &input);
                        if (sym_ZSTD_isError(res))
                                return -EBADMSG;
 
                        if (input.pos > input.size ||
                            output.pos > output.size)
                                return -EIO;
 
                        if (output.pos > 0) {
                                r = callback(output.dst, output.pos, userdata);
                                if (r < 0)
                                        return r;
                        }
 
                        if (input.pos == input.size) {
                                if (res == 0) {
                                        /* The current stream has finished. */
                                        c->finished = true;
                                        return 0;
                                }
 
                                if (output.pos == COMPRESS_PIPE_BUFFER_SIZE)
                                        /* Remaining data exist in the library buffer. We need to read them. */
                                        continue;
 
                                if (size == 0)
                                        return -EBADMSG; /* truncated */
 
                                return 0; /* No progress possible with the current input. */
                        }
                }
        }
#endif
 
#if HAVE_ZLIB
        case COMPRESSION_GZIP:
                if (size > UINT_MAX)
                        return -EFBIG;
 
                c->gzip.next_in = (void*) data;
                c->gzip.avail_in = size;
 
                for (;;) {
                        if (c->finished) {
                                if (sym_inflateReset(&c->gzip) != Z_OK)
                                        return -EIO;
 
                                c->finished = false;
                        }
 
                        c->gzip.next_out = buffer;
                        c->gzip.avail_out = COMPRESS_PIPE_BUFFER_SIZE;
 
                        /* Z_BUF_ERROR is not fatal if all input is used up: it just means no progress was
                         * possible, which happens if the previous iteration filled the output buffer exactly
                         * while also consuming all input. That case is handled by the avail_in check
                         * below. If there's still input left however, Z_BUF_ERROR means we are stuck, hence
                         * refuse it, so that we don't loop forever. */
                        int zr = sym_inflate(&c->gzip, Z_NO_FLUSH);
                        if (!IN_SET(zr, Z_OK, Z_STREAM_END) &&
                            !(zr == Z_BUF_ERROR && c->gzip.avail_in == 0))
                                return -EBADMSG;
 
                        if (c->gzip.avail_out > COMPRESS_PIPE_BUFFER_SIZE)
                                return -EIO;
 
                        size_t produced = COMPRESS_PIPE_BUFFER_SIZE - c->gzip.avail_out;
                        if (produced > 0) {
                                r = callback(buffer, produced, userdata);
                                if (r < 0)
                                        return r;
                        }
 
                        if (zr == Z_STREAM_END) {
                                c->finished = true;
 
                                if (c->gzip.avail_in > 0)
                                        continue; /* New stream may be pending. */
 
                                return 0;
                        }
 
                        if (c->gzip.avail_in == 0 && c->gzip.avail_out > 0) {
                                if (size == 0)
                                        return -EBADMSG; /* truncated */
 
                                return 0; /* No progress possible with the current input. */
                        }
                }
#endif
 
#if HAVE_BZIP2
        case COMPRESSION_BZIP2:
                if (size > UINT_MAX)
                        return -EFBIG;
 
                c->bzip2.next_in = (char*) data;
                c->bzip2.avail_in = size;
 
                for (;;) {
                        if (c->finished) {
                                sym_BZ2_bzDecompressEnd(&c->bzip2);
                                if (sym_BZ2_bzDecompressInit(&c->bzip2, /* verbosity= */ 0, /* small= */ 0) != BZ_OK)
                                        return -EIO;
 
                                c->finished = false;
                        }
 
                        c->bzip2.next_out = (char*) buffer;
                        c->bzip2.avail_out = COMPRESS_PIPE_BUFFER_SIZE;
 
                        int bzr = sym_BZ2_bzDecompress(&c->bzip2);
                        if (!IN_SET(bzr, BZ_OK, BZ_STREAM_END))
                                return -EBADMSG;
 
                        if (c->bzip2.avail_out > COMPRESS_PIPE_BUFFER_SIZE)
                                return -EIO;
 
                        size_t produced = COMPRESS_PIPE_BUFFER_SIZE - c->bzip2.avail_out;
                        if (produced > 0) {
                                r = callback(buffer, produced, userdata);
                                if (r < 0)
                                        return r;
                        }
 
                        if (bzr == BZ_STREAM_END) {
                                c->finished = true;
 
                                if (c->bzip2.avail_in > 0)
                                        continue; /* New stream may be pending. */
 
                                return 0;
                        }
 
                        if (c->bzip2.avail_in == 0 && c->bzip2.avail_out > 0) {
                                if (size == 0)
                                        return -EBADMSG; /* truncated */
 
                                return 0; /* No progress possible with the current input. */
                        }
                }
#endif
 
        default:
                assert_not_reached();
        }
}
 
int compressor_new(Compressor **ret, Compression type) {
        int r;
 
        assert(ret);
 
        r = dlopen_compress(type, LOG_DEBUG);
        if (r < 0)
                return r;
 
        _cleanup_(compressor_freep) Compressor *c = new0(Compressor, 1);
        if (!c)
                return -ENOMEM;
 
        c->type = _COMPRESSION_INVALID;
        /* Set encoding early so that compressor_freep calls the correct cleanup (compression vs
         * decompression) if any operation in the switch fails after setting c->type. This is safe
         * because _COMPRESSION_INVALID hits the default: break case regardless of the encoding flag. */
        c->encoding = true;
 
        switch (type) {
 
#if HAVE_XZ
        case COMPRESSION_XZ: {
                lzma_ret xzr = sym_lzma_easy_encoder(&c->xz, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
                if (xzr != LZMA_OK)
                        return -EIO;
 
                c->type = COMPRESSION_XZ;
                break;
        }
#endif
 
#if HAVE_LZ4
        case COMPRESSION_LZ4: {
                size_t rc = sym_LZ4F_createCompressionContext(&c->c_lz4, LZ4F_VERSION);
                if (sym_LZ4F_isError(rc))
                        return -ENOMEM;
 
                c->type = COMPRESSION_LZ4;
 
                /* Generate the frame header and stash it for the first compressor_start call */
                size_t header_bound = sym_LZ4F_compressBound(0, &lz4_preferences);
                c->lz4_header = malloc(header_bound);
                if (!c->lz4_header)
                        return -ENOMEM;
 
                c->lz4_header_size = sym_LZ4F_compressBegin(c->c_lz4, c->lz4_header, header_bound, &lz4_preferences);
                if (sym_LZ4F_isError(c->lz4_header_size))
                        return -EINVAL;
 
                break;
        }
#endif
 
#if HAVE_ZSTD
        case COMPRESSION_ZSTD:
                c->c_zstd = sym_ZSTD_createCCtx();
                if (!c->c_zstd)
                        return -ENOMEM;
 
                c->type = COMPRESSION_ZSTD;
 
                size_t z = sym_ZSTD_CCtx_setParameter(c->c_zstd, ZSTD_c_compressionLevel, ZSTD_CLEVEL_DEFAULT);
                if (sym_ZSTD_isError(z))
                        return -EIO;
 
                z = sym_ZSTD_CCtx_setParameter(c->c_zstd, ZSTD_c_checksumFlag, /* enable= */ 1);
                if (sym_ZSTD_isError(z))
                        log_debug("Failed to enable ZSTD checksum, ignoring: %s", sym_ZSTD_getErrorName(z));
 
                break;
#endif
 
#if HAVE_ZLIB
        case COMPRESSION_GZIP:
                r = sym_deflateInit2_(&c->gzip,
                                      Z_DEFAULT_COMPRESSION,
                                      /* method= */ Z_DEFLATED,
                                      /* windowBits= */ ZLIB_WBITS_GZIP,
                                      /* memLevel= */ 8,
                                      /* strategy= */ Z_DEFAULT_STRATEGY,
                                      ZLIB_VERSION, (int) sizeof(c->gzip));
                if (r != Z_OK)
                        return -EIO;
 
                c->type = COMPRESSION_GZIP;
                break;
#endif
 
#if HAVE_BZIP2
        case COMPRESSION_BZIP2:
                r = sym_BZ2_bzCompressInit(&c->bzip2, /* blockSize100k= */ 9, /* verbosity= */ 0, /* workFactor= */ 0);
                if (r != BZ_OK)
                        return -EIO;
 
                c->type = COMPRESSION_BZIP2;
                break;
#endif
 
        case COMPRESSION_NONE:
                c->type = COMPRESSION_NONE;
                break;
 
        default:
                return -EOPNOTSUPP;
        }
 
        *ret = TAKE_PTR(c);
        return 0;
}
 
#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
static int enlarge_buffer(void **buffer, size_t *buffer_size, size_t *buffer_allocated, size_t need) {
        assert(buffer);
        assert(buffer_size);
        assert(buffer_allocated);
 
        need = MAX3(need, *buffer_size + 1, (size_t) COMPRESS_PIPE_BUFFER_SIZE);
        if (*buffer_allocated >= need)
                return 0;
 
        if (!greedy_realloc(buffer, need, 1))
                return -ENOMEM;
 
        *buffer_allocated = MALLOC_SIZEOF_SAFE(*buffer);
        return 1;
}
#endif
 
int compressor_start(
                Compressor *c,
                const void *data,
                size_t size,
                void **buffer,
                size_t *buffer_size,
                size_t *buffer_allocated) {
 
#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
        int r;
#endif
 
        assert(c);
        assert(buffer);
        assert(buffer_size);
        assert(buffer_allocated);
 
        if (!c->encoding)
                return -EINVAL;
 
        if (size == 0)
                return 0;
 
        assert(data);
 
        *buffer_size = 0;
 
        switch (c->type) {
 
#if HAVE_XZ
        case COMPRESSION_XZ:
 
                c->xz.next_in = data;
                c->xz.avail_in = size;
 
                while (c->xz.avail_in > 0) {
                        lzma_ret lzr;
 
                        r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
                        if (r < 0)
                                return r;
 
                        c->xz.next_out = (uint8_t*) *buffer + *buffer_size;
                        c->xz.avail_out = *buffer_allocated - *buffer_size;
 
                        lzr = sym_lzma_code(&c->xz, LZMA_RUN);
                        if (lzr != LZMA_OK)
                                return -EIO;
 
                        *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out;
                }
 
                break;
#endif
 
#if HAVE_LZ4
        case COMPRESSION_LZ4: {
                /* Prepend any stashed frame header from compressor_new */
                if (c->lz4_header_size > 0) {
                        r = enlarge_buffer(buffer, buffer_size, buffer_allocated, c->lz4_header_size);
                        if (r < 0)
                                return r;
 
                        memcpy(*buffer, c->lz4_header, c->lz4_header_size);
                        *buffer_size = c->lz4_header_size;
                        c->lz4_header = mfree(c->lz4_header);
                        c->lz4_header_size = 0;
                }
 
                size_t bound = sym_LZ4F_compressBound(size, &lz4_preferences);
                r = enlarge_buffer(buffer, buffer_size, buffer_allocated, *buffer_size + bound);
                if (r < 0)
                        return r;
 
                size_t n = sym_LZ4F_compressUpdate(c->c_lz4,
                                                   (uint8_t*) *buffer + *buffer_size,
                                                   *buffer_allocated - *buffer_size,
                                                   data, size, NULL);
                if (sym_LZ4F_isError(n))
                        return -EIO;
 
                *buffer_size += n;
                break;
        }
#endif
 
#if HAVE_ZSTD
        case COMPRESSION_ZSTD: {
                ZSTD_inBuffer input = {
                        .src = data,
                        .size = size,
                };
 
                while (input.pos < input.size) {
                        r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
                        if (r < 0)
                                return r;
 
                        ZSTD_outBuffer output = {
                                .dst = ((uint8_t *) *buffer + *buffer_size),
                                .size = *buffer_allocated - *buffer_size,
                        };
 
                        size_t res = sym_ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_continue);
                        if (sym_ZSTD_isError(res))
                                return -EIO;
 
                        *buffer_size += output.pos;
                }
 
                break;
        }
#endif
 
#if HAVE_ZLIB
        case COMPRESSION_GZIP:
                if (size > UINT_MAX)
                        return -EFBIG;
 
                c->gzip.next_in = (void*) data;
                c->gzip.avail_in = size;
 
                while (c->gzip.avail_in > 0) {
                        r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
                        if (r < 0)
                                return r;
 
                        size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
                        c->gzip.next_out = (uint8_t*) *buffer + *buffer_size;
                        c->gzip.avail_out = avail;
 
                        r = sym_deflate(&c->gzip, Z_NO_FLUSH);
                        if (r != Z_OK)
                                return -EIO;
 
                        *buffer_size += avail - c->gzip.avail_out;
                }
 
                break;
#endif
 
#if HAVE_BZIP2
        case COMPRESSION_BZIP2:
                if (size > UINT_MAX)
                        return -EFBIG;
 
                c->bzip2.next_in = (void*) data;
                c->bzip2.avail_in = size;
 
                while (c->bzip2.avail_in > 0) {
                        r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
                        if (r < 0)
                                return r;
 
                        size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
                        c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size);
                        c->bzip2.avail_out = avail;
 
                        r = sym_BZ2_bzCompress(&c->bzip2, BZ_RUN);
                        if (r != BZ_RUN_OK)
                                return -EIO;
 
                        *buffer_size += avail - c->bzip2.avail_out;
                }
 
                break;
#endif
 
        case COMPRESSION_NONE:
 
                if (*buffer_allocated < size) {
                        void *p;
 
                        p = realloc(*buffer, size);
                        if (!p)
                                return -ENOMEM;
 
                        *buffer = p;
                        *buffer_allocated = size;
                }
 
                memcpy(*buffer, data, size);
                *buffer_size = size;
                break;
 
        default:
                return -EOPNOTSUPP;
        }
 
        return 0;
}
 
int compressor_finish(Compressor *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated) {
#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
        int r;
#endif
 
        assert(c);
        assert(buffer);
        assert(buffer_size);
        assert(buffer_allocated);
 
        if (!c->encoding)
                return -EINVAL;
 
        *buffer_size = 0;
 
        switch (c->type) {
 
#if HAVE_XZ
        case COMPRESSION_XZ: {
                lzma_ret lzr;
 
                c->xz.avail_in = 0;
 
                do {
                        r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
                        if (r < 0)
                                return r;
 
                        c->xz.next_out = (uint8_t*) *buffer + *buffer_size;
                        c->xz.avail_out = *buffer_allocated - *buffer_size;
 
                        lzr = sym_lzma_code(&c->xz, LZMA_FINISH);
                        if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END))
                                return -EIO;
 
                        *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out;
                } while (lzr != LZMA_STREAM_END);
 
                break;
        }
#endif
 
#if HAVE_LZ4
        case COMPRESSION_LZ4: {
                size_t bound = sym_LZ4F_compressBound(0, &lz4_preferences);
                r = enlarge_buffer(buffer, buffer_size, buffer_allocated, bound);
                if (r < 0)
                        return r;
 
                size_t n = sym_LZ4F_compressEnd(c->c_lz4, *buffer, *buffer_allocated, NULL);
                if (sym_LZ4F_isError(n))
                        return -EIO;
 
                *buffer_size = n;
                break;
        }
#endif
 
#if HAVE_ZSTD
        case COMPRESSION_ZSTD: {
                ZSTD_inBuffer input = {};
                size_t res;
 
                do {
                        r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
                        if (r < 0)
                                return r;
 
                        ZSTD_outBuffer output = {
                                .dst = ((uint8_t *) *buffer + *buffer_size),
                                .size = *buffer_allocated - *buffer_size,
                        };
 
                        res = sym_ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_end);
                        if (sym_ZSTD_isError(res))
                                return -EIO;
 
                        *buffer_size += output.pos;
                } while (res != 0);
 
                break;
        }
#endif
 
#if HAVE_ZLIB
        case COMPRESSION_GZIP:
                c->gzip.avail_in = 0;
 
                do {
                        r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
                        if (r < 0)
                                return r;
 
                        size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
                        c->gzip.next_out = (uint8_t*) *buffer + *buffer_size;
                        c->gzip.avail_out = avail;
 
                        r = sym_deflate(&c->gzip, Z_FINISH);
                        if (!IN_SET(r, Z_OK, Z_STREAM_END))
                                return -EIO;
 
                        *buffer_size += avail - c->gzip.avail_out;
                } while (r != Z_STREAM_END);
 
                break;
#endif
 
#if HAVE_BZIP2
        case COMPRESSION_BZIP2:
                c->bzip2.avail_in = 0;
 
                do {
                        r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
                        if (r < 0)
                                return r;
 
                        size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
                        c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size);
                        c->bzip2.avail_out = avail;
 
                        r = sym_BZ2_bzCompress(&c->bzip2, BZ_FINISH);
                        if (!IN_SET(r, BZ_FINISH_OK, BZ_STREAM_END))
                                return -EIO;
 
                        *buffer_size += avail - c->bzip2.avail_out;
                } while (r != BZ_STREAM_END);
 
                break;
#endif
 
        case COMPRESSION_NONE:
                break;
 
        default:
                return -EOPNOTSUPP;
        }
 
        return 0;
}