HALF ECHO LEARN / REFERENCE COLLECTION
v262.2src/basic/compress.c
Git blob: 58fbbe3ce54f45bd88ce88d1ea5f7b4d345a0a9e
Lines 2001–2729 of 2,729
if (size == 0) {
c->finished = true;
return 0;
}
r = callback(data, size, userdata);
if (r < 0)
return r;
return 0;
#if HAVE_XZ
case COMPRESSION_XZ:
if (c->finished)
return -EBADMSG;
c->xz.next_in = data;
c->xz.avail_in = size;
for (;;) {
c->xz.next_out = buffer;
c->xz.avail_out = COMPRESS_PIPE_BUFFER_SIZE;
lzma_ret lzr = sym_lzma_code(&c->xz, size == 0 ? LZMA_FINISH : LZMA_RUN);
if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END))
return -EBADMSG;
if (c->xz.avail_out > COMPRESS_PIPE_BUFFER_SIZE)
return -EIO;
size_t produced = COMPRESS_PIPE_BUFFER_SIZE - c->xz.avail_out;
if (produced > 0) {
r = callback(buffer, produced, userdata);
if (r < 0)
return r;
}
if (lzr == LZMA_STREAM_END) {
if (c->xz.avail_in > 0)
return -EBADMSG; /* garbage data at the end */
c->finished = true;
return 0;
}
if (c->xz.avail_in == 0 && c->xz.avail_out > 0) {
if (size == 0)
return -EBADMSG; /* truncated */
return 0; /* No progress possible with the current input. */
}
}
#endif
#if HAVE_LZ4
case COMPRESSION_LZ4: {
const uint8_t *src = data;
size_t src_remaining = size;
c->finished = false;
for (;;) {
size_t produced = COMPRESS_PIPE_BUFFER_SIZE;
size_t consumed = src_remaining;
size_t rc = sym_LZ4F_decompress(c->d_lz4, buffer, &produced, src, &consumed, NULL);
if (sym_LZ4F_isError(rc))
return -EBADMSG;
if (consumed > src_remaining ||
produced > COMPRESS_PIPE_BUFFER_SIZE)
return -EIO;
if (consumed > 0) {
assert(src);
src += consumed;
src_remaining -= consumed;
}
if (produced > 0) {
r = callback(buffer, produced, userdata);
if (r < 0)
return r;
}
if (src_remaining == 0) {
if (rc == 0) {
/* The current stream has finished. */
c->finished = true;
return 0;
}
if (produced == COMPRESS_PIPE_BUFFER_SIZE)
/* Remaining data exist in the library buffer. We need to read them. */
continue;
if (size == 0)
return -EBADMSG; /* truncated */
return 0; /* No progress possible with the current input. */
}
}
}
#endif
#if HAVE_ZSTD
case COMPRESSION_ZSTD: {
ZSTD_inBuffer input = {
.src = (void*) data,
.size = size,
};
c->finished = false;
for (;;) {
ZSTD_outBuffer output = {
.dst = buffer,
.size = COMPRESS_PIPE_BUFFER_SIZE,
};
size_t res = sym_ZSTD_decompressStream(c->d_zstd, &output, &input);
if (sym_ZSTD_isError(res))
return -EBADMSG;
if (input.pos > input.size ||
output.pos > output.size)
return -EIO;
if (output.pos > 0) {
r = callback(output.dst, output.pos, userdata);
if (r < 0)
return r;
}
if (input.pos == input.size) {
if (res == 0) {
/* The current stream has finished. */
c->finished = true;
return 0;
}
if (output.pos == COMPRESS_PIPE_BUFFER_SIZE)
/* Remaining data exist in the library buffer. We need to read them. */
continue;
if (size == 0)
return -EBADMSG; /* truncated */
return 0; /* No progress possible with the current input. */
}
}
}
#endif
#if HAVE_ZLIB
case COMPRESSION_GZIP:
if (size > UINT_MAX)
return -EFBIG;
c->gzip.next_in = (void*) data;
c->gzip.avail_in = size;
for (;;) {
if (c->finished) {
if (sym_inflateReset(&c->gzip) != Z_OK)
return -EIO;
c->finished = false;
}
c->gzip.next_out = buffer;
c->gzip.avail_out = COMPRESS_PIPE_BUFFER_SIZE;
/* Z_BUF_ERROR is not fatal if all input is used up: it just means no progress was
* possible, which happens if the previous iteration filled the output buffer exactly
* while also consuming all input. That case is handled by the avail_in check
* below. If there's still input left however, Z_BUF_ERROR means we are stuck, hence
* refuse it, so that we don't loop forever. */
int zr = sym_inflate(&c->gzip, Z_NO_FLUSH);
if (!IN_SET(zr, Z_OK, Z_STREAM_END) &&
!(zr == Z_BUF_ERROR && c->gzip.avail_in == 0))
return -EBADMSG;
if (c->gzip.avail_out > COMPRESS_PIPE_BUFFER_SIZE)
return -EIO;
size_t produced = COMPRESS_PIPE_BUFFER_SIZE - c->gzip.avail_out;
if (produced > 0) {
r = callback(buffer, produced, userdata);
if (r < 0)
return r;
}
if (zr == Z_STREAM_END) {
c->finished = true;
if (c->gzip.avail_in > 0)
continue; /* New stream may be pending. */
return 0;
}
if (c->gzip.avail_in == 0 && c->gzip.avail_out > 0) {
if (size == 0)
return -EBADMSG; /* truncated */
return 0; /* No progress possible with the current input. */
}
}
#endif
#if HAVE_BZIP2
case COMPRESSION_BZIP2:
if (size > UINT_MAX)
return -EFBIG;
c->bzip2.next_in = (char*) data;
c->bzip2.avail_in = size;
for (;;) {
if (c->finished) {
sym_BZ2_bzDecompressEnd(&c->bzip2);
if (sym_BZ2_bzDecompressInit(&c->bzip2, /* verbosity= */ 0, /* small= */ 0) != BZ_OK)
return -EIO;
c->finished = false;
}
c->bzip2.next_out = (char*) buffer;
c->bzip2.avail_out = COMPRESS_PIPE_BUFFER_SIZE;
int bzr = sym_BZ2_bzDecompress(&c->bzip2);
if (!IN_SET(bzr, BZ_OK, BZ_STREAM_END))
return -EBADMSG;
if (c->bzip2.avail_out > COMPRESS_PIPE_BUFFER_SIZE)
return -EIO;
size_t produced = COMPRESS_PIPE_BUFFER_SIZE - c->bzip2.avail_out;
if (produced > 0) {
r = callback(buffer, produced, userdata);
if (r < 0)
return r;
}
if (bzr == BZ_STREAM_END) {
c->finished = true;
if (c->bzip2.avail_in > 0)
continue; /* New stream may be pending. */
return 0;
}
if (c->bzip2.avail_in == 0 && c->bzip2.avail_out > 0) {
if (size == 0)
return -EBADMSG; /* truncated */
return 0; /* No progress possible with the current input. */
}
}
#endif
default:
assert_not_reached();
}
}
int compressor_new(Compressor **ret, Compression type) {
int r;
assert(ret);
r = dlopen_compress(type, LOG_DEBUG);
if (r < 0)
return r;
_cleanup_(compressor_freep) Compressor *c = new0(Compressor, 1);
if (!c)
return -ENOMEM;
c->type = _COMPRESSION_INVALID;
/* Set encoding early so that compressor_freep calls the correct cleanup (compression vs
* decompression) if any operation in the switch fails after setting c->type. This is safe
* because _COMPRESSION_INVALID hits the default: break case regardless of the encoding flag. */
c->encoding = true;
switch (type) {
#if HAVE_XZ
case COMPRESSION_XZ: {
lzma_ret xzr = sym_lzma_easy_encoder(&c->xz, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
if (xzr != LZMA_OK)
return -EIO;
c->type = COMPRESSION_XZ;
break;
}
#endif
#if HAVE_LZ4
case COMPRESSION_LZ4: {
size_t rc = sym_LZ4F_createCompressionContext(&c->c_lz4, LZ4F_VERSION);
if (sym_LZ4F_isError(rc))
return -ENOMEM;
c->type = COMPRESSION_LZ4;
/* Generate the frame header and stash it for the first compressor_start call */
size_t header_bound = sym_LZ4F_compressBound(0, &lz4_preferences);
c->lz4_header = malloc(header_bound);
if (!c->lz4_header)
return -ENOMEM;
c->lz4_header_size = sym_LZ4F_compressBegin(c->c_lz4, c->lz4_header, header_bound, &lz4_preferences);
if (sym_LZ4F_isError(c->lz4_header_size))
return -EINVAL;
break;
}
#endif
#if HAVE_ZSTD
case COMPRESSION_ZSTD:
c->c_zstd = sym_ZSTD_createCCtx();
if (!c->c_zstd)
return -ENOMEM;
c->type = COMPRESSION_ZSTD;
size_t z = sym_ZSTD_CCtx_setParameter(c->c_zstd, ZSTD_c_compressionLevel, ZSTD_CLEVEL_DEFAULT);
if (sym_ZSTD_isError(z))
return -EIO;
z = sym_ZSTD_CCtx_setParameter(c->c_zstd, ZSTD_c_checksumFlag, /* enable= */ 1);
if (sym_ZSTD_isError(z))
log_debug("Failed to enable ZSTD checksum, ignoring: %s", sym_ZSTD_getErrorName(z));
break;
#endif
#if HAVE_ZLIB
case COMPRESSION_GZIP:
r = sym_deflateInit2_(&c->gzip,
Z_DEFAULT_COMPRESSION,
/* method= */ Z_DEFLATED,
/* windowBits= */ ZLIB_WBITS_GZIP,
/* memLevel= */ 8,
/* strategy= */ Z_DEFAULT_STRATEGY,
ZLIB_VERSION, (int) sizeof(c->gzip));
if (r != Z_OK)
return -EIO;
c->type = COMPRESSION_GZIP;
break;
#endif
#if HAVE_BZIP2
case COMPRESSION_BZIP2:
r = sym_BZ2_bzCompressInit(&c->bzip2, /* blockSize100k= */ 9, /* verbosity= */ 0, /* workFactor= */ 0);
if (r != BZ_OK)
return -EIO;
c->type = COMPRESSION_BZIP2;
break;
#endif
case COMPRESSION_NONE:
c->type = COMPRESSION_NONE;
break;
default:
return -EOPNOTSUPP;
}
*ret = TAKE_PTR(c);
return 0;
}
#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
static int enlarge_buffer(void **buffer, size_t *buffer_size, size_t *buffer_allocated, size_t need) {
assert(buffer);
assert(buffer_size);
assert(buffer_allocated);
need = MAX3(need, *buffer_size + 1, (size_t) COMPRESS_PIPE_BUFFER_SIZE);
if (*buffer_allocated >= need)
return 0;
if (!greedy_realloc(buffer, need, 1))
return -ENOMEM;
*buffer_allocated = MALLOC_SIZEOF_SAFE(*buffer);
return 1;
}
#endif
int compressor_start(
Compressor *c,
const void *data,
size_t size,
void **buffer,
size_t *buffer_size,
size_t *buffer_allocated) {
#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
int r;
#endif
assert(c);
assert(buffer);
assert(buffer_size);
assert(buffer_allocated);
if (!c->encoding)
return -EINVAL;
if (size == 0)
return 0;
assert(data);
*buffer_size = 0;
switch (c->type) {
#if HAVE_XZ
case COMPRESSION_XZ:
c->xz.next_in = data;
c->xz.avail_in = size;
while (c->xz.avail_in > 0) {
lzma_ret lzr;
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
if (r < 0)
return r;
c->xz.next_out = (uint8_t*) *buffer + *buffer_size;
c->xz.avail_out = *buffer_allocated - *buffer_size;
lzr = sym_lzma_code(&c->xz, LZMA_RUN);
if (lzr != LZMA_OK)
return -EIO;
*buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out;
}
break;
#endif
#if HAVE_LZ4
case COMPRESSION_LZ4: {
/* Prepend any stashed frame header from compressor_new */
if (c->lz4_header_size > 0) {
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, c->lz4_header_size);
if (r < 0)
return r;
memcpy(*buffer, c->lz4_header, c->lz4_header_size);
*buffer_size = c->lz4_header_size;
c->lz4_header = mfree(c->lz4_header);
c->lz4_header_size = 0;
}
size_t bound = sym_LZ4F_compressBound(size, &lz4_preferences);
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, *buffer_size + bound);
if (r < 0)
return r;
size_t n = sym_LZ4F_compressUpdate(c->c_lz4,
(uint8_t*) *buffer + *buffer_size,
*buffer_allocated - *buffer_size,
data, size, NULL);
if (sym_LZ4F_isError(n))
return -EIO;
*buffer_size += n;
break;
}
#endif
#if HAVE_ZSTD
case COMPRESSION_ZSTD: {
ZSTD_inBuffer input = {
.src = data,
.size = size,
};
while (input.pos < input.size) {
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
if (r < 0)
return r;
ZSTD_outBuffer output = {
.dst = ((uint8_t *) *buffer + *buffer_size),
.size = *buffer_allocated - *buffer_size,
};
size_t res = sym_ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_continue);
if (sym_ZSTD_isError(res))
return -EIO;
*buffer_size += output.pos;
}
break;
}
#endif
#if HAVE_ZLIB
case COMPRESSION_GZIP:
if (size > UINT_MAX)
return -EFBIG;
c->gzip.next_in = (void*) data;
c->gzip.avail_in = size;
while (c->gzip.avail_in > 0) {
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
if (r < 0)
return r;
size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
c->gzip.next_out = (uint8_t*) *buffer + *buffer_size;
c->gzip.avail_out = avail;
r = sym_deflate(&c->gzip, Z_NO_FLUSH);
if (r != Z_OK)
return -EIO;
*buffer_size += avail - c->gzip.avail_out;
}
break;
#endif
#if HAVE_BZIP2
case COMPRESSION_BZIP2:
if (size > UINT_MAX)
return -EFBIG;
c->bzip2.next_in = (void*) data;
c->bzip2.avail_in = size;
while (c->bzip2.avail_in > 0) {
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
if (r < 0)
return r;
size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size);
c->bzip2.avail_out = avail;
r = sym_BZ2_bzCompress(&c->bzip2, BZ_RUN);
if (r != BZ_RUN_OK)
return -EIO;
*buffer_size += avail - c->bzip2.avail_out;
}
break;
#endif
case COMPRESSION_NONE:
if (*buffer_allocated < size) {
void *p;
p = realloc(*buffer, size);
if (!p)
return -ENOMEM;
*buffer = p;
*buffer_allocated = size;
}
memcpy(*buffer, data, size);
*buffer_size = size;
break;
default:
return -EOPNOTSUPP;
}
return 0;
}
int compressor_finish(Compressor *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated) {
#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
int r;
#endif
assert(c);
assert(buffer);
assert(buffer_size);
assert(buffer_allocated);
if (!c->encoding)
return -EINVAL;
*buffer_size = 0;
switch (c->type) {
#if HAVE_XZ
case COMPRESSION_XZ: {
lzma_ret lzr;
c->xz.avail_in = 0;
do {
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
if (r < 0)
return r;
c->xz.next_out = (uint8_t*) *buffer + *buffer_size;
c->xz.avail_out = *buffer_allocated - *buffer_size;
lzr = sym_lzma_code(&c->xz, LZMA_FINISH);
if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END))
return -EIO;
*buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out;
} while (lzr != LZMA_STREAM_END);
break;
}
#endif
#if HAVE_LZ4
case COMPRESSION_LZ4: {
size_t bound = sym_LZ4F_compressBound(0, &lz4_preferences);
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, bound);
if (r < 0)
return r;
size_t n = sym_LZ4F_compressEnd(c->c_lz4, *buffer, *buffer_allocated, NULL);
if (sym_LZ4F_isError(n))
return -EIO;
*buffer_size = n;
break;
}
#endif
#if HAVE_ZSTD
case COMPRESSION_ZSTD: {
ZSTD_inBuffer input = {};
size_t res;
do {
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
if (r < 0)
return r;
ZSTD_outBuffer output = {
.dst = ((uint8_t *) *buffer + *buffer_size),
.size = *buffer_allocated - *buffer_size,
};
res = sym_ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_end);
if (sym_ZSTD_isError(res))
return -EIO;
*buffer_size += output.pos;
} while (res != 0);
break;
}
#endif
#if HAVE_ZLIB
case COMPRESSION_GZIP:
c->gzip.avail_in = 0;
do {
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
if (r < 0)
return r;
size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
c->gzip.next_out = (uint8_t*) *buffer + *buffer_size;
c->gzip.avail_out = avail;
r = sym_deflate(&c->gzip, Z_FINISH);
if (!IN_SET(r, Z_OK, Z_STREAM_END))
return -EIO;
*buffer_size += avail - c->gzip.avail_out;
} while (r != Z_STREAM_END);
break;
#endif
#if HAVE_BZIP2
case COMPRESSION_BZIP2:
c->bzip2.avail_in = 0;
do {
r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
if (r < 0)
return r;
size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size);
c->bzip2.avail_out = avail;
r = sym_BZ2_bzCompress(&c->bzip2, BZ_FINISH);
if (!IN_SET(r, BZ_FINISH_OK, BZ_STREAM_END))
return -EIO;
*buffer_size += avail - c->bzip2.avail_out;
} while (r != BZ_STREAM_END);
break;
#endif
case COMPRESSION_NONE:
break;
default:
return -EOPNOTSUPP;
}
return 0;
}