HALF ECHO LEARN / REFERENCE COLLECTION

v262.2

src/libsystemd/sd-journal/journal-file.c

178,971 bytesGit mode 100644Download original fileLGPL-2.1-or-later

Git blob: bed0c812a6957f446f48f2101888a46dca5a6bca

Lines 1–1000 of 4,786

/* SPDX-License-Identifier: LGPL-2.1-or-later */
 
#include <fcntl.h>
#include <linux/magic.h>
#include <pthread.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <sys/statvfs.h>
#include <sys/uio.h>
#include <threads.h>
#include <unistd.h>
 
#include "sd-event.h"
 
#include "alloc-util.h"
#include "chattr-util.h"
#include "compress.h"
#include "env-util.h"
#include "fd-util.h"
#include "format-util.h"
#include "fs-util.h"
#include "hashmap.h"
#include "id128-util.h"
#include "io-util.h"
#include "journal-authenticate-internal.h"
#include "journal-def.h"
#include "journal-file.h"
#include "journal-internal.h"
#include "log.h"
#include "log-ratelimit.h"
#include "lookup3.h"
#include "memory-util.h"
#include "parse-util.h"
#include "path-util.h"
#include "prioq.h"
#include "random-util.h"
#include "sort-util.h"
#include "stat-util.h"
#include "string-table.h"
#include "string-util.h"
#include "strv.h"
#include "siphash24.h"
#include "sync-util.h"
#include "time-util.h"
#include "user-util.h"
#include "xattr-util.h"
 
#define DEFAULT_DATA_HASH_TABLE_SIZE 2047U
#define DEFAULT_FIELD_HASH_TABLE_SIZE 1023U
 
#define DEFAULT_COMPRESS_THRESHOLD 512U
#define MIN_COMPRESS_THRESHOLD 8U
 
/* This is the minimum journal file size */
#define JOURNAL_FILE_SIZE_MIN (512 * U64_KB)             /* 512 KiB */
#define JOURNAL_COMPACT_SIZE_MAX ((uint64_t) UINT32_MAX) /* 4 GiB */
 
/* These are the lower and upper bounds if we deduce the max_use value from the file system size */
#define MAX_USE_LOWER (1 * U64_MB)                       /* 1 MiB */
#define MAX_USE_UPPER (4 * U64_GB)                       /* 4 GiB */
 
/* Those are the lower and upper bounds for the minimal use limit,
 * i.e. how much we'll use even if keep_free suggests otherwise. */
#define MIN_USE_LOW  (1  * U64_MB)                       /* 1 MiB */
#define MIN_USE_HIGH (16 * U64_MB)                       /* 16 MiB */
 
/* This is the upper bound if we deduce max_size from max_use */
#define MAX_SIZE_UPPER (128 * U64_MB)                    /* 128 MiB */
 
/* This is the upper bound if we deduce the keep_free value from the file system size */
#define KEEP_FREE_UPPER (4 * U64_GB)                     /* 4 GiB */
 
/* This is the keep_free value when we can't determine the system size */
#define DEFAULT_KEEP_FREE (1 * U64_MB)                   /* 1 MB */
 
/* This is the default maximum number of journal files to keep around. */
#define DEFAULT_N_MAX_FILES 100
 
/* n_data was the first entry we added after the initial file format design */
#define HEADER_SIZE_MIN ALIGN64(offsetof(Header, n_data))
 
/* How many entries to keep in the entry array chain cache at max */
#define CHAIN_CACHE_MAX 20
 
/* How much to increase the journal file size at once each time we allocate something new. */
#define FILE_SIZE_INCREASE (8 * U64_MB)                  /* 8MB */
 
/* Reread fstat() of the file for detecting deletions at least this often */
#define LAST_STAT_REFRESH_USEC (5*USEC_PER_SEC)
 
/* Longest hash chain to rotate after */
#define HASH_CHAIN_DEPTH_MAX 100
 
#ifdef __clang__
#  pragma GCC diagnostic ignored "-Waddress-of-packed-member"
#endif
 
DEFINE_HASH_OPS_WITH_VALUE_DESTRUCTOR(
                journal_file_hash_ops_by_path,
                char, path_hash_func, path_compare,
                JournalFile, journal_file_close);
 
static int mmap_prot_from_open_flags(int flags) {
        switch (flags & O_ACCMODE_STRICT) {
        case O_RDONLY:
                return PROT_READ;
        case O_WRONLY:
                return PROT_WRITE;
        case O_RDWR:
                return PROT_READ|PROT_WRITE;
        default:
                assert_not_reached();
        }
}
 
int journal_file_tail_end_by_pread(JournalFile *f, uint64_t *ret_offset) {
        uint64_t p;
        int r;
 
        assert(f);
        assert(f->header);
        assert(ret_offset);
 
        /* Same as journal_file_tail_end_by_mmap() below, but operates with pread() to avoid the mmap cache
         * (and thus is thread safe) */
 
        p = le64toh(f->header->tail_object_offset);
        if (p == 0)
                p = le64toh(f->header->header_size);
        else {
                Object tail;
                uint64_t sz;
 
                r = journal_file_read_object_header(f, OBJECT_UNUSED, p, &tail);
                if (r < 0)
                        return r;
 
                sz = le64toh(tail.object.size);
                if (sz > UINT64_MAX - sizeof(uint64_t) + 1)
                        return -EBADMSG;
 
                sz = ALIGN64(sz);
                if (p > UINT64_MAX - sz)
                        return -EBADMSG;
 
                p += sz;
        }
 
        *ret_offset = p;
 
        return 0;
}
 
int journal_file_tail_end_by_mmap(JournalFile *f, uint64_t *ret_offset) {
        uint64_t p;
        int r;
 
        assert(f);
        assert(f->header);
        assert(ret_offset);
 
        /* Same as journal_file_tail_end_by_pread() above, but operates with the usual mmap logic */
 
        p = le64toh(f->header->tail_object_offset);
        if (p == 0)
                p = le64toh(f->header->header_size);
        else {
                Object *tail;
                uint64_t sz;
 
                r = journal_file_move_to_object(f, OBJECT_UNUSED, p, &tail);
                if (r < 0)
                        return r;
 
                sz = le64toh(READ_NOW(tail->object.size));
                if (sz > UINT64_MAX - sizeof(uint64_t) + 1)
                        return -EBADMSG;
 
                sz = ALIGN64(sz);
                if (p > UINT64_MAX - sz)
                        return -EBADMSG;
 
                p += sz;
        }
 
        *ret_offset = p;
 
        return 0;
}
 
int journal_file_set_offline_thread_join(JournalFile *f) {
        int r;
 
        assert(f);
 
        if (f->offline_state == OFFLINE_JOINED)
                return 0;
 
        log_debug("Joining journal offlining thread for %s.", f->path);
 
        r = pthread_join(f->offline_thread, NULL);
        if (r)
                return -r;
 
        log_debug("Journal offlining thread for %s joined.", f->path);
 
        f->offline_state = OFFLINE_JOINED;
 
        if (mmap_cache_fd_got_sigbus(f->cache_fd))
                return -EIO;
 
        return 0;
}
 
static int journal_file_set_online(JournalFile *f) {
        bool wait = true;
 
        assert(f);
 
        if (!journal_file_writable(f))
                return -EPERM;
 
        if (f->fd < 0 || !f->header)
                return -EINVAL;
 
        while (wait) {
                switch (f->offline_state) {
                case OFFLINE_JOINED:
                        /* No offline thread, no need to wait. */
                        wait = false;
                        break;
 
                case OFFLINE_SYNCING: {
                                OfflineState tmp_state = OFFLINE_SYNCING;
                                if (!__atomic_compare_exchange_n(&f->offline_state, &tmp_state, OFFLINE_CANCEL,
                                    false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
                                        continue;
                        }
                        /* Canceled syncing prior to offlining, no need to wait. */
                        wait = false;
                        break;
 
                case OFFLINE_AGAIN_FROM_SYNCING: {
                                OfflineState tmp_state = OFFLINE_AGAIN_FROM_SYNCING;
                                if (!__atomic_compare_exchange_n(&f->offline_state, &tmp_state, OFFLINE_CANCEL,
                                    false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
                                        continue;
                        }
                        /* Canceled restart from syncing, no need to wait. */
                        wait = false;
                        break;
 
                case OFFLINE_AGAIN_FROM_OFFLINING: {
                                OfflineState tmp_state = OFFLINE_AGAIN_FROM_OFFLINING;
                                if (!__atomic_compare_exchange_n(&f->offline_state, &tmp_state, OFFLINE_CANCEL,
                                    false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
                                        continue;
                        }
                        /* Canceled restart from offlining, must wait for offlining to complete however. */
                        _fallthrough_;
                default: {
                        int r;
 
                        r = journal_file_set_offline_thread_join(f);
                        if (r < 0)
                                return r;
 
                        wait = false;
                }}
        }
 
        if (mmap_cache_fd_got_sigbus(f->cache_fd))
                return -EIO;
 
        switch (f->header->state) {
                case STATE_ONLINE:
                        return 0;
 
                case STATE_OFFLINE:
                        f->header->state = STATE_ONLINE;
                        (void) fsync(f->fd);
                        return 0;
 
                default:
                        return -EINVAL;
        }
}
 
JournalFile* journal_file_close(JournalFile *f) {
        if (!f)
                return NULL;
 
        assert(f->newest_boot_id_prioq_idx == PRIOQ_IDX_NULL);
 
        sd_event_source_disable_unref(f->post_change_timer);
 
        if (f->cache_fd)
                mmap_cache_fd_free(f->cache_fd);
 
        if (f->close_fd)
                safe_close(f->fd);
        free(f->path);
 
        ordered_hashmap_free(f->chain_cache);
 
#if HAVE_COMPRESSION
        free(f->compress_buffer);
#endif
 
        journal_file_auth_done(f);
 
        return mfree(f);
}
 
static bool keyed_hash_requested(void) {
        static thread_local int cached = -1;
        int r;
 
        if (cached < 0) {
                r = secure_getenv_bool("SYSTEMD_JOURNAL_KEYED_HASH");
                if (r < 0) {
                        if (r != -ENXIO)
                                log_debug_errno(r, "Failed to parse $SYSTEMD_JOURNAL_KEYED_HASH environment variable, ignoring: %m");
                        cached = true;
                } else
                        cached = r;
        }
 
        return cached;
}
 
static bool compact_mode_requested(void) {
        static thread_local int cached = -1;
        int r;
 
        if (cached < 0) {
                r = secure_getenv_bool("SYSTEMD_JOURNAL_COMPACT");
                if (r < 0) {
                        if (r != -ENXIO)
                                log_debug_errno(r, "Failed to parse $SYSTEMD_JOURNAL_COMPACT environment variable, ignoring: %m");
                        cached = true;
                } else
                        cached = r;
        }
 
        return cached;
}
 
#if HAVE_COMPRESSION
static Compression getenv_compression(void) {
        Compression c;
        const char *e;
        int r;
 
        e = getenv("SYSTEMD_JOURNAL_COMPRESS");
        if (!e)
                return DEFAULT_COMPRESSION;
 
        r = parse_boolean(e);
        if (r >= 0)
                return r ? DEFAULT_COMPRESSION : COMPRESSION_NONE;
 
        c = compression_from_string_harder(e);
        if (c < 0) {
                log_debug_errno(c, "Failed to parse SYSTEMD_JOURNAL_COMPRESS value, ignoring: %s", e);
                return DEFAULT_COMPRESSION;
        }
 
        if (!compression_supported_journal(c)) {
                log_debug("Unsupported compression algorithm specified, ignoring: %s", e);
                return DEFAULT_COMPRESSION;
        }
 
        return c;
}
#endif
 
static Compression compression_requested(void) {
#if HAVE_COMPRESSION
        static thread_local Compression cached = _COMPRESSION_INVALID;
 
        if (cached < 0)
                cached = getenv_compression();
 
        return cached;
#else
        return COMPRESSION_NONE;
#endif
}
 
static int journal_file_init_header(
                JournalFile *f,
                JournalFileFlags file_flags,
                JournalFile *template) {
 
        int r;
 
        assert(f);
 
        /* Try to load the FSPRG state, and if we can't, then just don't do sealing */
        bool seal = FLAGS_SET(file_flags, JOURNAL_SEAL) && journal_file_auth_load(f) >= 0;
 
        Header h = {
                .header_size = htole64(ALIGN64(sizeof(h))),
                .incompatible_flags = htole32(
                                FLAGS_SET(file_flags, JOURNAL_COMPRESS) * COMPRESSION_TO_HEADER_INCOMPATIBLE_FLAG(compression_requested()) |
                                keyed_hash_requested() * HEADER_INCOMPATIBLE_KEYED_HASH |
                                compact_mode_requested() * HEADER_INCOMPATIBLE_COMPACT),
                .compatible_flags = htole32(
                                (seal * (HEADER_COMPATIBLE_SEALED | HEADER_COMPATIBLE_SEALED_CONTINUOUS) ) |
                                HEADER_COMPATIBLE_TAIL_ENTRY_BOOT_ID),
        };
 
        assert_cc(sizeof(h.signature) == sizeof(HEADER_SIGNATURE));
        memcpy(h.signature, HEADER_SIGNATURE, sizeof(HEADER_SIGNATURE));
 
        r = sd_id128_randomize(&h.file_id);
        if (r < 0)
                return r;
 
        r = sd_id128_get_machine(&h.machine_id);
        if (r < 0 && !ERRNO_IS_MACHINE_ID_UNSET(r))
                return r; /* If we have no valid machine ID (test environment?), let's simply leave the
                           * machine ID field all zeroes. */
 
        if (template) {
                h.seqnum_id = template->header->seqnum_id;
                h.tail_entry_seqnum = template->header->tail_entry_seqnum;
        } else
                h.seqnum_id = h.file_id;
 
        return pwrite_full(f->fd, &h, sizeof(h), 0);
}
 
static int journal_file_refresh_header(JournalFile *f) {
        int r;
 
        assert(f);
        assert(f->header);
 
        /* We used to update the header's boot ID field here, but we don't do that anymore, as per
         * HEADER_COMPATIBLE_TAIL_ENTRY_BOOT_ID */
 
        r = journal_file_set_online(f);
 
        /* Sync the online state to disk; likely just created a new file, also sync the directory this file
         * is located in. */
        (void) fsync_full(f->fd);
 
        return r;
}
 
static bool warn_wrong_flags(const JournalFile *f, bool compatible) {
        const uint32_t any = compatible ? HEADER_COMPATIBLE_ANY : HEADER_INCOMPATIBLE_ANY;
        uint32_t supported = compatible ? HEADER_COMPATIBLE_SUPPORTED : HEADER_INCOMPATIBLE_SUPPORTED;
        const char *type = compatible ? "compatible" : "incompatible";
        uint32_t flags;
 
        assert(f);
        assert(f->header);
 
        /* When sealing is not supported, refuse to write to an already sealed journal file, but still allow
         * reading sealed journal files. */
        if (compatible && journal_file_writable(f) && !journal_auth_supported())
                supported &= ~(HEADER_COMPATIBLE_SEALED | HEADER_COMPATIBLE_SEALED_CONTINUOUS);
 
        flags = le32toh(compatible ? f->header->compatible_flags : f->header->incompatible_flags);
 
        if (flags & ~supported) {
                if (flags & ~any)
                        log_debug("Journal file %s has unknown %s flags 0x%"PRIx32,
                                  f->path, type, flags & ~any);
                flags = (flags & any) & ~supported;
                if (flags) {
                        const char* strv[6];
                        size_t n = 0;
                        _cleanup_free_ char *t = NULL;
 
                        if (compatible) {
                                if (flags & HEADER_COMPATIBLE_SEALED)
                                        strv[n++] = "sealed";
                                if (flags & HEADER_COMPATIBLE_SEALED_CONTINUOUS)
                                        strv[n++] = "sealed-continuous";
                        } else {
                                if (flags & HEADER_INCOMPATIBLE_COMPRESSED_XZ)
                                        strv[n++] = "xz-compressed";
                                if (flags & HEADER_INCOMPATIBLE_COMPRESSED_LZ4)
                                        strv[n++] = "lz4-compressed";
                                if (flags & HEADER_INCOMPATIBLE_COMPRESSED_ZSTD)
                                        strv[n++] = "zstd-compressed";
                                if (flags & HEADER_INCOMPATIBLE_KEYED_HASH)
                                        strv[n++] = "keyed-hash";
                                if (flags & HEADER_INCOMPATIBLE_COMPACT)
                                        strv[n++] = "compact";
                        }
                        strv[n] = NULL;
                        assert(n < ELEMENTSOF(strv));
 
                        t = strv_join((char**) strv, ", ");
                        log_debug("Journal file %s uses %s %s %s disabled at compilation time.",
                                  f->path, type, n > 1 ? "flags" : "flag", strnull(t));
                }
                return true;
        }
 
        return false;
}
 
static bool offset_is_valid(uint64_t offset, uint64_t header_size, uint64_t tail_object_offset) {
        if (offset == 0)
                return true;
        if (!VALID64(offset))
                return false;
        if (offset < header_size)
                return false;
        if (offset > tail_object_offset)
                return false;
        return true;
}
 
static bool hash_table_is_valid(uint64_t offset, uint64_t size, uint64_t header_size, uint64_t arena_size, uint64_t tail_object_offset) {
        if ((offset == 0) != (size == 0))
                return false;
        if (offset == 0)
                return true;
        if (offset <= offsetof(Object, hash_table.items))
                return false;
        offset -= offsetof(Object, hash_table.items);
        if (!offset_is_valid(offset, header_size, tail_object_offset))
                return false;
        assert(offset <= header_size + arena_size);
        if (size > header_size + arena_size - offset)
                return false;
        return true;
}
 
static int journal_file_verify_header(JournalFile *f) {
        uint64_t arena_size, header_size;
 
        assert(f);
        assert(f->header);
 
        if (memcmp(f->header->signature, HEADER_SIGNATURE, 8) != 0)
                return -EBADMSG;
 
        /* In both read and write mode we refuse to open files with incompatible
         * flags we don't know. */
        if (warn_wrong_flags(f, false))
                return -EPROTONOSUPPORT;
 
        /* When open for writing we refuse to open files with compatible flags, too. */
        if (journal_file_writable(f) && warn_wrong_flags(f, true))
                return -EPROTONOSUPPORT;
 
        if (f->header->state >= _STATE_MAX)
                return -EBADMSG;
 
        header_size = le64toh(READ_NOW(f->header->header_size));
 
        /* The first addition was n_data, so check that we are at least this large */
        if (header_size < HEADER_SIZE_MIN)
                return -EBADMSG;
 
        /* When open for writing we refuse to open files with a mismatch of the header size, i.e. writing to
         * files implementing older or new header structures. */
        if (journal_file_writable(f) && header_size != sizeof(Header))
                return -EPROTONOSUPPORT;
 
        /* Don't write to journal files without the new boot ID update behavior guarantee. */
        if (journal_file_writable(f) && !JOURNAL_HEADER_TAIL_ENTRY_BOOT_ID(f->header))
                return -EPROTONOSUPPORT;
 
        if (JOURNAL_HEADER_SEALED(f->header) && !JOURNAL_HEADER_CONTAINS(f->header, n_entry_arrays))
                return -EBADMSG;
 
        arena_size = le64toh(READ_NOW(f->header->arena_size));
 
        if (UINT64_MAX - header_size < arena_size)
                return -ENODATA;
 
        uint64_t file_size = (uint64_t) f->last_stat.st_size;
 
        /* Probably an unclean shutdown where the header was written, but the arena data was not. On write we
         * should ask the caller to rotate, but on read, we can still work it out with bounds checks. */
        bool truncated = false;
        if (header_size + arena_size > file_size) {
                if (journal_file_writable(f))
                        return -ENODATA;
 
                /* This shouldn't happen given file_size is page aligned via fallocate(), but just in case
                 * things are _really_ messed up... */
                uint64_t available = ALIGN_DOWN_U64(file_size, sizeof(uint64_t));
                if (header_size > available || available - header_size < offsetof(ObjectHeader, payload))
                        return -ENODATA;
 
                log_debug("Journal file %s claims a %" PRIu64 " byte arena but is only %" PRIu64
                          " bytes on disk, clamping for recovery.",
                          f->path,
                          arena_size,
                          file_size);
                arena_size = available - header_size;
                truncated = true;
        }
 
        uint64_t tail_object_offset = le64toh(f->header->tail_object_offset);
        if (truncated)
                /* The tail may be in the lost region, so cap it at the last possible object header start. */
                tail_object_offset = MIN(
                                tail_object_offset,
                                header_size + arena_size - offsetof(ObjectHeader, payload));
        if (!offset_is_valid(tail_object_offset, header_size, UINT64_MAX))
                return -ENODATA;
        if (header_size + arena_size < tail_object_offset)
                return -ENODATA;
        if (header_size + arena_size - tail_object_offset < offsetof(ObjectHeader, payload))
                return -ENODATA;
 
        if (!hash_table_is_valid(le64toh(f->header->data_hash_table_offset),
                                 le64toh(f->header->data_hash_table_size),
                                 header_size, arena_size, tail_object_offset))
                return -ENODATA;
 
        if (!hash_table_is_valid(le64toh(f->header->field_hash_table_offset),
                                 le64toh(f->header->field_hash_table_size),
                                 header_size, arena_size, tail_object_offset))
                return -ENODATA;
 
        uint64_t entry_array_offset = le64toh(f->header->entry_array_offset);
        if (!offset_is_valid(entry_array_offset, header_size, tail_object_offset))
                return -ENODATA;
 
        if (!truncated && JOURNAL_HEADER_CONTAINS(f->header, tail_entry_array_offset)) {
                uint32_t offset = le32toh(f->header->tail_entry_array_offset);
                uint32_t n = le32toh(f->header->tail_entry_array_n_entries);
 
                if (!offset_is_valid(offset, header_size, tail_object_offset))
                        return -ENODATA;
                if (entry_array_offset > offset)
                        return -ENODATA;
                if (entry_array_offset == 0 && offset != 0)
                        return -ENODATA;
                if ((offset == 0) != (n == 0))
                        return -ENODATA;
                assert(offset <= header_size + arena_size);
                if ((uint64_t) n * journal_file_entry_array_item_size(f) > header_size + arena_size - offset)
                        return -ENODATA;
        }
 
        if (!truncated && JOURNAL_HEADER_CONTAINS(f->header, tail_entry_offset)) {
                uint64_t offset = le64toh(f->header->tail_entry_offset);
 
                if (!offset_is_valid(offset, header_size, tail_object_offset))
                        return -ENODATA;
 
                if (offset > 0) {
                        /* When there is an entry object, then these fields must be filled. */
                        if (sd_id128_is_null(f->header->tail_entry_boot_id))
                                return -ENODATA;
                        if (!VALID_REALTIME(le64toh(f->header->head_entry_realtime)))
                                return -ENODATA;
                        if (!VALID_REALTIME(le64toh(f->header->tail_entry_realtime)))
                                return -ENODATA;
                        if (!VALID_MONOTONIC(le64toh(f->header->tail_entry_monotonic)))
                                return -ENODATA;
                } else {
                        /* Otherwise, the fields must be zero. */
                        if (JOURNAL_HEADER_TAIL_ENTRY_BOOT_ID(f->header) &&
                            !sd_id128_is_null(f->header->tail_entry_boot_id))
                                return -ENODATA;
                        if (f->header->head_entry_realtime != 0)
                                return -ENODATA;
                        if (f->header->tail_entry_realtime != 0)
                                return -ENODATA;
                        if (f->header->tail_entry_monotonic != 0)
                                return -ENODATA;
                }
        }
 
        /* Verify number of objects */
        uint64_t n_objects = le64toh(f->header->n_objects);
        if (!truncated && n_objects > arena_size / offsetof(ObjectHeader, payload))
                return -ENODATA;
 
        uint64_t n_entries = le64toh(f->header->n_entries);
        if (n_entries > n_objects)
                return -ENODATA;
 
        if (JOURNAL_HEADER_CONTAINS(f->header, n_data) &&
            le64toh(f->header->n_data) > n_objects)
                return -ENODATA;
 
        if (JOURNAL_HEADER_CONTAINS(f->header, n_fields) &&
            le64toh(f->header->n_fields) > n_objects)
                return -ENODATA;
 
        if (JOURNAL_HEADER_CONTAINS(f->header, n_tags) &&
            le64toh(f->header->n_tags) > n_objects)
                return -ENODATA;
 
        if (JOURNAL_HEADER_CONTAINS(f->header, n_entry_arrays) &&
            le64toh(f->header->n_entry_arrays) > n_objects)
                return -ENODATA;
 
        if (JOURNAL_HEADER_CONTAINS(f->header, tail_entry_array_n_entries) &&
            le32toh(f->header->tail_entry_array_n_entries) > n_entries)
                return -ENODATA;
 
        if (journal_file_writable(f)) {
                sd_id128_t machine_id;
                uint8_t state;
                int r;
 
                r = sd_id128_get_machine(&machine_id);
                if (ERRNO_IS_NEG_MACHINE_ID_UNSET(r)) /* Gracefully handle the machine ID not being initialized yet */
                        machine_id = SD_ID128_NULL;
                else if (r < 0)
                        return r;
 
                if (!sd_id128_equal(machine_id, f->header->machine_id))
                        return log_debug_errno(SYNTHETIC_ERRNO(EHOSTDOWN),
                                               "Trying to open journal file from different host for writing, refusing.");
 
                state = f->header->state;
 
                if (state == STATE_ARCHIVED)
                        return -ESHUTDOWN; /* Already archived */
                if (state == STATE_ONLINE)
                        return log_debug_errno(SYNTHETIC_ERRNO(EBUSY),
                                               "Journal file %s is already online. Assuming unclean closing.",
                                               f->path);
                if (state != STATE_OFFLINE)
                        return log_debug_errno(SYNTHETIC_ERRNO(EBUSY),
                                               "Journal file %s has unknown state %i.",
                                               f->path, state);
 
                if (f->header->field_hash_table_size == 0 || f->header->data_hash_table_size == 0)
                        return -EBADMSG;
        }
 
        return 0;
}
 
int journal_file_fstat(JournalFile *f) {
        int r;
 
        assert(f);
        assert(f->fd >= 0);
 
        if (fstat(f->fd, &f->last_stat) < 0)
                return -errno;
 
        f->last_stat_usec = now(CLOCK_MONOTONIC);
 
        /* Refuse dealing with files that aren't regular */
        r = stat_verify_regular(&f->last_stat);
        if (r < 0)
                return r;
 
        /* Refuse appending to files that are already deleted */
        r = stat_verify_linked(&f->last_stat);
        if (r < 0)
                return r;
 
        return 0;
}
 
static int journal_file_allocate(JournalFile *f, uint64_t offset, uint64_t size) {
        uint64_t old_size, new_size, old_header_size, old_arena_size;
        int r;
 
        assert(f);
        assert(f->header);
 
        /* We assume that this file is not sparse, and we know that for sure, since we always call
         * posix_fallocate() ourselves */
 
        if (size > PAGE_ALIGN_DOWN_U64(UINT64_MAX) - offset)
                return -EINVAL;
 
        if (mmap_cache_fd_got_sigbus(f->cache_fd))
                return -EIO;
 
        old_header_size = le64toh(READ_NOW(f->header->header_size));
        old_arena_size = le64toh(READ_NOW(f->header->arena_size));
        if (old_arena_size > PAGE_ALIGN_DOWN_U64(UINT64_MAX) - old_header_size)
                return -EBADMSG;
 
        old_size = old_header_size + old_arena_size;
 
        new_size = MAX(PAGE_ALIGN_U64(offset + size), old_header_size);
 
        if (new_size <= old_size) {
 
                /* We already pre-allocated enough space, but before
                 * we write to it, let's check with fstat() if the
                 * file got deleted, in order make sure we don't throw
                 * away the data immediately. Don't check fstat() for
                 * all writes though, but only once ever 10s. */
 
                if (f->last_stat_usec + LAST_STAT_REFRESH_USEC > now(CLOCK_MONOTONIC))
                        return 0;
 
                return journal_file_fstat(f);
        }
 
        /* Allocate more space. */
 
        if (f->metrics.max_size > 0 && new_size > f->metrics.max_size)
                return -E2BIG;
 
        /* Refuse to go over 4G in compact mode so offsets can be stored in 32-bit. */
        if (JOURNAL_HEADER_COMPACT(f->header) && new_size > UINT32_MAX)
                return -E2BIG;
 
        if (new_size > f->metrics.min_size && f->metrics.keep_free > 0) {
                struct statvfs svfs;
 
                if (fstatvfs(f->fd, &svfs) >= 0) {
                        uint64_t available;
 
                        available = LESS_BY(u64_multiply_safe(svfs.f_bfree, svfs.f_bsize), f->metrics.keep_free);
 
                        if (new_size - old_size > available)
                                return -E2BIG;
                }
        }
 
        /* Increase by larger blocks at once */
        new_size = ROUND_UP(new_size, FILE_SIZE_INCREASE);
        if (f->metrics.max_size > 0 && new_size > f->metrics.max_size)
                new_size = f->metrics.max_size;
 
        /* Note that the glibc fallocate() fallback is very
           inefficient, hence we try to minimize the allocation area
           as we can. */
        r = posix_fallocate_loop(f->fd, old_size, new_size - old_size);
        if (r < 0)
                return r;
 
        f->header->arena_size = htole64(new_size - old_header_size);
 
        return journal_file_fstat(f);
}
 
static int journal_file_move_to(
                JournalFile *f,
                ObjectType type,
                bool keep_always,
                uint64_t offset,
                uint64_t size,
                void **ret) {
 
        int r;
 
        assert(f);
        assert(ret);
 
        /* This function may clear, overwrite, or alter previously cached entries with the same type. After
         * this function has been called, all previously read objects with the same type may be invalidated,
         * hence must be re-read before use. */
 
        if (size <= 0)
                return -EINVAL;
 
        if (size > UINT64_MAX - offset)
                return -EBADMSG;
 
        /* Avoid SIGBUS on invalid accesses */
        if (offset + size > (uint64_t) f->last_stat.st_size) {
                /* Hmm, out of range? Let's refresh the fstat() data
                 * first, before we trust that check. */
 
                r = journal_file_fstat(f);
                if (r < 0)
                        return r;
 
                if (offset + size > (uint64_t) f->last_stat.st_size)
                        return -EADDRNOTAVAIL;
        }
 
        return mmap_cache_fd_get(f->cache_fd, type_to_category(type), keep_always, offset, size, &f->last_stat, ret);
}
 
static uint64_t minimum_header_size(JournalFile *f, Object *o) {
 
        static const uint64_t table[] = {
                [OBJECT_DATA]             = sizeof(DataObject),
                [OBJECT_FIELD]            = sizeof(FieldObject),
                [OBJECT_ENTRY]            = sizeof(EntryObject),
                [OBJECT_DATA_HASH_TABLE]  = sizeof(HashTableObject),
                [OBJECT_FIELD_HASH_TABLE] = sizeof(HashTableObject),
                [OBJECT_ENTRY_ARRAY]      = sizeof(EntryArrayObject),
                [OBJECT_TAG]              = sizeof(TagObject),
        };
 
        assert(f);
        assert(o);
 
        if (o->object.type == OBJECT_DATA)
                return journal_file_data_payload_offset(f);
 
        if (o->object.type >= ELEMENTSOF(table) || table[o->object.type] <= 0)
                return offsetof(ObjectHeader, payload);
 
        return table[o->object.type];
}
 
static int check_object_header(JournalFile *f, Object *o, ObjectType type, uint64_t offset) {
        uint64_t s;
 
        assert(f);
        assert(o);
 
        s = le64toh(READ_NOW(o->object.size));
        if (s == 0)
                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                       "Attempt to move to uninitialized object: %" PRIu64,
                                       offset);
 
        if (s < offsetof(ObjectHeader, payload))
                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                       "Attempt to move to overly short object with size %"PRIu64": %" PRIu64,
                                       s, offset);
 
        if (o->object.type <= OBJECT_UNUSED || o->object.type >= _OBJECT_TYPE_MAX)
                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                       "Attempt to move to object with invalid type (%u): %" PRIu64,
                                       o->object.type, offset);
 
        if (type > OBJECT_UNUSED && o->object.type != type)
                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                       "Found %s object while expecting %s object: %" PRIu64,
                                       journal_object_type_to_string(o->object.type),
                                       journal_object_type_to_string(type),
                                       offset);
 
        if (s < minimum_header_size(f, o))
                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                       "Size of %s object (%"PRIu64") is smaller than the minimum object size (%"PRIu64"): %" PRIu64,
                                       journal_object_type_to_string(o->object.type),
                                       s,
                                       minimum_header_size(f, o),
                                       offset);
 
        return 0;
}
 
/* Lightweight object checks. We want this to be fast, so that we won't
 * slowdown every journal_file_move_to_object() call too much. */
static int check_object(JournalFile *f, Object *o, uint64_t offset) {
        assert(f);
        assert(o);
 
        switch (o->object.type) {
 
        case OBJECT_DATA:
                if ((le64toh(o->data.entry_offset) == 0) ^ (le64toh(o->data.n_entries) == 0))
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Bad data n_entries: %" PRIu64 ": %" PRIu64,
                                               le64toh(o->data.n_entries),
                                               offset);
 
                if (le64toh(o->object.size) <= journal_file_data_payload_offset(f))
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Bad data size (<= %zu): %" PRIu64 ": %" PRIu64,
                                               journal_file_data_payload_offset(f),
                                               le64toh(o->object.size),
                                               offset);
 
                if (!VALID64(le64toh(o->data.next_hash_offset)) ||
                    !VALID64(le64toh(o->data.next_field_offset)) ||
                    !VALID64(le64toh(o->data.entry_offset)) ||
                    !VALID64(le64toh(o->data.entry_array_offset)))
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Invalid offset, next_hash_offset=" OFSfmt ", next_field_offset=" OFSfmt ", entry_offset=" OFSfmt ", entry_array_offset=" OFSfmt ": %" PRIu64,
                                               le64toh(o->data.next_hash_offset),
                                               le64toh(o->data.next_field_offset),
                                               le64toh(o->data.entry_offset),
                                               le64toh(o->data.entry_array_offset),
                                               offset);
 
                /* Objects are only ever appended, hence a hash chain has to move strictly forward. */
                if (le64toh(o->data.next_hash_offset) > 0 && le64toh(o->data.next_hash_offset) <= offset)
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Invalid data object next_hash_offset: %" PRIu64 ": %" PRIu64,
                                               le64toh(o->data.next_hash_offset),
                                               offset);
 
                /* New data objects are prepended to the list of their field, hence that list has to move
                 * strictly backward. */
                if (le64toh(o->data.next_field_offset) >= offset)
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Invalid data object next_field_offset: %" PRIu64 ": %" PRIu64,
                                               le64toh(o->data.next_field_offset),
                                               offset);
 
                break;
 
        case OBJECT_FIELD:
                if (le64toh(o->object.size) <= offsetof(Object, field.payload))