HALF ECHO LEARN / REFERENCE COLLECTION

v262.2

src/libsystemd/sd-journal/journal-file.c

178,971 bytesGit mode 100644Download original fileLGPL-2.1-or-later

Git blob: bed0c812a6957f446f48f2101888a46dca5a6bca

Lines 1001–2000 of 4,786

                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Bad field size (<= %zu): %" PRIu64 ": %" PRIu64,
                                               offsetof(Object, field.payload),
                                               le64toh(o->object.size),
                                               offset);
 
                if (!VALID64(le64toh(o->field.next_hash_offset)) ||
                    !VALID64(le64toh(o->field.head_data_offset)))
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Invalid offset, next_hash_offset=" OFSfmt ", head_data_offset=" OFSfmt ": %" PRIu64,
                                               le64toh(o->field.next_hash_offset),
                                               le64toh(o->field.head_data_offset),
                                               offset);
 
                if (le64toh(o->field.next_hash_offset) > 0 && le64toh(o->field.next_hash_offset) <= offset)
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Invalid field object next_hash_offset: %" PRIu64 ": %" PRIu64,
                                               le64toh(o->field.next_hash_offset),
                                               offset);
                break;
 
        case OBJECT_ENTRY: {
                uint64_t sz;
 
                sz = le64toh(READ_NOW(o->object.size));
                if (sz < offsetof(Object, entry.items) ||
                    (sz - offsetof(Object, entry.items)) % journal_file_entry_item_size(f) != 0)
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Bad entry size (<= %zu): %" PRIu64 ": %" PRIu64,
                                               offsetof(Object, entry.items),
                                               sz,
                                               offset);
 
                if ((sz - offsetof(Object, entry.items)) / journal_file_entry_item_size(f) <= 0)
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Invalid number items in entry: %" PRIu64 ": %" PRIu64,
                                               (sz - offsetof(Object, entry.items)) / journal_file_entry_item_size(f),
                                               offset);
 
                if (le64toh(o->entry.seqnum) <= 0)
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Invalid entry seqnum: %" PRIx64 ": %" PRIu64,
                                               le64toh(o->entry.seqnum),
                                               offset);
 
                if (!VALID_REALTIME(le64toh(o->entry.realtime)))
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Invalid entry realtime timestamp: %" PRIu64 ": %" PRIu64,
                                               le64toh(o->entry.realtime),
                                               offset);
 
                if (!VALID_MONOTONIC(le64toh(o->entry.monotonic)))
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Invalid entry monotonic timestamp: %" PRIu64 ": %" PRIu64,
                                               le64toh(o->entry.monotonic),
                                               offset);
 
                if (sd_id128_is_null(o->entry.boot_id))
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Invalid object entry with an empty boot ID: %" PRIu64,
                                               offset);
 
                break;
        }
 
        case OBJECT_DATA_HASH_TABLE:
        case OBJECT_FIELD_HASH_TABLE: {
                uint64_t sz;
 
                sz = le64toh(READ_NOW(o->object.size));
                if (sz < offsetof(Object, hash_table.items) ||
                    (sz - offsetof(Object, hash_table.items)) % sizeof(HashItem) != 0 ||
                    (sz - offsetof(Object, hash_table.items)) / sizeof(HashItem) <= 0)
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Invalid %s hash table size: %" PRIu64 ": %" PRIu64,
                                               journal_object_type_to_string(o->object.type),
                                               sz,
                                               offset);
 
                break;
        }
 
        case OBJECT_ENTRY_ARRAY: {
                uint64_t sz, next;
 
                sz = le64toh(READ_NOW(o->object.size));
                if (sz < offsetof(Object, entry_array.items) ||
                    (sz - offsetof(Object, entry_array.items)) % journal_file_entry_array_item_size(f) != 0 ||
                    (sz - offsetof(Object, entry_array.items)) / journal_file_entry_array_item_size(f) <= 0)
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Invalid object entry array size: %" PRIu64 ": %" PRIu64,
                                               sz,
                                               offset);
                /* Here, we request that the offset of each entry array object is in strictly increasing order. */
                next = le64toh(o->entry_array.next_entry_array_offset);
                if (!VALID64(next) || (next > 0 && next <= offset))
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Invalid object entry array next_entry_array_offset: %" PRIu64 ": %" PRIu64,
                                               next,
                                               offset);
 
                break;
        }
 
        case OBJECT_TAG:
                if (le64toh(o->object.size) != sizeof(TagObject))
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Invalid object tag size: %" PRIu64 ": %" PRIu64,
                                               le64toh(o->object.size),
                                               offset);
 
                if (!VALID_EPOCH(le64toh(o->tag.epoch)))
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Invalid object tag epoch: %" PRIu64 ": %" PRIu64,
                                               le64toh(o->tag.epoch), offset);
 
                break;
        }
 
        return 0;
}
 
int journal_file_move_to_object(JournalFile *f, ObjectType type, uint64_t offset, Object **ret) {
        int r;
        Object *o;
 
        assert(f);
 
        /* Even if this function fails, it may clear, overwrite, or alter previously cached entries with the
         * same type. After this function has been called, all previously read objects with the same type may
         * be invalidated, hence must be re-read before use. */
 
        /* Objects may only be located at multiple of 64 bit */
        if (!VALID64(offset))
                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                       "Attempt to move to %s object at non-64-bit boundary: %" PRIu64,
                                       journal_object_type_to_string(type),
                                       offset);
 
        /* Object may not be located in the file header */
        if (offset < le64toh(f->header->header_size))
                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                       "Attempt to move to %s object located in file header: %" PRIu64,
                                       journal_object_type_to_string(type),
                                       offset);
 
        r = journal_file_move_to(f, type, false, offset, offsetof(ObjectHeader, payload), (void**) &o);
        if (r < 0)
                return r;
 
        r = check_object_header(f, o, type, offset);
        if (r < 0)
                return r;
 
        r = journal_file_move_to(f, type, false, offset, le64toh(READ_NOW(o->object.size)), (void**) &o);
        if (r < 0)
                return r;
 
        r = check_object_header(f, o, type, offset);
        if (r < 0)
                return r;
 
        r = check_object(f, o, offset);
        if (r < 0)
                return r;
 
        if (ret)
                *ret = o;
 
        return 0;
}
 
int journal_file_pin_object(JournalFile *f, Object *o) {
        assert(f);
        assert(o);
 
        /* This attaches the mmap window that provides the object to the 'pinning' category. So, reading
         * another object with the same type will not invalidate the object, until this function is called
         * for another object. */
        return mmap_cache_fd_pin(f->cache_fd, type_to_category(o->object.type), o, le64toh(o->object.size));
}
 
int journal_file_read_object_header(JournalFile *f, ObjectType type, uint64_t offset, Object *ret) {
        ssize_t n;
        Object o;
        int r;
 
        assert(f);
 
        /* Objects may only be located at multiple of 64 bit */
        if (!VALID64(offset))
                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                       "Attempt to read %s object at non-64-bit boundary: %" PRIu64,
                                       journal_object_type_to_string(type), offset);
 
        /* Object may not be located in the file header */
        if (offset < le64toh(f->header->header_size))
                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                       "Attempt to read %s object located in file header: %" PRIu64,
                                       journal_object_type_to_string(type), offset);
 
        /* This will likely read too much data but it avoids having to call pread() twice. */
        n = pread(f->fd, &o, sizeof(o), offset);
        if (n < 0)
                return log_debug_errno(errno, "Failed to read journal %s object at offset: %" PRIu64,
                                       journal_object_type_to_string(type), offset);
 
        if ((size_t) n < sizeof(o.object))
                return log_debug_errno(SYNTHETIC_ERRNO(EIO),
                                       "Failed to read short %s object at offset: %" PRIu64,
                                       journal_object_type_to_string(type), offset);
 
        r = check_object_header(f, &o, type, offset);
        if (r < 0)
                return r;
 
        if ((size_t) n < minimum_header_size(f, &o))
                return log_debug_errno(SYNTHETIC_ERRNO(EIO),
                                       "Short read while reading %s object: %" PRIu64,
                                       journal_object_type_to_string(type), offset);
 
        r = check_object(f, &o, offset);
        if (r < 0)
                return r;
 
        if (ret)
                *ret = o;
 
        return 0;
}
 
static uint64_t inc_seqnum(uint64_t seqnum) {
        if (seqnum < UINT64_MAX-1)
                return seqnum + 1;
 
        return 1; /* skip over UINT64_MAX and 0 when we run out of seqnums and start again */
}
 
static uint64_t journal_file_entry_seqnum(
                JournalFile *f,
                uint64_t *seqnum) {
 
        uint64_t next_seqnum;
 
        assert(f);
        assert(f->header);
 
        /* Picks a new sequence number for the entry we are about to add and returns it. */
 
        next_seqnum = inc_seqnum(le64toh(f->header->tail_entry_seqnum));
 
        /* If an external seqnum counter was passed, we update both the local and the external one, and set
         * it to the maximum of both */
        if (seqnum)
                *seqnum = next_seqnum = MAX(inc_seqnum(*seqnum), next_seqnum);
 
        f->header->tail_entry_seqnum = htole64(next_seqnum);
 
        if (f->header->head_entry_seqnum == 0)
                f->header->head_entry_seqnum = htole64(next_seqnum);
 
        return next_seqnum;
}
 
int journal_file_append_object(
                JournalFile *f,
                ObjectType type,
                uint64_t size,
                Object **ret_object,
                uint64_t *ret_offset) {
 
        int r;
        uint64_t p;
        Object *o;
 
        assert(f);
        assert(f->header);
        assert(type > OBJECT_UNUSED && type < _OBJECT_TYPE_MAX);
        assert(size >= offsetof(ObjectHeader, payload));
 
        r = journal_file_set_online(f);
        if (r < 0)
                return r;
 
        r = journal_file_tail_end_by_mmap(f, &p);
        if (r < 0)
                return r;
 
        r = journal_file_allocate(f, p, size);
        if (r < 0)
                return r;
 
        r = journal_file_move_to(f, type, false, p, size, (void**) &o);
        if (r < 0)
                return r;
 
        o->object = (ObjectHeader) {
                .type = type,
                .size = htole64(size),
        };
 
        f->header->tail_object_offset = htole64(p);
        f->header->n_objects = htole64(le64toh(f->header->n_objects) + 1);
 
        if (ret_object)
                *ret_object = o;
 
        if (ret_offset)
                *ret_offset = p;
 
        return 0;
}
 
static int journal_file_setup_data_hash_table(JournalFile *f) {
        uint64_t s, p;
        int r;
 
        assert(f);
        assert(f->header);
 
        /* We estimate that we need 1 hash table entry per 768 bytes
           of journal file and we want to make sure we never get
           beyond 75% fill level. Calculate the hash table size for
           the maximum file size based on these metrics. */
 
        s = MAX(f->metrics.max_size * 4 / 768 / 3,
                DEFAULT_DATA_HASH_TABLE_SIZE);
 
        log_debug("Reserving %"PRIu64" entries in data hash table.", s);
 
        /* We deliberately don't zero the hash table items: the table is placed in a newly created file, in
         * space that was just allocated via posix_fallocate() and hence reads as zeroes anyway. Writing the
         * zeroes through the memory map would not only write out the whole table once for no reason: the
         * sequential write faults would also make the kernel's mmap readahead instantiate most of the table
         * in large folios, on file systems that support them. And since write faults on shared file mappings
         * dirty the whole folio, each later update of a single hash bucket would then cause the whole folio
         * to be written back. See #40262. */
 
        r = journal_file_append_object(f,
                                       OBJECT_DATA_HASH_TABLE,
                                       offsetof(Object, hash_table.items) + s * sizeof(HashItem),
                                       /* ret_object= */ NULL, &p);
        if (r < 0)
                return r;
 
        f->header->data_hash_table_offset = htole64(p + offsetof(Object, hash_table.items));
        f->header->data_hash_table_size = htole64(s * sizeof(HashItem));
 
        return 0;
}
 
static int journal_file_setup_field_hash_table(JournalFile *f) {
        uint64_t s, p;
        int r;
 
        assert(f);
        assert(f->header);
 
        /* We use a fixed size hash table for the fields as this
         * number should grow very slowly only */
 
        s = DEFAULT_FIELD_HASH_TABLE_SIZE;
        log_debug("Reserving %"PRIu64" entries in field hash table.", s);
 
        /* No need to zero the items, see journal_file_setup_data_hash_table(). */
        r = journal_file_append_object(f,
                                       OBJECT_FIELD_HASH_TABLE,
                                       offsetof(Object, hash_table.items) + s * sizeof(HashItem),
                                       /* ret_object= */ NULL, &p);
        if (r < 0)
                return r;
 
        f->header->field_hash_table_offset = htole64(p + offsetof(Object, hash_table.items));
        f->header->field_hash_table_size = htole64(s * sizeof(HashItem));
 
        return 0;
}
 
int journal_file_map_data_hash_table(JournalFile *f) {
        uint64_t s, p;
        void *t;
        int r;
 
        assert(f);
        assert(f->header);
 
        if (f->data_hash_table)
                return 0;
 
        p = le64toh(f->header->data_hash_table_offset);
        s = le64toh(f->header->data_hash_table_size);
 
        r = journal_file_move_to(f,
                                 OBJECT_DATA_HASH_TABLE,
                                 true,
                                 p, s,
                                 &t);
        if (r < 0)
                return r;
 
        f->data_hash_table = t;
        return 0;
}
 
int journal_file_map_field_hash_table(JournalFile *f) {
        uint64_t s, p;
        void *t;
        int r;
 
        assert(f);
        assert(f->header);
 
        if (f->field_hash_table)
                return 0;
 
        p = le64toh(f->header->field_hash_table_offset);
        s = le64toh(f->header->field_hash_table_size);
 
        r = journal_file_move_to(f,
                                 OBJECT_FIELD_HASH_TABLE,
                                 true,
                                 p, s,
                                 &t);
        if (r < 0)
                return r;
 
        f->field_hash_table = t;
        return 0;
}
 
static int journal_file_link_field(
                JournalFile *f,
                Object *o,
                uint64_t offset,
                uint64_t hash) {
 
        uint64_t p, h, m;
        int r;
 
        assert(f);
        assert(f->header);
        assert(f->field_hash_table);
        assert(o);
        assert(offset > 0);
 
        if (o->object.type != OBJECT_FIELD)
                return -EBADMSG;
 
        m = le64toh(READ_NOW(f->header->field_hash_table_size)) / sizeof(HashItem);
        if (m <= 0)
                return -EBADMSG;
 
        /* This might alter the window we are looking at */
        o->field.next_hash_offset = o->field.head_data_offset = 0;
 
        h = hash % m;
        p = le64toh(f->field_hash_table[h].tail_hash_offset);
        if (p == 0)
                f->field_hash_table[h].head_hash_offset = htole64(offset);
        else {
                r = journal_file_move_to_object(f, OBJECT_FIELD, p, &o);
                if (r < 0)
                        return r;
 
                o->field.next_hash_offset = htole64(offset);
        }
 
        f->field_hash_table[h].tail_hash_offset = htole64(offset);
 
        if (JOURNAL_HEADER_CONTAINS(f->header, n_fields))
                f->header->n_fields = htole64(le64toh(f->header->n_fields) + 1);
 
        return 0;
}
 
static int journal_file_link_data(
                JournalFile *f,
                Object *o,
                uint64_t offset,
                uint64_t hash) {
 
        uint64_t p, h, m;
        int r;
 
        assert(f);
        assert(f->header);
        assert(f->data_hash_table);
        assert(o);
        assert(offset > 0);
 
        if (o->object.type != OBJECT_DATA)
                return -EBADMSG;
 
        m = le64toh(READ_NOW(f->header->data_hash_table_size)) / sizeof(HashItem);
        if (m <= 0)
                return -EBADMSG;
 
        /* This might alter the window we are looking at */
        o->data.next_hash_offset = o->data.next_field_offset = 0;
        o->data.entry_offset = o->data.entry_array_offset = 0;
        o->data.n_entries = 0;
 
        h = hash % m;
        p = le64toh(f->data_hash_table[h].tail_hash_offset);
        if (p == 0)
                /* Only entry in the hash table is easy */
                f->data_hash_table[h].head_hash_offset = htole64(offset);
        else {
                /* Move back to the previous data object, to patch in
                 * pointer */
 
                r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
                if (r < 0)
                        return r;
 
                o->data.next_hash_offset = htole64(offset);
        }
 
        f->data_hash_table[h].tail_hash_offset = htole64(offset);
 
        if (JOURNAL_HEADER_CONTAINS(f->header, n_data))
                f->header->n_data = htole64(le64toh(f->header->n_data) + 1);
 
        return 0;
}
 
static int get_next_hash_offset(
                JournalFile *f,
                uint64_t *p,
                le64_t *next_hash_offset,
                uint64_t *depth,
                le64_t *header_max_depth) {
 
        uint64_t nextp;
 
        assert(f);
        assert(p);
        assert(next_hash_offset);
        assert(depth);
 
        nextp = le64toh(READ_NOW(*next_hash_offset));
        if (nextp > 0) {
                if (nextp <= *p) /* Refuse going in loops */
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Detected hash item loop in %s, refusing.", f->path);
 
                (*depth)++;
 
                /* If the depth of this hash chain is larger than all others we have seen so far, record it */
                if (header_max_depth && journal_file_writable(f))
                        *header_max_depth = htole64(MAX(*depth, le64toh(*header_max_depth)));
        }
 
        *p = nextp;
        return 0;
}
 
static bool chain_tail_lost(JournalFile *f, int r, uint64_t offset, ObjectType type) {
        assert(f);
 
        /* Only accept truncation when reading. On writing it's not possible to know how to safely proceed. */
        if (journal_file_writable(f) || !IN_SET(r, -EBADMSG, -EADDRNOTAVAIL))
                return false;
 
        log_debug_errno(r, "Failed to read %s at offset %" PRIu64 " of %s, treating it as the end of the chain: %m",
                        journal_object_type_to_string(type), offset, f->path);
        return true;
}
 
int journal_file_find_field_object_with_hash(
                JournalFile *f,
                const void *field,
                uint64_t size,
                uint64_t hash,
                Object **ret_object,
                uint64_t *ret_offset) {
 
        uint64_t p, osize, h, m, depth = 0;
        int r;
 
        assert(f);
        assert(f->header);
        assert(field);
        assert(size > 0);
 
        /* If the field hash table is empty, we can't find anything */
        if (le64toh(f->header->field_hash_table_size) <= 0)
                return 0;
 
        /* Map the field hash table, if it isn't mapped yet. */
        r = journal_file_map_field_hash_table(f);
        if (r < 0)
                return r;
 
        osize = offsetof(Object, field.payload) + size;
 
        m = le64toh(READ_NOW(f->header->field_hash_table_size)) / sizeof(HashItem);
        if (m <= 0)
                return -EBADMSG;
 
        h = hash % m;
        p = le64toh(f->field_hash_table[h].head_hash_offset);
        while (p > 0) {
                Object *o;
 
                r = journal_file_move_to_object(f, OBJECT_FIELD, p, &o);
                if (chain_tail_lost(f, r, p, OBJECT_FIELD))
                        break;
                if (r < 0)
                        return r;
 
                if (le64toh(o->field.hash) == hash &&
                    le64toh(o->object.size) == osize &&
                    memcmp(o->field.payload, field, size) == 0) {
 
                        if (ret_object)
                                *ret_object = o;
                        if (ret_offset)
                                *ret_offset = p;
 
                        return 1;
                }
 
                r = get_next_hash_offset(
                                f,
                                &p,
                                &o->field.next_hash_offset,
                                &depth,
                                JOURNAL_HEADER_CONTAINS(f->header, field_hash_chain_depth) ? &f->header->field_hash_chain_depth : NULL);
                if (r < 0)
                        return r;
        }
 
        return 0;
}
 
uint64_t journal_file_hash_data(
                JournalFile *f,
                const void *data,
                size_t sz) {
 
        assert(f);
        assert(f->header);
        assert(data || sz == 0);
 
        /* We try to unify our codebase on siphash, hence new-styled journal files utilizing the keyed hash
         * function use siphash. Old journal files use the Jenkins hash. */
 
        if (JOURNAL_HEADER_KEYED_HASH(f->header))
                return siphash24(data, sz, f->header->file_id.bytes);
 
        return jenkins_hash64(data, sz);
}
 
int journal_file_find_field_object(
                JournalFile *f,
                const void *field,
                uint64_t size,
                Object **ret_object,
                uint64_t *ret_offset) {
 
        assert(f);
        assert(field);
        assert(size > 0);
 
        return journal_file_find_field_object_with_hash(
                        f,
                        field, size,
                        journal_file_hash_data(f, field, size),
                        ret_object, ret_offset);
}
 
int journal_file_find_data_object_with_hash(
                JournalFile *f,
                const void *data,
                uint64_t size,
                uint64_t hash,
                Object **ret_object,
                uint64_t *ret_offset) {
 
        uint64_t p, h, m, depth = 0;
        int r;
 
        assert(f);
        assert(f->header);
        assert(data || size == 0);
 
        /* If there's no data hash table, then there's no entry. */
        if (le64toh(f->header->data_hash_table_size) <= 0)
                return 0;
 
        /* Map the data hash table, if it isn't mapped yet. */
        r = journal_file_map_data_hash_table(f);
        if (r < 0)
                return r;
 
        m = le64toh(READ_NOW(f->header->data_hash_table_size)) / sizeof(HashItem);
        if (m <= 0)
                return -EBADMSG;
 
        h = hash % m;
        p = le64toh(f->data_hash_table[h].head_hash_offset);
 
        while (p > 0) {
                Object *o;
                const void *d;
                size_t rsize;
 
                r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
                if (chain_tail_lost(f, r, p, OBJECT_DATA))
                        break;
                if (r < 0)
                        return r;
 
                if (le64toh(o->data.hash) != hash)
                        goto next;
 
                r = journal_file_data_payload(f, o, p, NULL, 0, 0, &d, &rsize);
                if (r < 0)
                        return r;
                assert(r > 0); /* journal_file_data_payload() always returns > 0 if no field is provided. */
 
                if (memcmp_nn(data, size, d, rsize) == 0) {
                         if (ret_object)
                                *ret_object = o;
 
                        if (ret_offset)
                                *ret_offset = p;
 
                        return 1;
                }
 
        next:
                r = get_next_hash_offset(
                                f,
                                &p,
                                &o->data.next_hash_offset,
                                &depth,
                                JOURNAL_HEADER_CONTAINS(f->header, data_hash_chain_depth) ? &f->header->data_hash_chain_depth : NULL);
                if (r < 0)
                        return r;
        }
 
        return 0;
}
 
int journal_file_find_data_object(
                JournalFile *f,
                const void *data,
                uint64_t size,
                Object **ret_object,
                uint64_t *ret_offset) {
 
        assert(f);
        assert(data || size == 0);
 
        return journal_file_find_data_object_with_hash(
                        f,
                        data, size,
                        journal_file_hash_data(f, data, size),
                        ret_object, ret_offset);
}
 
bool journal_field_valid(const char *p, size_t l, bool allow_protected) {
        /* We kinda enforce POSIX syntax recommendations for
           environment variables here, but make a couple of additional
           requirements.
 
           http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap08.html */
 
        assert(p);
 
        if (l == SIZE_MAX)
                l = strlen(p);
 
        /* No empty field names */
        if (l <= 0)
                return false;
 
        /* Don't allow names longer than 64 chars */
        if (l > 64)
                return false;
 
        /* Variables starting with an underscore are protected */
        if (!allow_protected && p[0] == '_')
                return false;
 
        /* Don't allow digits as first character */
        if (ascii_isdigit(p[0]))
                return false;
 
        /* Only allow A-Z0-9 and '_' */
        for (const char *a = p; a < p + l; a++)
                if ((*a < 'A' || *a > 'Z') &&
                    !ascii_isdigit(*a) &&
                    *a != '_')
                        return false;
 
        return true;
}
 
static int journal_file_append_field(
                JournalFile *f,
                const void *field,
                uint64_t size,
                Object **ret_object,
                uint64_t *ret_offset) {
 
        uint64_t hash, p;
        uint64_t osize;
        Object *o;
        int r;
 
        assert(f);
        assert(field);
        assert(size > 0);
 
        if (!journal_field_valid(field, size, true))
                return -EBADMSG;
 
        hash = journal_file_hash_data(f, field, size);
 
        r = journal_file_find_field_object_with_hash(f, field, size, hash, ret_object, ret_offset);
        if (r < 0)
                return r;
        if (r > 0)
                return 0;
 
        osize = offsetof(Object, field.payload) + size;
        r = journal_file_append_object(f, OBJECT_FIELD, osize, &o, &p);
        if (r < 0)
                return r;
 
        o->field.hash = htole64(hash);
        memcpy(o->field.payload, field, size);
 
        r = journal_file_link_field(f, o, p, hash);
        if (r < 0)
                return r;
 
        /* The linking might have altered the window, so let's only pass the offset to hmac which will
         * move to the object again if needed. */
 
        r = journal_file_auth_put_object(f, OBJECT_FIELD, NULL, p);
        if (r < 0)
                return r;
 
        if (ret_object) {
                r = journal_file_move_to_object(f, OBJECT_FIELD, p, ret_object);
                if (r < 0)
                        return r;
        }
 
        if (ret_offset)
                *ret_offset = p;
 
        return 0;
}
 
static int maybe_compress_payload(
                JournalFile *f,
                uint8_t *dst,
                const uint8_t *src,
                uint64_t size,
                size_t *rsize,
                Compression *ret_compression) {
 
        assert(f);
        assert(f->header);
        assert(ret_compression);
 
#if HAVE_COMPRESSION
        Compression c;
        int r;
 
        c = JOURNAL_FILE_COMPRESSION(f);
        if (c == COMPRESSION_NONE || size < f->compress_threshold_bytes) {
                *ret_compression = COMPRESSION_NONE;
                return 0;
        }
 
        r = compress_blob_journal(c, src, size, dst, size - 1, rsize, /* level= */ -1);
        if (r < 0)
                return log_debug_errno(r, "Failed to compress data object using %s, ignoring: %m", compression_to_string(c));
 
        log_debug("Compressed data object %"PRIu64" -> %zu using %s", size, *rsize, compression_to_string(c));
 
        *ret_compression = c;
        return 1; /* compressed */
#else
        *ret_compression = COMPRESSION_NONE;
        return 0;
#endif
}
 
static int journal_file_append_data(
                JournalFile *f,
                const void *data,
                uint64_t size,
                Object **ret_object,
                uint64_t *ret_offset) {
 
        uint64_t hash, p, osize;
        Object *o, *fo;
        size_t rsize = 0;
        const void *eq;
        int r;
 
        assert(f);
 
        /* Return a recognizable error for data that is submitted for *writing*. Note that we leave EBADMSG
         * for bad messages we *read*. And EINVAL is too generic an error (which might be triggered by all
         * kinds of other places). Hence use a separate EUCLEAN here. */
        if (!data || size == 0)
                return -EUCLEAN;
 
        hash = journal_file_hash_data(f, data, size);
 
        r = journal_file_find_data_object_with_hash(f, data, size, hash, ret_object, ret_offset);
        if (r < 0)
                return r;
        if (r > 0)
                return 0;
 
        eq = memchr(data, '=', size);
        if (!eq)
                return -EUCLEAN;
 
        osize = journal_file_data_payload_offset(f) + size;
        r = journal_file_append_object(f, OBJECT_DATA, osize, &o, &p);
        if (r < 0)
                return r;
 
        o->data.hash = htole64(hash);
 
        Compression c;
        r = maybe_compress_payload(f, journal_file_data_payload_field(f, o), data, size, &rsize, &c);
        if (r <= 0)
                /* We don't really care failures, let's continue without compression */
                memcpy_safe(journal_file_data_payload_field(f, o), data, size);
        else {
                assert(c >= 0 && c < _COMPRESSION_MAX && c != COMPRESSION_NONE);
 
                o->object.size = htole64(journal_file_data_payload_offset(f) + rsize);
                o->object.flags |= COMPRESSION_TO_OBJECT_FLAG(c);
        }
 
        r = journal_file_link_data(f, o, p, hash);
        if (r < 0)
                return r;
 
        /* The linking might have altered the window, so let's refresh our pointer. */
        r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
        if (r < 0)
                return r;
 
        r = journal_file_auth_put_object(f, OBJECT_DATA, o, p);
        if (r < 0)
                return r;
 
        /* Create field object ... */
        r = journal_file_append_field(f, data, (uint8_t*) eq - (uint8_t*) data, &fo, NULL);
        if (r < 0)
                return r;
 
        /* ... and link it in. */
        o->data.next_field_offset = fo->field.head_data_offset;
        fo->field.head_data_offset = htole64(p);
 
        if (ret_object)
                *ret_object = o;
 
        if (ret_offset)
                *ret_offset = p;
 
        return 0;
}
 
static int maybe_decompress_payload(
                JournalFile *f,
                const uint8_t *payload,
                uint64_t size,
                Compression compression,
                const char *field,
                size_t field_length,
                size_t data_threshold,
                const void **ret_data,
                size_t *ret_size) {
 
        assert(f);
 
        /* We can't read objects larger than 4G on a 32-bit machine */
#if __SIZEOF_SIZE_T__ == 4
        if (size > UINT32_MAX)
                return -E2BIG;
#endif
 
        if (compression != COMPRESSION_NONE) {
#if HAVE_COMPRESSION
                size_t rsize;