From 9204fc642acef5cfc3a411fdec3ce9a5fd9f8d37 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 26 Jan 2023 16:49:36 +0100 Subject: [PATCH 1/6] journal-file: don't update boot_id in journal header on open The header of the journal file contains a boot ID field that is currently updated whenever we open the journal file. This is not ideal: pretty often we want to archive a journal file, and need to open it for that. Archiving a foreign journal file should not mark it as ours, it should just change the status flag in the file header. The boot ID in the header is aleady rewritten whenever we write a journal entry to the file anyway, hence all this patch effectively does is slightly "delay" when the boot ID in the header is updated: instead of immediately on open it is updated on the first entry that is written. Net effect: archived journal files don't all look like they were written to on a boot newer then they actually were And more importantly: the "tail_entry_monotonic" field suddenly becomes useful, since we know which boot it belongs to. Generally, monotonic timestamps without boot ID information are useless, and this fixes it. A new (compatible) header flag marks file where the boot_id can be understood this way. This can be used by code that wants to make use of the "tail_entry_monotonic" field to ensure it actually can do so safely. This also renames the structure definition in journal-def accordingly, to indicate we now follow the stricter semantics for it. --- src/journal/test-journal-interleaving.c | 4 +- .../sd-journal/journal-authenticate.c | 2 +- src/libsystemd/sd-journal/journal-def.h | 41 +++++++++---------- src/libsystemd/sd-journal/journal-file.c | 22 +++++----- src/libsystemd/sd-journal/journal-file.h | 3 ++ src/libsystemd/sd-journal/journal-verify.c | 3 +- 6 files changed, 40 insertions(+), 35 deletions(-) diff --git a/src/journal/test-journal-interleaving.c b/src/journal/test-journal-interleaving.c index 55d717da31..7fec6d9eea 100644 --- a/src/journal/test-journal-interleaving.c +++ b/src/journal/test-journal-interleaving.c @@ -230,7 +230,7 @@ static void test_sequence_numbers_one(void) { assert_se(one->file->header->state == STATE_ONLINE); assert_se(!sd_id128_equal(one->file->header->file_id, one->file->header->machine_id)); - assert_se(!sd_id128_equal(one->file->header->file_id, one->file->header->boot_id)); + assert_se(!sd_id128_equal(one->file->header->file_id, one->file->header->tail_entry_boot_id)); assert_se(sd_id128_equal(one->file->header->file_id, one->file->header->seqnum_id)); memcpy(&seqnum_id, &one->file->header->seqnum_id, sizeof(sd_id128_t)); @@ -241,7 +241,7 @@ static void test_sequence_numbers_one(void) { assert_se(two->file->header->state == STATE_ONLINE); assert_se(!sd_id128_equal(two->file->header->file_id, one->file->header->file_id)); assert_se(sd_id128_equal(one->file->header->machine_id, one->file->header->machine_id)); - assert_se(sd_id128_equal(one->file->header->boot_id, one->file->header->boot_id)); + assert_se(sd_id128_equal(one->file->header->tail_entry_boot_id, one->file->header->tail_entry_boot_id)); assert_se(sd_id128_equal(one->file->header->seqnum_id, one->file->header->seqnum_id)); append_number(two, 3, &seqnum); diff --git a/src/libsystemd/sd-journal/journal-authenticate.c b/src/libsystemd/sd-journal/journal-authenticate.c index 3c5d9d7e49..159e215367 100644 --- a/src/libsystemd/sd-journal/journal-authenticate.c +++ b/src/libsystemd/sd-journal/journal-authenticate.c @@ -300,7 +300,7 @@ int journal_file_hmac_put_header(JournalFile *f) { * n_entry_arrays. */ gcry_md_write(f->hmac, f->header->signature, offsetof(Header, state) - offsetof(Header, signature)); - gcry_md_write(f->hmac, &f->header->file_id, offsetof(Header, boot_id) - offsetof(Header, file_id)); + gcry_md_write(f->hmac, &f->header->file_id, offsetof(Header, tail_entry_boot_id) - offsetof(Header, file_id)); gcry_md_write(f->hmac, &f->header->seqnum_id, offsetof(Header, arena_size) - offsetof(Header, seqnum_id)); gcry_md_write(f->hmac, &f->header->data_hash_table_offset, offsetof(Header, tail_object_offset) - offsetof(Header, data_hash_table_offset)); diff --git a/src/libsystemd/sd-journal/journal-def.h b/src/libsystemd/sd-journal/journal-def.h index d35290d3c7..fb22fc45f3 100644 --- a/src/libsystemd/sd-journal/journal-def.h +++ b/src/libsystemd/sd-journal/journal-def.h @@ -173,32 +173,31 @@ enum { HEADER_INCOMPATIBLE_KEYED_HASH = 1 << 2, HEADER_INCOMPATIBLE_COMPRESSED_ZSTD = 1 << 3, HEADER_INCOMPATIBLE_COMPACT = 1 << 4, + + HEADER_INCOMPATIBLE_ANY = HEADER_INCOMPATIBLE_COMPRESSED_XZ | + HEADER_INCOMPATIBLE_COMPRESSED_LZ4 | + HEADER_INCOMPATIBLE_KEYED_HASH | + HEADER_INCOMPATIBLE_COMPRESSED_ZSTD | + HEADER_INCOMPATIBLE_COMPACT, + + HEADER_INCOMPATIBLE_SUPPORTED = (HAVE_XZ ? HEADER_INCOMPATIBLE_COMPRESSED_XZ : 0) | + (HAVE_LZ4 ? HEADER_INCOMPATIBLE_COMPRESSED_LZ4 : 0) | + (HAVE_ZSTD ? HEADER_INCOMPATIBLE_COMPRESSED_ZSTD : 0) | + HEADER_INCOMPATIBLE_KEYED_HASH | + HEADER_INCOMPATIBLE_COMPACT, }; -#define HEADER_INCOMPATIBLE_ANY \ - (HEADER_INCOMPATIBLE_COMPRESSED_XZ | \ - HEADER_INCOMPATIBLE_COMPRESSED_LZ4 | \ - HEADER_INCOMPATIBLE_KEYED_HASH | \ - HEADER_INCOMPATIBLE_COMPRESSED_ZSTD | \ - HEADER_INCOMPATIBLE_COMPACT) - -#define HEADER_INCOMPATIBLE_SUPPORTED \ - ((HAVE_XZ ? HEADER_INCOMPATIBLE_COMPRESSED_XZ : 0) | \ - (HAVE_LZ4 ? HEADER_INCOMPATIBLE_COMPRESSED_LZ4 : 0) | \ - (HAVE_ZSTD ? HEADER_INCOMPATIBLE_COMPRESSED_ZSTD : 0) | \ - HEADER_INCOMPATIBLE_KEYED_HASH | \ - HEADER_INCOMPATIBLE_COMPACT) enum { - HEADER_COMPATIBLE_SEALED = 1 << 0, + HEADER_COMPATIBLE_SEALED = 1 << 0, + HEADER_COMPATIBLE_TAIL_ENTRY_BOOT_ID = 1 << 1, /* if set, the last_entry_boot_id field in the header is exclusively refreshed when an entry is appended */ + HEADER_COMPATIBLE_ANY = HEADER_COMPATIBLE_SEALED| + HEADER_COMPATIBLE_TAIL_ENTRY_BOOT_ID, + + HEADER_COMPATIBLE_SUPPORTED = (HAVE_GCRYPT ? HEADER_COMPATIBLE_SEALED : 0) | + HEADER_COMPATIBLE_TAIL_ENTRY_BOOT_ID, }; -#define HEADER_COMPATIBLE_ANY HEADER_COMPATIBLE_SEALED -#if HAVE_GCRYPT -# define HEADER_COMPATIBLE_SUPPORTED HEADER_COMPATIBLE_SEALED -#else -# define HEADER_COMPATIBLE_SUPPORTED 0 -#endif #define HEADER_SIGNATURE \ ((const uint8_t[]) { 'L', 'P', 'K', 'S', 'H', 'H', 'R', 'H' }) @@ -211,7 +210,7 @@ enum { uint8_t reserved[7]; \ sd_id128_t file_id; \ sd_id128_t machine_id; \ - sd_id128_t boot_id; /* last writer */ \ + sd_id128_t tail_entry_boot_id; \ sd_id128_t seqnum_id; \ le64_t header_size; \ le64_t arena_size; \ diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index aab33dbfcc..d361b35a6e 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -357,7 +357,9 @@ static int journal_file_init_header( FLAGS_SET(file_flags, JOURNAL_COMPRESS) * COMPRESSION_TO_HEADER_INCOMPATIBLE_FLAG(DEFAULT_COMPRESSION) | keyed_hash_requested() * HEADER_INCOMPATIBLE_KEYED_HASH | compact_mode_requested() * HEADER_INCOMPATIBLE_COMPACT), - .compatible_flags = htole32(seal * HEADER_COMPATIBLE_SEALED), + .compatible_flags = htole32( + (seal * HEADER_COMPATIBLE_SEALED) | + HEADER_COMPATIBLE_TAIL_ENTRY_BOOT_ID), }; assert_cc(sizeof(h.signature) == sizeof(HEADER_SIGNATURE)); @@ -397,9 +399,8 @@ static int journal_file_refresh_header(JournalFile *f) { f->header->machine_id = SD_ID128_NULL; } - r = sd_id128_get_boot(&f->header->boot_id); - if (r < 0) - return r; + /* We used to update the header's boot ID field here, but we don't do that anymore, as per + * HEADER_COMPATIBLE_TAIL_ENTRY_BOOT_ID */ r = journal_file_set_online(f); @@ -2126,9 +2127,9 @@ static int journal_file_append_entry_internal( "timestamp %" PRIu64 ", refusing entry.", ts->realtime, le64toh(f->header->tail_entry_realtime)); - if (!sd_id128_is_null(f->header->boot_id) && boot_id) { + if (!sd_id128_is_null(f->header->tail_entry_boot_id) && boot_id) { - if (!sd_id128_equal(f->header->boot_id, *boot_id)) + if (!sd_id128_equal(f->header->tail_entry_boot_id, *boot_id)) return log_debug_errno(SYNTHETIC_ERRNO(EREMOTE), "Boot ID to write is different from previous boot id, refusing entry."); @@ -2166,8 +2167,8 @@ static int journal_file_append_entry_internal( o->entry.monotonic = htole64(ts->monotonic); o->entry.xor_hash = htole64(xor_hash); if (boot_id) - f->header->boot_id = *boot_id; - o->entry.boot_id = f->header->boot_id; + f->header->tail_entry_boot_id = *boot_id; + o->entry.boot_id = f->header->tail_entry_boot_id; for (size_t i = 0; i < n_items; i++) write_entry_item(f, o, i, &items[i]); @@ -3567,7 +3568,7 @@ void journal_file_print_header(JournalFile *f) { "Boot ID: %s\n" "Sequential number ID: %s\n" "State: %s\n" - "Compatible flags:%s%s\n" + "Compatible flags:%s%s%s\n" "Incompatible flags:%s%s%s%s%s%s\n" "Header size: %"PRIu64"\n" "Arena size: %"PRIu64"\n" @@ -3584,12 +3585,13 @@ void journal_file_print_header(JournalFile *f) { f->path, SD_ID128_TO_STRING(f->header->file_id), SD_ID128_TO_STRING(f->header->machine_id), - SD_ID128_TO_STRING(f->header->boot_id), + SD_ID128_TO_STRING(f->header->tail_entry_boot_id), SD_ID128_TO_STRING(f->header->seqnum_id), f->header->state == STATE_OFFLINE ? "OFFLINE" : f->header->state == STATE_ONLINE ? "ONLINE" : f->header->state == STATE_ARCHIVED ? "ARCHIVED" : "UNKNOWN", JOURNAL_HEADER_SEALED(f->header) ? " SEALED" : "", + JOURNAL_HEADER_TAIL_ENTRY_BOOT_ID(f->header) ? " TAIL_ENTRY_BOOT_ID" : "", (le32toh(f->header->compatible_flags) & ~HEADER_COMPATIBLE_ANY) ? " ???" : "", JOURNAL_HEADER_COMPRESSED_XZ(f->header) ? " COMPRESSED-XZ" : "", JOURNAL_HEADER_COMPRESSED_LZ4(f->header) ? " COMPRESSED-LZ4" : "", diff --git a/src/libsystemd/sd-journal/journal-file.h b/src/libsystemd/sd-journal/journal-file.h index 07f1f5d180..c1f1ab4e4f 100644 --- a/src/libsystemd/sd-journal/journal-file.h +++ b/src/libsystemd/sd-journal/journal-file.h @@ -180,6 +180,9 @@ static inline bool VALID_EPOCH(uint64_t u) { #define JOURNAL_HEADER_SEALED(h) \ FLAGS_SET(le32toh((h)->compatible_flags), HEADER_COMPATIBLE_SEALED) +#define JOURNAL_HEADER_TAIL_ENTRY_BOOT_ID(h) \ + FLAGS_SET(le32toh((h)->compatible_flags), HEADER_COMPATIBLE_TAIL_ENTRY_BOOT_ID) + #define JOURNAL_HEADER_COMPRESSED_XZ(h) \ FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_COMPRESSED_XZ) diff --git a/src/libsystemd/sd-journal/journal-verify.c b/src/libsystemd/sd-journal/journal-verify.c index b4ce3881a4..8232f53eb6 100644 --- a/src/libsystemd/sd-journal/journal-verify.c +++ b/src/libsystemd/sd-journal/journal-verify.c @@ -1297,7 +1297,8 @@ int journal_file_verify( } if (entry_monotonic_set && - (sd_id128_equal(entry_boot_id, f->header->boot_id) && + (sd_id128_equal(entry_boot_id, f->header->tail_entry_boot_id) && + JOURNAL_HEADER_TAIL_ENTRY_BOOT_ID(f->header) && entry_monotonic != le64toh(f->header->tail_entry_monotonic))) { error(0, "Invalid tail monotonic timestamp (%"PRIu64" != %"PRIu64")", From 8e64ec04705e029cc891cd1382865ce25b04685c Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 26 Jan 2023 17:12:25 +0100 Subject: [PATCH 2/6] journal-file: write machine ID when create the file, not when we open it for writing This doesn't actually change much, but makes the code less surprising. Status quo ante: 1. Open a journal file 2. If newly created set header machine ID to zero 3. If existing and open for write check if machine ID in header matches local one, if not, refuse. 4. if open for writing, now refresh the machine ID from the local system Of course, step 4 is pretty much pointless for existing files, as the check in 3 made sure it is already in order or we'd refuse operating on it anyway. With this patch this is simplified to: 1. Open a journal file 2. If newly created initialized machine ID to local machine ID 3. If existing, compare machine ID in header with local one, if not matching refuse. Outcome is the same. --- src/libsystemd/sd-journal/journal-file.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index d361b35a6e..7300ed0781 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -369,6 +369,11 @@ static int journal_file_init_header( if (r < 0) return r; + r = sd_id128_get_machine(&h.machine_id); + if (r < 0 && !ERRNO_IS_MACHINE_ID_UNSET(r)) + return r; /* If we have no valid machine ID (test environment?), let's simply leave the + * machine ID field all zeroes. */ + if (template) { h.seqnum_id = template->header->seqnum_id; h.tail_entry_seqnum = template->header->tail_entry_seqnum; @@ -390,15 +395,6 @@ static int journal_file_refresh_header(JournalFile *f) { assert(f); assert(f->header); - r = sd_id128_get_machine(&f->header->machine_id); - if (r < 0) { - if (!ERRNO_IS_MACHINE_ID_UNSET(r)) - return r; - - /* don't have a machine-id, let's continue without */ - f->header->machine_id = SD_ID128_NULL; - } - /* We used to update the header's boot ID field here, but we don't do that anymore, as per * HEADER_COMPATIBLE_TAIL_ENTRY_BOOT_ID */ From 51ab0afed48dc55a211fbb610e188221446eb61f Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Tue, 31 Jan 2023 13:37:12 +0100 Subject: [PATCH 3/6] journal-file: lazily fill in machine ID into journal header, if needed Previously, if we ran in an environment where /etc/machine-id was not defined, we'd never bother to write it ever again. So it would stay at all zeroes till the end of times. Let's make this more robust: whenever we try to append an entry, let's try to refresh it from the status quo if not initialized yet. Moreover, when copying records from a different journal file, let's propagate the machine ID from there. This should make things more robust and systematic, and match how we propagate the boot ID and the seqnum ID to some level. --- src/libsystemd/sd-journal/journal-file.c | 43 ++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index 7300ed0781..3e721ef937 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -2087,6 +2087,7 @@ static int journal_file_append_entry_internal( JournalFile *f, const dual_timestamp *ts, const sd_id128_t *boot_id, + const sd_id128_t *machine_id, uint64_t xor_hash, const EntryItem items[], size_t n_items, @@ -2152,6 +2153,10 @@ static int journal_file_append_entry_internal( } } + if (machine_id && sd_id128_is_null(f->header->machine_id)) + /* Initialize machine ID when not set yet */ + f->header->machine_id = *machine_id; + osize = offsetof(Object, entry.items) + (n_items * journal_file_entry_item_size(f)); r = journal_file_append_object(f, OBJECT_ENTRY, osize, &o, &np); @@ -2311,7 +2316,7 @@ int journal_file_append_entry( EntryItem *items; uint64_t xor_hash = 0; struct dual_timestamp _ts; - sd_id128_t _boot_id; + sd_id128_t _boot_id, _machine_id, *machine_id; int r; assert(f); @@ -2341,6 +2346,16 @@ int journal_file_append_entry( boot_id = &_boot_id; } + r = sd_id128_get_machine(&_machine_id); + if (r < 0) { + if (!ERRNO_IS_MACHINE_ID_UNSET(r)) + return r; + + /* If the machine ID is not initialized yet, handle gracefully */ + machine_id = NULL; + } else + machine_id = &_machine_id; + #if HAVE_GCRYPT r = journal_file_maybe_append_tag(f, ts->realtime); if (r < 0) @@ -2390,7 +2405,18 @@ int journal_file_append_entry( typesafe_qsort(items, n_iovec, entry_item_cmp); n_iovec = remove_duplicate_entry_items(items, n_iovec); - r = journal_file_append_entry_internal(f, ts, boot_id, xor_hash, items, n_iovec, seqnum, seqnum_id, ret_object, ret_offset); + r = journal_file_append_entry_internal( + f, + ts, + boot_id, + machine_id, + xor_hash, + items, + n_iovec, + seqnum, + seqnum_id, + ret_object, + ret_offset); /* If the memory mapping triggered a SIGBUS then we return an * IO error and ignore the error code passed down to us, since @@ -4190,7 +4216,18 @@ int journal_file_copy_entry( return r; } - r = journal_file_append_entry_internal(to, &ts, boot_id, xor_hash, items, n, seqnum, seqnum_id, NULL, NULL); + r = journal_file_append_entry_internal( + to, + &ts, + boot_id, + &from->header->machine_id, + xor_hash, + items, + n, + seqnum, + seqnum_id, + /* ret_object= */ NULL, + /* ret_offset= */ NULL); if (mmap_cache_fd_got_sigbus(to->cache_fd)) return -EIO; From 07f1c7aa9db7a98e0ba1eb1b0d42f171925fc5eb Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 1 Feb 2023 12:23:54 +0100 Subject: [PATCH 4/6] journal-file: allow opening journal files for write when machine ID is not initialized MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We allow reading them, and we allow creating them, but we so far did not allow opening existing ones for write – if the machine ID is not initialized. Let's fix that. (This is just to fix an asymmetry. I have no immediate use for this. But test code should in theory be able to use this, if it runs in an incompletely initialized environment.) --- src/libsystemd/sd-journal/journal-file.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index 3e721ef937..05c66815ae 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -511,8 +511,12 @@ static int journal_file_verify_header(JournalFile *f) { int r; r = sd_id128_get_machine(&machine_id); - if (r < 0) - return r; + if (r < 0) { + if (!ERRNO_IS_MACHINE_ID_UNSET(r)) /* handle graceful if machine ID is not initialized yet */ + return r; + + machine_id = SD_ID128_NULL; + } if (!sd_id128_equal(machine_id, f->header->machine_id)) return log_debug_errno(SYNTHETIC_ERRNO(EHOSTDOWN), From ced1196802035ea07290f49009886047a513fe32 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 26 Jan 2023 17:24:15 +0100 Subject: [PATCH 5/6] journal-file: drop checking if files are from the future at time of open We nowadays check for ordering anyway at time of writing entries, hence we don't have to do that at moment of opening, too. Benefit of dropping this check: we can safely archive files from the future instead of marking them as broken. --- src/journal/journald-server.c | 4 ---- src/journal/managed-journal-file.c | 3 +-- src/libsystemd/sd-journal/journal-file.c | 8 -------- 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index dd31007a4d..de08c4e965 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -873,10 +873,6 @@ static bool shall_try_append_again(JournalFile *f, int r) { log_ratelimit_warning(JOURNAL_LOG_RATELIMIT, "%s: Journal file has been deleted, rotating.", f->path); return true; - case -ETXTBSY: /* Journal file is from the future */ - log_ratelimit_warning(JOURNAL_LOG_RATELIMIT, "%s: Journal file is from the future, rotating.", f->path); - return true; - case -EREMCHG: /* Wallclock time (CLOCK_REALTIME) jumped backwards relative to last journal entry */ log_ratelimit_warning(JOURNAL_LOG_RATELIMIT, "%s: Realtime clock jumped backwards relative to last journal entry, rotating.", f->path); return true; diff --git a/src/journal/managed-journal-file.c b/src/journal/managed-journal-file.c index 81aecfe7cb..538d999de0 100644 --- a/src/journal/managed-journal-file.c +++ b/src/journal/managed-journal-file.c @@ -542,8 +542,7 @@ int managed_journal_file_open_reliably( -EBUSY, /* Unclean shutdown */ -ESHUTDOWN, /* Already archived */ -EIO, /* IO error, including SIGBUS on mmap */ - -EIDRM, /* File has been deleted */ - -ETXTBSY)) /* File is from the future */ + -EIDRM)) /* File has been deleted */ return r; if ((open_flags & O_ACCMODE) == O_RDONLY) diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index 05c66815ae..b1064a6982 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -537,14 +537,6 @@ static int journal_file_verify_header(JournalFile *f) { if (f->header->field_hash_table_size == 0 || f->header->data_hash_table_size == 0) return -EBADMSG; - - /* Don't permit appending to files from the future. Because otherwise the realtime timestamps wouldn't - * be strictly ordered in the entries in the file anymore, and we can't have that since it breaks - * bisection. */ - if (le64toh(f->header->tail_entry_realtime) > now(CLOCK_REALTIME)) - return log_debug_errno(SYNTHETIC_ERRNO(ETXTBSY), - "Journal file %s is from the future, refusing to append new data to it that'd be older.", - f->path); } return 0; From f010478168c3ba4abe66df60b6393db8f5e2f217 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Tue, 31 Jan 2023 16:09:58 +0100 Subject: [PATCH 6/6] docs: document the new HEADER_COMPATIBLE_TAIL_ENTRY_BOOT_ID flag --- docs/JOURNAL_FILE_FORMAT.md | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/JOURNAL_FILE_FORMAT.md b/docs/JOURNAL_FILE_FORMAT.md index 2d0debd858..712f3bce36 100644 --- a/docs/JOURNAL_FILE_FORMAT.md +++ b/docs/JOURNAL_FILE_FORMAT.md @@ -151,7 +151,7 @@ _packed_ struct Header { uint8_t reserved[7]; sd_id128_t file_id; sd_id128_t machine_id; - sd_id128_t boot_id; /* last writer */ + sd_id128_t tail_entry_boot_id; sd_id128_t seqnum_id; le64_t header_size; le64_t arena_size; @@ -192,8 +192,18 @@ new one. When journal file is first created the **file_id** is randomly and uniquely initialized. -When a writer opens a file it shall initialize the **boot_id** to the current -boot id of the system. +When a writer creates a file it shall initialize the **tail_entry_boot_id** to +the current boot ID of the system. When appending an entry it shall update the +field to the boot ID of that entry, so that it is guaranteed that the +**tail_entry_monotonic** field refers to a timestamp of the monotonic clock +associated with the boot with the ID indicated by the **tail_entry_boot_id** +field. (Compatibility note: in older versions of the journal, the field was +also supposed to be updated whenever the file was opened for any form of +writing, including when opened to mark it as archived. This behaviour has been +deemed problematic since without an associated boot ID the +**tail_entry_monotonic** field is useless. To indicate whether the boot ID is +updated only on append the JOURNAL_COMPATIBLE_TAIL_ENTRY_BOOT_ID is set. If it +is not set, the **tail_entry_monotonic** field is not usable). The currently used part of the file is the **header_size** plus the **arena_size** field of the header. If a writer needs to write to a file where @@ -222,7 +232,12 @@ timestamp of the last or first entry in the file, respectively, or 0 if no entry has been written yet. **tail_entry_monotonic** is the monotonic timestamp of the last entry in the -file, referring to monotonic time of the boot identified by **boot_id**. +file, referring to monotonic time of the boot identified by +**tail_entry_boot_id**, but only if the +JOURNAL_COMPATIBLE_TAIL_ENTRY_BOOT_ID feature flag is set, see above. If it +is not set, this field might refer to a different boot then the one in the +**tail_entry_boot_id** field, for example when the file was ultimately +archived. **data_hash_chain_depth** is a counter of the deepest chain in the data hash table, minus one. This is updated whenever a chain is found that is longer than @@ -268,7 +283,8 @@ enum { }; enum { - HEADER_COMPATIBLE_SEALED = 1 << 0, + HEADER_COMPATIBLE_SEALED = 1 << 0, + HEADER_COMPATIBLE_TAIL_ENTRY_BOOT_ID = 1 << 1, }; ``` @@ -288,6 +304,12 @@ format that uses less space on disk compared to the original format. HEADER_COMPATIBLE_SEALED indicates that the file includes TAG objects required for Forward Secure Sealing. +HEADER_COMPATIBLE_TAIL_ENTRY_BOOT_ID indicates whether the +**tail_entry_boot_id** field is strictly updated on initial creation of the +file and whenever an entry is updated (in which case the flag is set), or also +when the file is archived (in which case it is unset). New files should always +set this flag (and thus not update the **tail_entry_boot_id** except when +creating the file and when appending an entry to it. ## Dirty Detection