dasharo/systemd
0
0
Fork 0
mirror of https://github.com/Dasharo/systemd.git synced 2026-03-06 15:02:31 -08:00
Code Issues Packages Projects Releases Wiki Activity

journald: add minimal client metadata caching

Browse Source 
Cache client metadata, in order to be improve runtime behaviour under
pressure.

This is inspired by @vcaputo's work, specifically:

https://github.com/systemd/systemd/pull/2280

That code implements related but different semantics.

For a longer explanation what this change implements please have a look
at the long source comment this patch adds to journald-context.c.

After this commit:

        # time bash -c 'dd bs=$((1024*1024)) count=$((1*1024)) if=/dev/urandom | systemd-cat'
        1024+0 records in
        1024+0 records out
        1073741824 bytes (1.1 GB, 1.0 GiB) copied, 11.2783 s, 95.2 MB/s

        real	0m11.283s
        user	0m0.007s
        sys	0m6.216s

Before this commit:

        # time bash -c 'dd bs=$((1024*1024)) count=$((1*1024)) if=/dev/urandom | systemd-cat'
        1024+0 records in
        1024+0 records out
        1073741824 bytes (1.1 GB, 1.0 GiB) copied, 52.0788 s, 20.6 MB/s

        real	0m52.099s
        user	0m0.014s
        sys	0m7.170s

As side effect, this corrects the journal's rate limiter feature: we now
always use the unit name as key for the ratelimiter.
This commit is contained in:
Lennart Poettering
2017-07-17 23:36:35 +02:00
parent 47b33c7d52
commit 22e3a02b9d
10 changed files with 852 additions and 372 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/journal/journald-audit.c
View File

@@ -413,7 +413,7 @@ static void process_audit_string(Server *s, int type, const char *data, size_t s
goto finish;
}
server_dispatch_message(s, iov, n_iov, n_iov_allocated, NULL, NULL, NULL, 0, NULL, LOG_NOTICE, 0);
server_dispatch_message(s, iov, n_iov, n_iov_allocated, NULL, NULL, LOG_NOTICE, 0);
finish:
/* free() all entries that map_all_fields() added. All others

588
src/journal/journald-context.c Normal file
View File

File diff suppressed because it is too large Load Diff

92
src/journal/journald-context.h Normal file
View File

@@ -0,0 +1,92 @@
#pragma once
/***
This file is part of systemd.
Copyright 2017 Lennart Poettering
systemd is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
systemd is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/
#include <inttypes.h>
#include <sys/types.h>
#include "sd-id128.h"
typedef struct ClientContext ClientContext;
#include "journald-server.h"
struct ClientContext {
unsigned n_ref;
unsigned lru_index;
usec_t timestamp;
bool in_lru;
pid_t pid;
uid_t uid;
gid_t gid;
char *comm;
char *exe;
char *cmdline;
char *capeff;
uint32_t auditid;
uid_t loginuid;
char *cgroup;
char *session;
uid_t owner_uid;
char *unit;
char *user_unit;
char *slice;
char *user_slice;
sd_id128_t invocation_id;
char *label;
size_t label_size;
};
int client_context_get(
Server *s,
pid_t pid,
const struct ucred *ucred,
const char *label, size_t label_len,
const char *unit_id,
ClientContext **ret);
int client_context_acquire(
Server *s,
pid_t pid,
const struct ucred *ucred,
const char *label, size_t label_len,
const char *unit_id,
ClientContext **ret);
ClientContext* client_context_release(Server *s, ClientContext *c);
void client_context_maybe_refresh(
Server *s,
ClientContext *c,
const struct ucred *ucred,
const char *label, size_t label_size,
const char *unit_id,
usec_t tstamp);
void client_context_acquire_default(Server *s);
void client_context_flush_all(Server *s);

2
src/journal/journald-kmsg.c
View File

@@ -310,7 +310,7 @@ static void dev_kmsg_record(Server *s, const char *p, size_t l) {
if (cunescape_length_with_prefix(p, pl, "MESSAGE=", UNESCAPE_RELAX, &message) >= 0)
IOVEC_SET_STRING(iovec[n++], message);
server_dispatch_message(s, iovec, n, ELEMENTSOF(iovec), NULL, NULL, NULL, 0, NULL, priority, 0);
server_dispatch_message(s, iovec, n, ELEMENTSOF(iovec), NULL, NULL, priority, 0);
finish:
for (j = 0; j < z; j++)

15
src/journal/journald-native.c
View File

@@ -37,6 +37,7 @@
#include "memfd-util.h"
#include "parse-util.h"
#include "path-util.h"
#include "process-util.h"
#include "selinux-util.h"
#include "socket-util.h"
#include "string-util.h"
@@ -142,6 +143,7 @@ static void server_process_entry_meta(
static int server_process_entry(
Server *s,
const void *buffer, size_t *remaining,
ClientContext *context,
const struct ucred *ucred,
const struct timeval *tv,
const char *label, size_t label_len) {
@@ -303,7 +305,7 @@ static int server_process_entry(
server_forward_wall(s, priority, identifier, message, ucred);
}
server_dispatch_message(s, iovec, n, m, ucred, tv, label, label_len, NULL, priority, object_pid);
server_dispatch_message(s, iovec, n, m, context, tv, priority, object_pid);
finish:
for (j = 0; j < n; j++) {
@@ -329,16 +331,23 @@ void server_process_native_message(
const struct timeval *tv,
const char *label, size_t label_len) {
int r;
size_t remaining = buffer_size;
ClientContext *context;
int r;
assert(s);
assert(buffer || buffer_size == 0);
if (ucred && pid_is_valid(ucred->pid)) {
r = client_context_get(s, ucred->pid, ucred, label, label_len, NULL, &context);
if (r < 0)
log_warning_errno(r, "Failed to retrieve credentials for PID " PID_FMT ", ignoring: %m", ucred->pid);
}
do {
r = server_process_entry(s,
(const uint8_t*) buffer + (buffer_size - remaining), &remaining,
ucred, tv, label, label_len);
context, ucred, tv, label, label_len);
} while (r == 0);
}

448
src/journal/journald-server.c
View File

File diff suppressed because it is too large Load Diff

11
src/journal/journald-server.h
View File

@@ -28,9 +28,11 @@ typedef struct Server Server;
#include "hashmap.h"
#include "journal-file.h"
#include "journald-context.h"
#include "journald-rate-limit.h"
#include "journald-stream.h"
#include "list.h"
#include "prioq.h"
typedef enum Storage {
STORAGE_AUTO,
@@ -166,6 +168,13 @@ struct Server {
usec_t watchdog_usec;
usec_t last_realtime_clock;
/* Caching of client metadata */
Hashmap *client_contexts;
Prioq *client_contexts_lru;
ClientContext *my_context; /* the context of journald itself */
ClientContext *pid1_context; /* the context of PID 1 */
};
#define SERVER_MACHINE_ID(s) ((s)->machine_id_field + strlen("_MACHINE_ID="))
@@ -176,7 +185,7 @@ struct Server {
#define N_IOVEC_OBJECT_FIELDS 14
#define N_IOVEC_PAYLOAD_FIELDS 15
void server_dispatch_message(Server *s, struct iovec *iovec, unsigned n, unsigned m, const struct ucred *ucred, const struct timeval *tv, const char *label, size_t label_len, const char *unit_id, int priority, pid_t object_pid);
void server_dispatch_message(Server *s, struct iovec *iovec, unsigned n, unsigned m, ClientContext *c, const struct timeval *tv, int priority, pid_t object_pid);
void server_driver_message(Server *s, const char *message_id, const char *format, ...) _printf_(3,0) _sentinel_;
/* gperf lookup function */

21
src/journal/journald-stream.c
View File

@@ -34,6 +34,7 @@
#include "fileio.h"
#include "io-util.h"
#include "journald-console.h"
#include "journald-context.h"
#include "journald-kmsg.h"
#include "journald-server.h"
#include "journald-stream.h"
@@ -41,6 +42,7 @@
#include "journald-wall.h"
#include "mkdir.h"
#include "parse-util.h"
#include "process-util.h"
#include "selinux-util.h"
#include "socket-util.h"
#include "stdio-util.h"
@@ -87,6 +89,8 @@ struct StdoutStream {
char *state_file;
ClientContext *context;
LIST_FIELDS(StdoutStream, stdout_stream);
LIST_FIELDS(StdoutStream, stdout_stream_notify_queue);
};
@@ -96,6 +100,10 @@ void stdout_stream_free(StdoutStream *s) {
return;
if (s->server) {
if (s->context)
client_context_release(s->server, s->context);
assert(s->server->n_stdout_streams > 0);
s->server->n_stdout_streams--;
LIST_REMOVE(stdout_stream, s->server->stdout_streams, s);
@@ -233,7 +241,7 @@ static int stdout_stream_log(StdoutStream *s, const char *p) {
char syslog_facility[sizeof("SYSLOG_FACILITY=")-1 + DECIMAL_STR_MAX(int) + 1];
_cleanup_free_ char *message = NULL, *syslog_identifier = NULL;
unsigned n = 0;
size_t label_len;
int r;
assert(s);
assert(p);
@@ -278,8 +286,15 @@ static int stdout_stream_log(StdoutStream *s, const char *p) {
if (message)
IOVEC_SET_STRING(iovec[n++], message);
label_len = s->label ? strlen(s->label) : 0;
server_dispatch_message(s->server, iovec, n, ELEMENTSOF(iovec), &s->ucred, NULL, s->label, label_len, s->unit_id, priority, 0);
if (s->context)
(void) client_context_maybe_refresh(s->server, s->context, NULL, NULL, 0, NULL, USEC_INFINITY);
else if (pid_is_valid(s->ucred.pid)) {
r = client_context_acquire(s->server, s->ucred.pid, &s->ucred, s->label, strlen_ptr(s->label), s->unit_id, &s->context);
if (r < 0)
log_warning_errno(r, "Failed to acquire client context, ignoring: %m");
}
server_dispatch_message(s->server, iovec, n, ELEMENTSOF(iovec), s->context, NULL, priority, 0);
return 0;
}

15
src/journal/journald-syslog.c
View File

@@ -325,11 +325,12 @@ void server_process_syslog_message(
char syslog_priority[sizeof("PRIORITY=") + DECIMAL_STR_MAX(int)],
syslog_facility[sizeof("SYSLOG_FACILITY=") + DECIMAL_STR_MAX(int)];
const char *message = NULL, *syslog_identifier = NULL, *syslog_pid = NULL;
struct iovec iovec[N_IOVEC_META_FIELDS + 6];
unsigned n = 0;
int priority = LOG_USER | LOG_INFO;
_cleanup_free_ char *identifier = NULL, *pid = NULL;
struct iovec iovec[N_IOVEC_META_FIELDS + 6];
int priority = LOG_USER | LOG_INFO, r;
ClientContext *context = NULL;
const char *orig;
unsigned n = 0;
assert(s);
assert(buf);
@@ -376,7 +377,13 @@ void server_process_syslog_message(
if (message)
IOVEC_SET_STRING(iovec[n++], message);
server_dispatch_message(s, iovec, n, ELEMENTSOF(iovec), ucred, tv, label, label_len, NULL, priority, 0);
if (ucred && pid_is_valid(ucred->pid)) {
r = client_context_get(s, ucred->pid, ucred, label, label_len, NULL, &context);
if (r < 0)
log_warning_errno(r, "Failed to retrieve credentials for PID " PID_FMT ", ignoring: %m", ucred->pid);
}
server_dispatch_message(s, iovec, n, ELEMENTSOF(iovec), context, tv, priority, 0);
}
int server_open_syslog_socket(Server *s) {

30
src/journal/meson.build
View File

@@ -59,24 +59,26 @@ journal_internal_sources += [audit_type_to_name]
############################################################
libjournal_core_sources = files('''
journald-kmsg.c
journald-kmsg.h
journald-syslog.c
journald-syslog.h
journald-stream.c
journald-stream.h
journald-server.c
journald-server.h
journald-console.c
journald-console.h
journald-wall.c
journald-wall.h
journald-native.c
journald-native.h
journald-audit.c
journald-audit.h
journald-console.c
journald-console.h
journald-context.c
journald-context.h
journald-kmsg.c
journald-kmsg.h
journald-native.c
journald-native.h
journald-rate-limit.c
journald-rate-limit.h
journald-server.c
journald-server.h
journald-stream.c
journald-stream.h
journald-syslog.c
journald-syslog.h
journald-wall.c
journald-wall.h
journal-internal.h
'''.split())