diff --git a/man/systemd-sysext.xml b/man/systemd-sysext.xml
index d32804409b..ea1af001a7 100644
--- a/man/systemd-sysext.xml
+++ b/man/systemd-sysext.xml
@@ -69,8 +69,10 @@
/var/ included in a system extension image will not appear in
the respective hierarchies after activation.
- System extension images are strictly read-only, and the host /usr/ and
- /opt/ hierarchies become read-only too while they are activated.
+ System extension images are strictly read-only by default. On mutable host file systems,
+ /usr/ and /opt/ hierarchies become read-only while extensions
+ are merged, unless mutability is enabled. Mutability may be enabled via the
+ option; see "Mutability" below for more information.
System extensions are supposed to be purely additive, i.e. they are supposed to include only files
that do not exist in the underlying basic OS image. However, the underlying mechanism (overlayfs) also
@@ -158,6 +160,11 @@
same as sysext images. The merged hierarchy will be mounted with nosuid and
(if not disabled via ) noexec.
+ Just like sysexts, confexts are strictly read-only by default. Merging confexts on mutable host
+ file systems will result in /etc/ becoming read-only. As with sysexts, mutability
+ can be enabled via the option. Refer to "Mutability" below for more
+ information.
+
Confexts are looked for in the directories /run/confexts/,
/var/lib/confexts/, /usr/lib/confexts/ and
/usr/local/lib/confexts/. The first listed directory is not suitable for
@@ -205,6 +212,55 @@
to tie the most frequently configured options to runtime updateable flags that can be changed without a
system reboot. This will help reduce servicing times when there is a need for changing the OS configuration.
+
+ Mutability
+ By default, merging system extensions on mutable host file systems will render /usr/
+ and /opt/ hierarchies read-only. Merging configuration extensions will have the same
+ effect on /etc/. Mutable mode allows writes to these locations when extensions are
+ merged.
+
+ The following modes are supported:
+
+ : Force immutable mode even if write routing
+ directories exist below /var/lib/extensions.mutable/.
+ This is the default.
+ : Automatic mode. Mutability is disabled by default
+ and only enabled if a corresponding write routing directory exists below
+ /var/lib/extensions.mutable/.
+ : Force mutable mode and automatically create write routing
+ directories below /var/lib/extensions.mutable/ when required.
+ : Force immutable mode like above, but
+ merge the contents of directories below /var/lib/extensions.mutable/ into the host
+ file system.
+
+ See "Options" below on specifying modes using the command line option.
+
+ Mutable mode routes writes to subdirectories in /var/lib/extensions.mutable/.
+
+ Writes to /usr/ are directed to /var/lib/extensions.mutable/usr/,
+ writes to /opt/ are directed to /var/lib/extensions.mutable/opt/, and
+ writes to /etc/ land in /var/lib/extensions.mutable/etc/.
+
+
+ If usr/, opt/, or etc/
+ in /var/lib/extensions.mutable/ are symlinks, then writes are directed to the
+ symlinks' targets.
+ Consequently, to retain mutability of a host file system, create symlinks
+
+ /var/lib/extensions.mutable/etc/ → /etc/
+ /var/lib/extensions.mutable/usr/ → /usr/
+ /var/lib/extensions.mutable/opt/ → /opt/
+
+ to route writes back to the original base directory hierarchy.
+
+ Alternatively, a temporary file system may be mounted to
+ /var/lib/extensions.mutable/, or symlinks in
+ /var/lib/extensions.mutable/ may point to sub-directories on a temporary
+ file system (e.g. below /tmp/) to only allow ephemeral changes.
+
+
+
+
Commands
@@ -313,6 +369,45 @@
+
+ BOOL|auto|import
+ Set mutable mode.
+
+
+
+
+ force immutable mode even with write routing directories present.
+ This is the default.
+
+
+
+
+
+ enable mutable mode individually for /usr/,
+ /opt/, and /etc/ if write routing sub-directories
+ or symlinks are present in /var/lib/extensions.mutable/; disable otherwise.
+ See "Mutability" above for more information on write routing.
+
+
+
+
+
+ force mutable mode. Write routing directories will be created in
+ /var/lib/extensions.mutable/ if not present.
+
+
+
+
+
+ immutable mode, but with contents of write routing directories in
+ /var/lib/extensions.mutable/ also merged into the host file system.
+
+
+
+
+
+
+
BOOL
diff --git a/src/shared/mount-util.c b/src/shared/mount-util.c
index bff1bf9f49..77b18c375c 100644
--- a/src/shared/mount-util.c
+++ b/src/shared/mount-util.c
@@ -453,6 +453,16 @@ int bind_remount_one_with_mountinfo(
return 0;
}
+int bind_remount_one(const char *path, unsigned long new_flags, unsigned long flags_mask) {
+ _cleanup_fclose_ FILE *proc_self_mountinfo = NULL;
+
+ proc_self_mountinfo = fopen("/proc/self/mountinfo", "re");
+ if (!proc_self_mountinfo)
+ return log_debug_errno(errno, "Failed to open /proc/self/mountinfo: %m");
+
+ return bind_remount_one_with_mountinfo(path, new_flags, flags_mask, proc_self_mountinfo);
+}
+
static int mount_switch_root_pivot(int fd_newroot, const char *path) {
assert(fd_newroot >= 0);
assert(path);
diff --git a/src/shared/mount-util.h b/src/shared/mount-util.h
index 2f9f394ab0..26d96b27b7 100644
--- a/src/shared/mount-util.h
+++ b/src/shared/mount-util.h
@@ -26,6 +26,7 @@ static inline int bind_remount_recursive(const char *prefix, unsigned long new_f
}
int bind_remount_one_with_mountinfo(const char *path, unsigned long new_flags, unsigned long flags_mask, FILE *proc_self_mountinfo);
+int bind_remount_one(const char *path, unsigned long new_flags, unsigned long flags_mask);
int mount_switch_root_full(const char *path, unsigned long mount_propagation_flag, bool force_ms_move);
static inline int mount_switch_root(const char *path, unsigned long mount_propagation_flag) {
diff --git a/src/sysext/sysext.c b/src/sysext/sysext.c
index 6c1bfdb77e..cb55d62322 100644
--- a/src/sysext/sysext.c
+++ b/src/sysext/sysext.c
@@ -39,15 +39,27 @@
#include "pager.h"
#include "parse-argument.h"
#include "parse-util.h"
+#include "path-util.h"
#include "pretty-print.h"
#include "process-util.h"
+#include "rm-rf.h"
#include "sort-util.h"
+#include "string-util.h"
#include "terminal-util.h"
#include "user-util.h"
#include "varlink.h"
#include "varlink-io.systemd.sysext.h"
#include "verbs.h"
+typedef enum MutableMode {
+ MUTABLE_YES,
+ MUTABLE_NO,
+ MUTABLE_AUTO,
+ MUTABLE_IMPORT,
+ _MUTABLE_MAX,
+ _MUTABLE_INVALID = -EINVAL,
+} MutableMode;
+
static char **arg_hierarchies = NULL; /* "/usr" + "/opt" by default for sysext and /etc by default for confext */
static char *arg_root = NULL;
static JsonFormatFlags arg_json_format_flags = JSON_FORMAT_OFF;
@@ -58,6 +70,7 @@ static bool arg_no_reload = false;
static int arg_noexec = -1;
static ImagePolicy *arg_image_policy = NULL;
static bool arg_varlink = false;
+static MutableMode arg_mutable = MUTABLE_NO;
/* Is set to IMAGE_CONFEXT when systemd is called with the confext functionality instead of the default */
static ImageClass arg_image_class = IMAGE_SYSEXT;
@@ -252,11 +265,22 @@ static int unmerge_hierarchy(
ImageClass image_class,
const char *p) {
+ _cleanup_free_ char *dot_dir = NULL, *work_dir_info_file = NULL;
int r;
assert(p);
+ dot_dir = path_join(p, image_class_info[image_class].dot_directory_name);
+ if (!dot_dir)
+ return log_oom();
+
+ work_dir_info_file = path_join(dot_dir, "work_dir");
+ if (!work_dir_info_file)
+ return log_oom();
+
for (;;) {
+ _cleanup_free_ char *escaped_work_dir_in_root = NULL, *work_dir = NULL;
+
/* We only unmount /usr/ if it is a mount point and really one of ours, in order not to break
* systems where /usr/ is a mount point of its own already. */
@@ -266,9 +290,40 @@ static int unmerge_hierarchy(
if (r == 0)
break;
+ r = read_one_line_file(work_dir_info_file, &escaped_work_dir_in_root);
+ if (r < 0) {
+ if (r != -ENOENT)
+ return log_error_errno(r, "Failed to read '%s': %m", work_dir_info_file);
+ } else {
+ _cleanup_free_ char *work_dir_in_root = NULL;
+ ssize_t l;
+
+ l = cunescape_length(escaped_work_dir_in_root, r, 0, &work_dir_in_root);
+ if (l < 0)
+ return log_error_errno(l, "Failed to unescape work directory path: %m");
+ work_dir = path_join(arg_root, work_dir_in_root);
+ if (!work_dir)
+ return log_oom();
+ }
+
+ r = umount_verbose(LOG_DEBUG, dot_dir, MNT_DETACH|UMOUNT_NOFOLLOW);
+ if (r < 0) {
+ /* EINVAL is possibly "not a mount point". Let it slide as it's expected to occur if
+ * the whole hierarchy was read-only, so the dot directory inside it was not
+ * bind-mounted as read-only. */
+ if (r != -EINVAL)
+ return log_error_errno(r, "Failed to unmount '%s': %m", dot_dir);
+ }
+
r = umount_verbose(LOG_ERR, p, MNT_DETACH|UMOUNT_NOFOLLOW);
if (r < 0)
- return log_error_errno(r, "Failed to unmount file system '%s': %m", p);
+ return r;
+
+ if (work_dir) {
+ r = rm_rf(work_dir, REMOVE_ROOT | REMOVE_MISSING_OK | REMOVE_PHYSICAL);
+ if (r < 0)
+ return log_error_errno(r, "Failed to remove '%s': %m", work_dir);
+ }
log_info("Unmerged '%s'.", p);
}
@@ -478,11 +533,38 @@ static int verb_status(int argc, char **argv, void *userdata) {
return ret;
}
+static int append_overlayfs_path_option(
+ char **options,
+ const char *separator,
+ const char *option,
+ const char *path) {
+
+ _cleanup_free_ char *escaped = NULL;
+
+ assert(options);
+ assert(separator);
+ assert(path);
+
+ escaped = shell_escape(path, ",:");
+ if (!escaped)
+ return log_oom();
+
+ if (option) {
+ if (!strextend(options, separator, option, "=", escaped))
+ return log_oom();
+ } else if (!strextend(options, separator, escaped))
+ return log_oom();
+
+ return 0;
+}
+
static int mount_overlayfs(
ImageClass image_class,
int noexec,
const char *where,
- char **layers) {
+ char **layers,
+ const char *upper_dir,
+ const char *work_dir) {
_cleanup_free_ char *options = NULL;
bool separator = false;
@@ -490,20 +572,16 @@ static int mount_overlayfs(
int r;
assert(where);
+ assert((upper_dir && work_dir) || (!upper_dir && !work_dir));
options = strdup("lowerdir=");
if (!options)
return log_oom();
STRV_FOREACH(l, layers) {
- _cleanup_free_ char *escaped = NULL;
-
- escaped = shell_escape(*l, ",:");
- if (!escaped)
- return log_oom();
-
- if (!strextend(&options, separator ? ":" : "", escaped))
- return log_oom();
+ r = append_overlayfs_path_option(&options, separator ? ":" : "", NULL, *l);
+ if (r < 0)
+ return r;
separator = true;
}
@@ -512,6 +590,22 @@ static int mount_overlayfs(
if (noexec >= 0)
SET_FLAG(flags, MS_NOEXEC, noexec);
+ if (upper_dir && work_dir) {
+ r = append_overlayfs_path_option(&options, ",", "upperdir", upper_dir);
+ if (r < 0)
+ return r;
+
+ flags &= ~MS_RDONLY;
+
+ r = append_overlayfs_path_option(&options, ",", "workdir", work_dir);
+ if (r < 0)
+ return r;
+ /* redirect_dir=on and noatime prevent unnecessary upcopies, metacopy=off prevents broken
+ * files from partial upcopies after umount. */
+ if (!strextend(&options, ",redirect_dir=on,noatime,metacopy=off"))
+ return log_oom();
+ }
+
/* Now mount the actual overlayfs */
r = mount_nofollow_verbose(LOG_ERR, image_class_info[image_class].short_identifier, where, "overlay", flags, options);
if (r < 0)
@@ -520,41 +614,451 @@ static int mount_overlayfs(
return 0;
}
-static int merge_hierarchy(
- ImageClass image_class,
- const char *hierarchy,
- int noexec,
- char **extensions,
- char **paths,
- const char *meta_path,
- const char *overlay_path) {
+static char *hierarchy_as_single_path_component(const char *hierarchy) {
+ /* We normally expect hierarchy to be /usr, /opt or /etc, but for debugging purposes the hierarchy
+ * could very well be like /foo/bar/baz/. So for a given hierarchy we generate a directory name by
+ * stripping the leading and trailing separators and replacing the rest of separators with dots. This
+ * makes the generated name to be the same for /foo/bar/baz and for /foo/bar.baz, but, again,
+ * speciyfing a different hierarchy is a debugging feature, so non-unique mapping should not be an
+ * issue in general case. */
+ const char *stripped = hierarchy;
+ _cleanup_free_ char *dir_name = NULL;
- _cleanup_free_ char *resolved_hierarchy = NULL, *f = NULL, *buf = NULL;
- _cleanup_strv_free_ char **layers = NULL;
- struct stat st;
+ assert(hierarchy);
+
+ stripped += strspn(stripped, "/");
+
+ dir_name = strdup(stripped);
+ if (!dir_name)
+ return NULL;
+ delete_trailing_chars(dir_name, "/");
+ string_replace_char(dir_name, '/', '.');
+ return TAKE_PTR(dir_name);
+}
+
+static char *determine_mutable_directory_path_for_hierarchy(const char *hierarchy) {
+ _cleanup_free_ char *dir_name = NULL;
+
+ assert(hierarchy);
+ dir_name = hierarchy_as_single_path_component(hierarchy);
+ if (!dir_name)
+ return NULL;
+
+ return path_join("/var/lib/extensions.mutable", dir_name);
+}
+
+static int paths_on_same_fs(const char *path1, const char *path2) {
+ struct stat st1, st2;
+
+ assert(path1);
+ assert(path2);
+
+ if (stat(path1, &st1))
+ return log_error_errno(errno, "Failed to stat '%s': %m", path1);
+
+ if (stat(path2, &st2))
+ return log_error_errno(errno, "Failed to stat '%s': %m", path2);
+
+ return st1.st_dev == st2.st_dev;
+}
+
+static int work_dir_for_hierarchy(
+ const char *hierarchy,
+ const char *resolved_upper_dir,
+ char **ret_work_dir) {
+
+ _cleanup_free_ char *parent = NULL;
int r;
assert(hierarchy);
+ assert(resolved_upper_dir);
+ assert(ret_work_dir);
+
+ r = path_extract_directory(resolved_upper_dir, &parent);
+ if (r < 0)
+ return log_error_errno(r, "Failed to get parent directory of upperdir '%s': %m", resolved_upper_dir);
+
+ /* TODO: paths_in_same_superblock? partition? device? */
+ r = paths_on_same_fs(resolved_upper_dir, parent);
+ if (r < 0)
+ return r;
+ if (!r)
+ return log_error_errno(SYNTHETIC_ERRNO(EXDEV), "Unable to find a suitable workdir location for upperdir '%s' for host hierarchy '%s' - parent directory of the upperdir is in a different filesystem", resolved_upper_dir, hierarchy);
+
+ _cleanup_free_ char *f = NULL, *dir_name = NULL;
+
+ f = hierarchy_as_single_path_component(hierarchy);
+ if (!f)
+ return log_oom();
+ dir_name = strjoin(".systemd-", f, "-workdir");
+ if (!dir_name)
+ return log_oom();
+
+ free(f);
+ f = path_join(parent, dir_name);
+ if (!f)
+ return log_oom();
+
+ *ret_work_dir = TAKE_PTR(f);
+ return 0;
+}
+
+typedef struct OverlayFSPaths {
+ char *hierarchy;
+ char *resolved_hierarchy;
+ char *resolved_mutable_directory;
+
+ /* NULL if merged fs is read-only */
+ char *upper_dir;
+ /* NULL if merged fs is read-only */
+ char *work_dir;
+ /* lowest index is top lowerdir, highest index is bottom lowerdir */
+ char **lower_dirs;
+} OverlayFSPaths;
+
+static OverlayFSPaths *overlayfs_paths_free(OverlayFSPaths *op) {
+ if (!op)
+ return NULL;
+
+ free(op->hierarchy);
+ free(op->resolved_hierarchy);
+ free(op->resolved_mutable_directory);
+
+ free(op->upper_dir);
+ free(op->work_dir);
+ strv_free(op->lower_dirs);
+
+ free(op);
+ return NULL;
+}
+DEFINE_TRIVIAL_CLEANUP_FUNC(OverlayFSPaths *, overlayfs_paths_free);
+
+static int resolve_hierarchy(const char *hierarchy, char **ret_resolved_hierarchy) {
+ _cleanup_free_ char *resolved_path = NULL;
+ int r;
+
+ assert(hierarchy);
+ assert(ret_resolved_hierarchy);
+
+ r = chase(hierarchy, arg_root, CHASE_PREFIX_ROOT, &resolved_path, NULL);
+ if (r < 0 && r != -ENOENT)
+ return log_error_errno(r, "Failed to resolve hierarchy '%s': %m", hierarchy);
+
+ *ret_resolved_hierarchy = TAKE_PTR(resolved_path);
+ return 0;
+}
+
+static int resolve_mutable_directory(const char *hierarchy, char **ret_resolved_mutable_directory) {
+ _cleanup_free_ char *path = NULL, *resolved_path = NULL;
+ int r;
+
+ assert(hierarchy);
+ assert(ret_resolved_mutable_directory);
+
+ if (arg_mutable == MUTABLE_NO) {
+ log_debug("Mutability for hierarchy '%s' is disabled, not resolving mutable directory.", hierarchy);
+ *ret_resolved_mutable_directory = NULL;
+ return 0;
+ }
+
+ path = determine_mutable_directory_path_for_hierarchy(hierarchy);
+ if (!path)
+ return log_oom();
+
+ if (arg_mutable == MUTABLE_YES) {
+ _cleanup_free_ char *path_in_root = NULL;
+
+ path_in_root = path_join(arg_root, path);
+ if (!path_in_root)
+ return log_oom();
+
+ r = mkdir_p(path_in_root, 0700);
+ if (r < 0)
+ return log_error_errno(r, "Failed to create a directory '%s': %m", path_in_root);
+ }
+
+ r = chase(path, arg_root, CHASE_PREFIX_ROOT, &resolved_path, NULL);
+ if (r < 0 && r != -ENOENT)
+ return log_error_errno(r, "Failed to resolve mutable directory '%s': %m", path);
+
+ *ret_resolved_mutable_directory = TAKE_PTR(resolved_path);
+ return 0;
+}
+
+static int overlayfs_paths_new(const char *hierarchy, OverlayFSPaths **ret_op) {
+ _cleanup_free_ char *hierarchy_copy = NULL, *resolved_hierarchy = NULL, *resolved_mutable_directory = NULL;
+ int r;
+
+ assert (hierarchy);
+ assert (ret_op);
+
+ hierarchy_copy = strdup(hierarchy);
+ if (!hierarchy_copy)
+ return log_oom();
+
+ r = resolve_hierarchy(hierarchy, &resolved_hierarchy);
+ if (r < 0)
+ return r;
+ r = resolve_mutable_directory(hierarchy, &resolved_mutable_directory);
+ if (r < 0)
+ return r;
+
+ OverlayFSPaths *op;
+ op = new(OverlayFSPaths, 1);
+ if (!op)
+ return log_oom();
+
+ *op = (OverlayFSPaths) {
+ .hierarchy = TAKE_PTR(hierarchy_copy),
+ .resolved_hierarchy = TAKE_PTR(resolved_hierarchy),
+ .resolved_mutable_directory = TAKE_PTR(resolved_mutable_directory),
+ };
+
+ *ret_op = TAKE_PTR(op);
+ return 0;
+}
+
+static int determine_top_lower_dirs(OverlayFSPaths *op, const char *meta_path) {
+ int r;
+
+ assert(op);
assert(meta_path);
+
+ /* Put the meta path (i.e. our synthesized stuff) at the top of the layer stack */
+ r = strv_extend(&op->lower_dirs, meta_path);
+ if (r < 0)
+ return log_oom();
+
+ /* If importing mutable layer and it actually exists, add it just below the meta path */
+ if (arg_mutable == MUTABLE_IMPORT && op->resolved_mutable_directory) {
+ r = strv_extend(&op->lower_dirs, op->resolved_mutable_directory);
+ if (r < 0)
+ return r;
+ }
+
+ return 0;
+}
+
+static int determine_middle_lower_dirs(OverlayFSPaths *op, char **paths, size_t *ret_extensions_used) {
+ size_t n = 0;
+ int r;
+
+ assert(op);
+ assert(paths);
+ assert(ret_extensions_used);
+
+ /* Put the extensions in the middle */
+ STRV_FOREACH(p, paths) {
+ _cleanup_free_ char *resolved = NULL;
+
+ r = chase(op->hierarchy, *p, CHASE_PREFIX_ROOT, &resolved, NULL);
+ if (r == -ENOENT) {
+ log_debug_errno(r, "Hierarchy '%s' in extension '%s' doesn't exist, not merging.", op->hierarchy, *p);
+ continue;
+ }
+ if (r < 0)
+ return log_error_errno(r, "Failed to resolve hierarchy '%s' in extension '%s': %m", op->hierarchy, *p);
+
+ r = dir_is_empty(resolved, /* ignore_hidden_or_backup= */ false);
+ if (r < 0)
+ return log_error_errno(r, "Failed to check if hierarchy '%s' in extension '%s' is empty: %m", resolved, *p);
+ if (r > 0) {
+ log_debug("Hierarchy '%s' in extension '%s' is empty, not merging.", op->hierarchy, *p);
+ continue;
+ }
+
+ r = strv_consume(&op->lower_dirs, TAKE_PTR(resolved));
+ if (r < 0)
+ return log_oom();
+ ++n;
+ }
+
+ *ret_extensions_used = n;
+ return 0;
+}
+
+static int hierarchy_as_lower_dir(OverlayFSPaths *op) {
+ int r;
+
+ /* return 0 if hierarchy should be used as lower dir, >0, if not */
+
+ assert(op);
+
+ if (!op->resolved_hierarchy) {
+ log_debug("Host hierarchy '%s' does not exist, will not be used as lowerdir", op->hierarchy);
+ return 1;
+ }
+
+ r = dir_is_empty(op->resolved_hierarchy, /* ignore_hidden_or_backup= */ false);
+ if (r < 0)
+ return log_error_errno(r, "Failed to check if host hierarchy '%s' is empty: %m", op->resolved_hierarchy);
+ if (r > 0) {
+ log_debug("Host hierarchy '%s' is empty, will not be used as lower dir.", op->resolved_hierarchy);
+ return 1;
+ }
+
+ if (arg_mutable == MUTABLE_IMPORT) {
+ log_debug("Mutability for host hierarchy '%s' is disabled, so it will be a lowerdir", op->resolved_hierarchy);
+ return 0;
+ }
+
+ if (!op->resolved_mutable_directory) {
+ log_debug("No mutable directory found, so host hierarchy '%s' will be used as lowerdir", op->resolved_hierarchy);
+ return 0;
+ }
+
+ if (path_equal(op->resolved_hierarchy, op->resolved_mutable_directory)) {
+ log_debug("Host hierarchy '%s' will serve as upperdir.", op->resolved_hierarchy);
+ return 1;
+ }
+ r = inode_same(op->resolved_hierarchy, op->resolved_mutable_directory, 0);
+ if (r < 0)
+ return log_error_errno(r, "Failed to check inode equality of hierarchy %s and its mutable directory %s: %m", op->resolved_hierarchy, op->resolved_mutable_directory);
+ if (r > 0) {
+ log_debug("Host hierarchy '%s' will serve as upperdir.", op->resolved_hierarchy);
+ return 1;
+ }
+
+ return 0;
+}
+
+static int determine_bottom_lower_dirs(OverlayFSPaths *op) {
+ int r;
+
+ assert(op);
+
+ r = hierarchy_as_lower_dir(op);
+ if (r < 0)
+ return r;
+ if (!r) {
+ r = strv_extend(&op->lower_dirs, op->resolved_hierarchy);
+ if (r < 0)
+ return r;
+ }
+
+ return 0;
+}
+
+static int determine_lower_dirs(
+ OverlayFSPaths *op,
+ char **paths,
+ const char *meta_path,
+ size_t *ret_extensions_used) {
+
+ int r;
+
+ assert(op);
+ assert(paths);
+ assert(meta_path);
+ assert(ret_extensions_used);
+
+ r = determine_top_lower_dirs(op, meta_path);
+ if (r < 0)
+ return r;
+
+ r = determine_middle_lower_dirs(op, paths, ret_extensions_used);
+ if (r < 0)
+ return r;
+
+ r = determine_bottom_lower_dirs(op);
+ if (r < 0)
+ return r;
+
+ return 0;
+}
+
+static int determine_upper_dir(OverlayFSPaths *op) {
+ int r;
+
+ assert(op);
+ assert(!op->upper_dir);
+
+ if (arg_mutable == MUTABLE_IMPORT) {
+ log_debug("Mutability is disabled, there will be no upperdir for host hierarchy '%s'", op->hierarchy);
+ return 0;
+ }
+
+ if (!op->resolved_mutable_directory) {
+ log_debug("No mutable directory found for host hierarchy '%s', there will be no upperdir", op->hierarchy);
+ return 0;
+ }
+
+ /* Require upper dir to be on writable filesystem if it's going to be used as an actual overlayfs
+ * upperdir, instead of a lowerdir as an imported path. */
+ r = path_is_read_only_fs(op->resolved_mutable_directory);
+ if (r < 0)
+ return log_error_errno(r, "Failed to determine if mutable directory '%s' is on read-only filesystem: %m", op->resolved_mutable_directory);
+ if (r > 0)
+ return log_error_errno(SYNTHETIC_ERRNO(EROFS), "Can't use '%s' as an upperdir as it is read-only.", op->resolved_mutable_directory);
+
+ op->upper_dir = strdup(op->resolved_mutable_directory);
+ if (!op->upper_dir)
+ return log_oom();
+
+ return 0;
+}
+
+static int determine_work_dir(OverlayFSPaths *op) {
+ _cleanup_free_ char *work_dir = NULL;
+ int r;
+
+ assert(op);
+ assert(!op->work_dir);
+
+ if (!op->upper_dir)
+ return 0;
+
+ if (arg_mutable == MUTABLE_IMPORT)
+ return 0;
+
+ r = work_dir_for_hierarchy(op->hierarchy, op->upper_dir, &work_dir);
+ if (r < 0)
+ return r;
+
+ op->work_dir = TAKE_PTR(work_dir);
+ return 0;
+}
+
+static int mount_overlayfs_with_op(
+ OverlayFSPaths *op,
+ ImageClass image_class,
+ int noexec,
+ const char *overlay_path,
+ const char *meta_path) {
+
+ int r;
+
+ assert(op);
assert(overlay_path);
- /* Resolve the path of the host's version of the hierarchy, i.e. what we want to use as lowest layer
- * in the overlayfs stack. */
- r = chase(hierarchy, arg_root, CHASE_PREFIX_ROOT, &resolved_hierarchy, NULL);
- if (r == -ENOENT)
- log_debug_errno(r, "Hierarchy '%s' on host doesn't exist, not merging.", hierarchy);
- else if (r < 0)
- return log_error_errno(r, "Failed to resolve host hierarchy '%s': %m", hierarchy);
- else {
- r = dir_is_empty(resolved_hierarchy, /* ignore_hidden_or_backup= */ false);
+ r = mkdir_p(overlay_path, 0700);
+ if (r < 0)
+ return log_error_errno(r, "Failed to make directory '%s': %m", overlay_path);
+
+ r = mkdir_p(meta_path, 0700);
+ if (r < 0)
+ return log_error_errno(r, "Failed to make directory '%s': %m", meta_path);
+
+ if (op->upper_dir && op->work_dir) {
+ r = mkdir_p(op->work_dir, 0700);
if (r < 0)
- return log_error_errno(r, "Failed to check if host hierarchy '%s' is empty: %m", resolved_hierarchy);
- if (r > 0) {
- log_debug("Host hierarchy '%s' is empty, not merging.", resolved_hierarchy);
- resolved_hierarchy = mfree(resolved_hierarchy);
- }
+ return log_error_errno(r, "Failed to make directory '%s': %m", op->work_dir);
}
+ r = mount_overlayfs(image_class, noexec, overlay_path, op->lower_dirs, op->upper_dir, op->work_dir);
+ if (r < 0)
+ return r;
+
+ return 0;
+}
+
+static int write_extensions_file(ImageClass image_class, char **extensions, const char *meta_path) {
+ _cleanup_free_ char *f = NULL, *buf = NULL;
+ int r;
+
+ assert(extensions);
+ assert(meta_path);
+
/* Let's generate a metadata file that lists all extensions we took into account for this
* hierarchy. We include this in the final fs, to make things nicely discoverable and
* recognizable. */
@@ -570,79 +1074,184 @@ static int merge_hierarchy(
if (r < 0)
return log_error_errno(r, "Failed to write extension meta file '%s': %m", f);
- /* Put the meta path (i.e. our synthesized stuff) at the top of the layer stack */
- layers = strv_new(meta_path);
- if (!layers)
- return log_oom();
+ return 0;
+}
- /* Put the extensions in the middle */
- STRV_FOREACH(p, paths) {
- _cleanup_free_ char *resolved = NULL;
+static int write_dev_file(ImageClass image_class, const char *meta_path, const char *overlay_path) {
+ _cleanup_free_ char *f = NULL;
+ struct stat st;
+ int r;
- r = chase(hierarchy, *p, CHASE_PREFIX_ROOT, &resolved, NULL);
- if (r == -ENOENT) {
- log_debug_errno(r, "Hierarchy '%s' in extension '%s' doesn't exist, not merging.", hierarchy, *p);
- continue;
- }
- if (r < 0)
- return log_error_errno(r, "Failed to resolve hierarchy '%s' in extension '%s': %m", hierarchy, *p);
-
- r = dir_is_empty(resolved, /* ignore_hidden_or_backup= */ false);
- if (r < 0)
- return log_error_errno(r, "Failed to check if hierarchy '%s' in extension '%s' is empty: %m", resolved, *p);
- if (r > 0) {
- log_debug("Hierarchy '%s' in extension '%s' is empty, not merging.", hierarchy, *p);
- continue;
- }
-
- r = strv_consume(&layers, TAKE_PTR(resolved));
- if (r < 0)
- return log_oom();
- }
-
- if (!layers[1]) /* No extension with files in this hierarchy? Then don't do anything. */
- return 0;
-
- if (resolved_hierarchy) {
- /* Add the host hierarchy as last (lowest) layer in the stack */
- r = strv_consume(&layers, TAKE_PTR(resolved_hierarchy));
- if (r < 0)
- return log_oom();
- }
-
- r = mkdir_p(overlay_path, 0700);
- if (r < 0)
- return log_error_errno(r, "Failed to make directory '%s': %m", overlay_path);
-
- r = mount_overlayfs(image_class, noexec, overlay_path, layers);
- if (r < 0)
- return r;
-
- /* The overlayfs superblock is read-only. Let's also mark the bind mount read-only. Extra turbo safety 😎 */
- r = bind_remount_recursive(overlay_path, MS_RDONLY, MS_RDONLY, NULL);
- if (r < 0)
- return log_error_errno(r, "Failed to make bind mount '%s' read-only: %m", overlay_path);
+ assert(meta_path);
+ assert(overlay_path);
/* Now we have mounted the new file system. Let's now figure out its .st_dev field, and make that
* available in the metadata directory. This is useful to detect whether the metadata dir actually
* belongs to the fs it is found on: if .st_dev of the top-level mount matches it, it's pretty likely
* we are looking at a live tree, and not an unpacked tar or so of one. */
if (stat(overlay_path, &st) < 0)
- return log_error_errno(r, "Failed to stat mount '%s': %m", overlay_path);
+ return log_error_errno(errno, "Failed to stat mount '%s': %m", overlay_path);
- free(f);
f = path_join(meta_path, image_class_info[image_class].dot_directory_name, "dev");
if (!f)
return log_oom();
+ /* Modifying the underlying layers while the overlayfs is mounted is technically undefined, but at
+ * least it won't crash or deadlock, as per the kernel docs about overlayfs:
+ * https://www.kernel.org/doc/html/latest/filesystems/overlayfs.html#changes-to-underlying-filesystems */
r = write_string_file(f, FORMAT_DEVNUM(st.st_dev), WRITE_STRING_FILE_CREATE);
if (r < 0)
return log_error_errno(r, "Failed to write '%s': %m", f);
+ return 0;
+}
+
+static int write_work_dir_file(ImageClass image_class, const char *meta_path, const char *work_dir) {
+ _cleanup_free_ char *escaped_work_dir_in_root = NULL, *f = NULL;
+ char *work_dir_in_root = NULL;
+ int r;
+
+ assert(meta_path);
+
+ if (!work_dir)
+ return 0;
+
+ work_dir_in_root = path_startswith(work_dir, empty_to_root(arg_root));
+ if (!work_dir_in_root)
+ return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Workdir '%s' must not be outside root '%s'", work_dir, empty_to_root(arg_root));
+
+ f = path_join(meta_path, image_class_info[image_class].dot_directory_name, "work_dir");
+ if (!f)
+ return log_oom();
+
+ /* Paths can have newlines for whatever reason, so better escape them to really get a single
+ * line file. */
+ escaped_work_dir_in_root = cescape(work_dir_in_root);
+ if (!escaped_work_dir_in_root)
+ return log_oom();
+ r = write_string_file(f, escaped_work_dir_in_root, WRITE_STRING_FILE_CREATE);
+ if (r < 0)
+ return log_error_errno(r, "Failed to write '%s': %m", f);
+
+ return 0;
+}
+
+static int store_info_in_meta(
+ ImageClass image_class,
+ char **extensions,
+ const char *meta_path,
+ const char *overlay_path,
+ const char *work_dir) {
+
+ int r;
+
+ assert(extensions);
+ assert(meta_path);
+ assert(overlay_path);
+ /* work_dir may be NULL */
+
+ r = write_extensions_file(image_class, extensions, meta_path);
+ if (r < 0)
+ return r;
+
+ r = write_dev_file(image_class, meta_path, overlay_path);
+ if (r < 0)
+ return r;
+
+ r = write_work_dir_file(image_class, meta_path, work_dir);
+ if (r < 0)
+ return r;
+
/* Make sure the top-level dir has an mtime marking the point we established the merge */
if (utimensat(AT_FDCWD, meta_path, NULL, AT_SYMLINK_NOFOLLOW) < 0)
return log_error_errno(r, "Failed fix mtime of '%s': %m", meta_path);
+ return 0;
+}
+
+static int make_mounts_read_only(ImageClass image_class, const char *overlay_path, bool mutable) {
+ int r;
+
+ assert(overlay_path);
+
+ if (mutable) {
+ /* Bind mount the meta path as read-only on mutable overlays to avoid accidental
+ * modifications of the contents of meta directory, which could lead to systemd thinking that
+ * this hierarchy is not our mount. */
+ _cleanup_free_ char *f = NULL;
+
+ f = path_join(overlay_path, image_class_info[image_class].dot_directory_name);
+ if (!f)
+ return log_oom();
+
+ r = mount_nofollow_verbose(LOG_ERR, f, f, NULL, MS_BIND, NULL);
+ if (r < 0)
+ return r;
+
+ r = bind_remount_one(f, MS_RDONLY, MS_RDONLY);
+ if (r < 0)
+ return log_error_errno(r, "Failed to remount '%s' as read-only: %m", f);
+ } else {
+ /* The overlayfs superblock is read-only. Let's also mark the bind mount read-only. Extra
+ * turbo safety 😎 */
+ r = bind_remount_recursive(overlay_path, MS_RDONLY, MS_RDONLY, NULL);
+ if (r < 0)
+ return log_error_errno(r, "Failed to make bind mount '%s' read-only: %m", overlay_path);
+ }
+
+ return 0;
+}
+
+static int merge_hierarchy(
+ ImageClass image_class,
+ const char *hierarchy,
+ int noexec,
+ char **extensions,
+ char **paths,
+ const char *meta_path,
+ const char *overlay_path) {
+
+ _cleanup_(overlayfs_paths_freep) OverlayFSPaths *op = NULL;
+ size_t extensions_used = 0;
+ int r;
+
+ assert(hierarchy);
+ assert(extensions);
+ assert(paths);
+ assert(meta_path);
+ assert(overlay_path);
+
+ r = overlayfs_paths_new(hierarchy, &op);
+ if (r < 0)
+ return r;
+
+ r = determine_lower_dirs(op, paths, meta_path, &extensions_used);
+ if (r < 0)
+ return r;
+
+ if (extensions_used == 0) /* No extension with files in this hierarchy? Then don't do anything. */
+ return 0;
+
+ r = determine_upper_dir(op);
+ if (r < 0)
+ return r;
+
+ r = determine_work_dir(op);
+ if (r < 0)
+ return r;
+
+ r = mount_overlayfs_with_op(op, image_class, noexec, overlay_path, meta_path);
+ if (r < 0)
+ return r;
+
+ r = store_info_in_meta(image_class, extensions, meta_path, overlay_path, op->work_dir);
+ if (r < 0)
+ return r;
+
+ r = make_mounts_read_only(image_class, overlay_path, op->upper_dir && op->work_dir);
+ if (r < 0)
+ return r;
+
return 1;
}
@@ -966,7 +1575,8 @@ static int merge_subprocess(
if (r < 0)
return log_error_errno(r, "Failed to create hierarchy mount point '%s': %m", resolved);
- r = mount_nofollow_verbose(LOG_ERR, p, resolved, NULL, MS_BIND, NULL);
+ /* Using MS_REC to potentially bring in our read-only bind mount of metadata. */
+ r = mount_nofollow_verbose(LOG_ERR, p, resolved, NULL, MS_BIND|MS_REC, NULL);
if (r < 0)
return r;
@@ -1419,6 +2029,7 @@ static int parse_argv(int argc, char *argv[]) {
ARG_IMAGE_POLICY,
ARG_NOEXEC,
ARG_NO_RELOAD,
+ ARG_MUTABLE,
};
static const struct option options[] = {
@@ -1432,6 +2043,7 @@ static int parse_argv(int argc, char *argv[]) {
{ "image-policy", required_argument, NULL, ARG_IMAGE_POLICY },
{ "noexec", required_argument, NULL, ARG_NOEXEC },
{ "no-reload", no_argument, NULL, ARG_NO_RELOAD },
+ { "mutable", required_argument, NULL, ARG_MUTABLE },
{}
};
@@ -1495,6 +2107,19 @@ static int parse_argv(int argc, char *argv[]) {
arg_no_reload = true;
break;
+ case ARG_MUTABLE:
+ if (streq(optarg, "auto"))
+ arg_mutable = MUTABLE_AUTO;
+ else if (streq(optarg, "import"))
+ arg_mutable = MUTABLE_IMPORT;
+ else {
+ r = parse_boolean(optarg);
+ if (r < 0)
+ return log_error_errno(r, "Failed to parse argument to --mutable=: %s", optarg);
+ arg_mutable = r ? MUTABLE_YES : MUTABLE_NO;
+ }
+ break;
+
case '?':
return -EINVAL;
diff --git a/src/test/test-mount-util.c b/src/test/test-mount-util.c
index 3e22ac67fc..77fce983b9 100644
--- a/src/test/test-mount-util.c
+++ b/src/test/test-mount-util.c
@@ -213,6 +213,25 @@ TEST(bind_remount_one) {
_exit(EXIT_SUCCESS);
}
+ assert_se(wait_for_terminate_and_check("test-remount-one-with-mountinfo", pid, WAIT_LOG) == EXIT_SUCCESS);
+
+ pid = fork();
+ assert_se(pid >= 0);
+
+ if (pid == 0) {
+ /* child */
+
+ assert_se(detach_mount_namespace() >= 0);
+
+ assert_se(bind_remount_one("/run", MS_RDONLY, MS_RDONLY) >= 0);
+ assert_se(bind_remount_one("/run", MS_NOEXEC, MS_RDONLY|MS_NOEXEC) >= 0);
+ assert_se(bind_remount_one("/proc/idontexist", MS_RDONLY, MS_RDONLY) == -ENOENT);
+ assert_se(bind_remount_one("/proc/self", MS_RDONLY, MS_RDONLY) == -EINVAL);
+ assert_se(bind_remount_one("/", MS_RDONLY, MS_RDONLY) >= 0);
+
+ _exit(EXIT_SUCCESS);
+ }
+
assert_se(wait_for_terminate_and_check("test-remount-one", pid, WAIT_LOG) == EXIT_SUCCESS);
}
diff --git a/test/units/testsuite-50.sh b/test/units/testsuite-50.sh
index e34cbd915b..370e8f01ed 100755
--- a/test/units/testsuite-50.sh
+++ b/test/units/testsuite-50.sh
@@ -8,10 +8,7 @@ set -o pipefail
export SYSTEMD_LOG_LEVEL=debug
-# shellcheck disable=SC2317
-cleanup() {(
- set +ex
-
+cleanup_image_dir() {
if [ -z "${image_dir}" ]; then
return
fi
@@ -20,6 +17,39 @@ cleanup() {(
umount "${image_dir}/app-nodistro"
umount "${image_dir}/service-scoped-test"
rm -rf "${image_dir}"
+}
+
+fake_roots_dir=/fake-roots
+
+cleanup_fake_rootfses() {
+ local tries=10 e
+ local -a lines fake_roots_mounts
+
+ while [[ ${tries} -gt 0 ]]; do
+ tries=$((tries - 1))
+ mapfile -t lines < <(mount | awk '{ print $3 }')
+ fake_roots_mounts=()
+ for e in "${lines[@]}"; do
+ if [[ ${e} = "${fake_roots_dir}"/* ]]; then
+ fake_roots_mounts+=( "${e}" )
+ fi
+ done
+ if [[ ${#fake_roots_mounts[@]} -eq 0 ]]; then
+ break
+ fi
+ for e in "${fake_roots_mounts[@]}"; do
+ umount "${e}"
+ done
+ done
+ rm -rf "${fake_roots_dir}"
+}
+
+# shellcheck disable=SC2317
+cleanup() {(
+ set +ex
+
+ cleanup_image_dir
+ cleanup_fake_rootfses
)}
udevadm control --log-level=debug
@@ -765,4 +795,690 @@ fi
(! systemd-run -P -p RootImage="/this/should/definitely/not/exist.img" false)
(! systemd-run -P -p ExtensionDirectories="/foo/bar /foo/baz" false)
+# general systemd-sysext tests
+
+shopt -s extglob
+
+die() {
+ echo "${*}"
+ exit 1
+}
+
+prep_root() {
+ local r=${1}; shift
+ local h=${1}; shift
+
+ mkdir -p "${r}${h}" "${r}/usr/lib" "${r}/var/lib/extensions" "${r}/var/lib/extensions.mutable"
+}
+
+gen_os_release() {
+ local r=${1}; shift
+
+ {
+ echo "ID=testtest"
+ echo "VERSION=1.2.3"
+ } >"${r}/usr/lib/os-release"
+}
+
+gen_test_ext_image() {
+ local r=${1}; shift
+ local h=${1}; shift
+
+ local n d f
+
+ n='test-extension'
+ d="${r}/var/lib/extensions/${n}"
+ f="${d}/usr/lib/extension-release.d/extension-release.${n}"
+ mkdir -p "$(dirname "${f}")"
+ echo "ID=_any" >"${f}"
+ mkdir -p "${d}/${h}"
+ touch "${d}${h}/preexisting-file-in-extension-image"
+}
+
+hierarchy_ext_mut_path() {
+ local r=${1}; shift
+ local h=${1}; shift
+
+ # /a/b/c -> a.b.c
+ local n=${h}
+ n="${n##+(/)}"
+ n="${n%%+(/)}"
+ n="${n//\//.}"
+
+ printf '%s' "${r}/var/lib/extensions.mutable/${n}"
+}
+
+prep_ext_mut() {
+ local p=${1}; shift
+
+ mkdir -p "${p}"
+ touch "${p}/preexisting-file-in-extensions-mutable"
+}
+
+make_ro() {
+ local r=${1}; shift
+ local h=${1}; shift
+
+ mount -o bind "${r}${h}" "${r}${h}"
+ mount -o bind,remount,ro "${r}${h}"
+}
+
+prep_hierarchy() {
+ local r=${1}; shift
+ local h=${1}; shift
+
+ touch "${r}${h}/preexisting-file-in-hierarchy"
+}
+
+prep_ro_hierarchy() {
+ local r=${1}; shift
+ local h=${1}; shift
+
+ prep_hierarchy "${r}" "${h}"
+ make_ro "${r}" "${h}"
+}
+
+# extra args:
+# "e" for checking for the preexisting file in extension
+# "h" for checking for the preexisting file in hierarchy
+# "u" for checking for the preexisting file in upperdir
+check_usual_suspects() {
+ local root=${1}; shift
+ local hierarchy=${1}; shift
+ local message=${1}; shift
+
+ local arg
+ # shellcheck disable=SC2034 # the variables below are used indirectly
+ local e='' h='' u=''
+
+ for arg; do
+ case ${arg} in
+ e|h|u)
+ local -n v=${arg}
+ v=x
+ unset -n v
+ ;;
+ *)
+ die "invalid arg to ${0}: ${arg@Q}"
+ ;;
+ esac
+ done
+
+ # var name, file name
+ local pairs=(
+ e:preexisting-file-in-extension-image
+ h:preexisting-file-in-hierarchy
+ u:preexisting-file-in-extensions-mutable
+ )
+ local pair name file desc full_path
+ for pair in "${pairs[@]}"; do
+ name=${pair%%:*}
+ file=${pair#*:}
+ desc=${file//-/ }
+ full_path="${root}${hierarchy}/${file}"
+ local -n v=${name}
+ if [[ -n ${v} ]]; then
+ test -f "${full_path}" || {
+ ls -la "$(dirname "${full_path}")"
+ die "${desc} is missing ${message}"
+ }
+ else
+ test ! -f "${full_path}" || {
+ ls -la "$(dirname "${full_path}")"
+ die "${desc} unexpectedly exists ${message}"
+ }
+ fi
+ unset -n v
+ done
+}
+
+check_usual_suspects_after_merge() {
+ local r=${1}; shift
+ local h=${1}; shift
+
+ check_usual_suspects "${r}" "${h}" "after merge" "${@}"
+}
+
+check_usual_suspects_after_unmerge() {
+ local r=${1}; shift
+ local h=${1}; shift
+
+ check_usual_suspects "${r}" "${h}" "after unmerge" "${@}"
+}
+
+
+#
+# no extension data in /var/lib/extensions.mutable/…, read-only hierarchy,
+# mutability disabled by default
+#
+# read-only merged
+#
+
+
+fake_root=${fake_roots_dir}/simple-read-only-with-read-only-hierarchy
+hierarchy=/usr
+
+prep_root "${fake_root}" "${hierarchy}"
+gen_os_release "${fake_root}"
+gen_test_ext_image "${fake_root}" "${hierarchy}"
+
+prep_ro_hierarchy "${fake_root}" "${hierarchy}"
+
+touch "${fake_root}${hierarchy}/should-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+
+# run systemd-sysext
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" merge
+
+touch "${fake_root}${hierarchy}/should-still-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+check_usual_suspects_after_merge "${fake_root}" "${hierarchy}" e h
+
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" unmerge
+
+check_usual_suspects_after_unmerge "${fake_root}" "${hierarchy}" h
+
+touch "${fake_root}${hierarchy}/should-still-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only after unmerge"
+
+#
+# no extension data in /var/lib/extensions.mutable/…, mutable hierarchy,
+# mutability disabled by default
+#
+# read-only merged
+#
+
+
+fake_root=${fake_roots_dir}/simple-read-only-with-mutable-hierarchy
+hierarchy=/usr
+
+prep_root "${fake_root}" "${hierarchy}"
+gen_os_release "${fake_root}"
+gen_test_ext_image "${fake_root}" "${hierarchy}"
+
+prep_hierarchy "${fake_root}" "${hierarchy}"
+
+touch "${fake_root}${hierarchy}/should-succeed-on-mutable-fs" || die "${fake_root}${hierarchy} is not mutable"
+
+# run systemd-sysext
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" merge
+
+touch "${fake_root}${hierarchy}/should-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+check_usual_suspects_after_merge "${fake_root}" "${hierarchy}" e h
+
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" unmerge
+
+check_usual_suspects_after_unmerge "${fake_root}" "${hierarchy}" h
+
+touch "${fake_root}${hierarchy}/should-succeed-on-mutable-fs-again" || die "${fake_root}${hierarchy} is not mutable after unmerge"
+
+
+#
+# no extension data in /var/lib/extensions.mutable/…, no hierarchy either,
+# mutability disabled by default
+#
+# read-only merged
+#
+
+
+fake_root=${fake_roots_dir}/simple-read-only-with-missing-hierarchy
+hierarchy=/opt
+
+prep_root "${fake_root}" "${hierarchy}"
+rmdir "${fake_root}/${hierarchy}"
+gen_os_release "${fake_root}"
+gen_test_ext_image "${fake_root}" "${hierarchy}"
+
+# run systemd-sysext
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" merge
+
+touch "${fake_root}${hierarchy}/should-still-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+check_usual_suspects_after_merge "${fake_root}" "${hierarchy}" e
+
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" unmerge
+
+check_usual_suspects_after_unmerge "${fake_root}" "${hierarchy}"
+
+
+#
+# no extension data in /var/lib/extensions.mutable/…, an empty hierarchy,
+# mutability disabled by default
+#
+# read-only merged
+#
+
+
+fake_root=${fake_roots_dir}/simple-read-only-with-empty-hierarchy
+hierarchy=/opt
+
+prep_root "${fake_root}" "${hierarchy}"
+gen_os_release "${fake_root}"
+gen_test_ext_image "${fake_root}" "${hierarchy}"
+
+make_ro "${fake_root}" "${hierarchy}"
+
+touch "${fake_root}${hierarchy}/should-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+
+# run systemd-sysext
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" merge
+
+touch "${fake_root}${hierarchy}/should-still-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+check_usual_suspects_after_merge "${fake_root}" "${hierarchy}" e
+
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" unmerge
+
+check_usual_suspects_after_unmerge "${fake_root}" "${hierarchy}"
+
+
+#
+# extension data in /var/lib/extensions.mutable/…, read-only hierarchy, mutability disabled-by-default
+#
+# read-only merged
+#
+
+
+fake_root=${fake_roots_dir}/simple-mutable-with-read-only-hierarchy-disabled
+hierarchy=/usr
+
+prep_root "${fake_root}" "${hierarchy}"
+gen_os_release "${fake_root}"
+gen_test_ext_image "${fake_root}" "${hierarchy}"
+
+ext_data_path=$(hierarchy_ext_mut_path "${fake_root}" "${hierarchy}")
+prep_ext_mut "${ext_data_path}"
+
+prep_ro_hierarchy "${fake_root}" "${hierarchy}"
+
+touch "${fake_root}${hierarchy}/should-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+
+# run systemd-sysext
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" merge
+
+touch "${fake_root}${hierarchy}/should-be-read-only" && die "${fake_root}${hierarchy} is not read-only"
+check_usual_suspects_after_merge "${fake_root}" "${hierarchy}" e h
+
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" unmerge
+
+check_usual_suspects_after_unmerge "${fake_root}" "${hierarchy}" h
+
+
+#
+# extension data in /var/lib/extensions.mutable/…, read-only hierarchy, auto-mutability
+#
+# mutable merged
+#
+
+
+fake_root=${fake_roots_dir}/simple-mutable-with-read-only-hierarchy
+hierarchy=/usr
+
+prep_root "${fake_root}" "${hierarchy}"
+gen_os_release "${fake_root}"
+gen_test_ext_image "${fake_root}" "${hierarchy}"
+
+ext_data_path=$(hierarchy_ext_mut_path "${fake_root}" "${hierarchy}")
+prep_ext_mut "${ext_data_path}"
+
+prep_ro_hierarchy "${fake_root}" "${hierarchy}"
+
+touch "${fake_root}${hierarchy}/should-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+
+# run systemd-sysext
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" --mutable=auto merge
+
+touch "${fake_root}${hierarchy}/now-is-mutable" || die "${fake_root}${hierarchy} is not mutable"
+check_usual_suspects_after_merge "${fake_root}" "${hierarchy}" e h u
+test -f "${ext_data_path}/now-is-mutable" || die "now-is-mutable is not stored in expected location"
+
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" unmerge
+
+check_usual_suspects_after_unmerge "${fake_root}" "${hierarchy}" h
+test -f "${ext_data_path}/now-is-mutable" || die "now-is-mutable disappeared from writable storage after unmerge"
+test ! -f "${fake_root}${hierarchy}/now-is-mutable" || die "now-is-mutable did not disappear from hierarchy after unmerge"
+
+
+#
+# extension data in /var/lib/extensions.mutable/…, missing hierarchy,
+# auto-mutability
+#
+# mutable merged
+#
+
+
+fake_root=${fake_roots_dir}/simple-mutable-with-missing-hierarchy
+hierarchy=/opt
+
+prep_root "${fake_root}" "${hierarchy}"
+rmdir "${fake_root}/${hierarchy}"
+gen_os_release "${fake_root}"
+gen_test_ext_image "${fake_root}" "${hierarchy}"
+
+ext_data_path=$(hierarchy_ext_mut_path "${fake_root}" "${hierarchy}")
+prep_ext_mut "${ext_data_path}"
+
+# run systemd-sysext
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" --mutable=auto merge
+
+touch "${fake_root}${hierarchy}/now-is-mutable" || die "${fake_root}${hierarchy} is not mutable"
+check_usual_suspects_after_merge "${fake_root}" "${hierarchy}" e u
+test -f "${ext_data_path}/now-is-mutable" || die "now-is-mutable is not stored in expected location"
+
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" unmerge
+
+check_usual_suspects_after_unmerge "${fake_root}" "${hierarchy}"
+test -f "${ext_data_path}/now-is-mutable" || die "now-is-mutable disappeared from writable storage after unmerge"
+test ! -f "${fake_root}${hierarchy}/now-is-mutable" || die "now-is-mutable did not disappear from hierarchy after unmerge"
+
+
+#
+# extension data in /var/lib/extensions.mutable/…, empty hierarchy, auto-mutability
+#
+# mutable merged
+#
+
+
+fake_root=${fake_roots_dir}/simple-mutable-with-empty-hierarchy
+hierarchy=/opt
+
+prep_root "${fake_root}" "${hierarchy}"
+gen_os_release "${fake_root}"
+gen_test_ext_image "${fake_root}" "${hierarchy}"
+
+ext_data_path=$(hierarchy_ext_mut_path "${fake_root}" "${hierarchy}")
+prep_ext_mut "${ext_data_path}"
+
+make_ro "${fake_root}" "${hierarchy}"
+
+touch "${fake_root}${hierarchy}/should-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+
+# run systemd-sysext
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" --mutable=auto merge
+
+touch "${fake_root}${hierarchy}/now-is-mutable" || die "${fake_root}${hierarchy} is not mutable"
+check_usual_suspects_after_merge "${fake_root}" "${hierarchy}" e u
+test -f "${ext_data_path}/now-is-mutable" || die "now-is-mutable is not stored in expected location"
+
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" unmerge
+
+check_usual_suspects_after_unmerge "${fake_root}" "${hierarchy}"
+test -f "${ext_data_path}/now-is-mutable" || die "now-is-mutable disappeared from writable storage after unmerge"
+test ! -f "${fake_root}${hierarchy}/now-is-mutable" || die "now-is-mutable did not disappear from hierarchy after unmerge"
+
+
+#
+# /var/lib/extensions.mutable/… is a symlink to /some/other/dir, read-only
+# hierarchy, auto-mutability
+#
+# mutable merged
+#
+
+
+fake_root=${fake_roots_dir}/mutable-symlink-with-read-only-hierarchy
+hierarchy=/usr
+
+prep_root "${fake_root}" "${hierarchy}"
+gen_os_release "${fake_root}"
+gen_test_ext_image "${fake_root}" "${hierarchy}"
+
+# generate extension writable data
+ext_data_path=$(hierarchy_ext_mut_path "${fake_root}" "${hierarchy}")
+real_ext_dir="${fake_root}/upperdir"
+prep_ext_mut "${real_ext_dir}"
+ln -sfTr "${real_ext_dir}" "${ext_data_path}"
+
+prep_ro_hierarchy "${fake_root}" "${hierarchy}"
+
+touch "${fake_root}${hierarchy}/should-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+
+# run systemd-sysext
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" --mutable=auto merge
+
+touch "${fake_root}${hierarchy}/now-is-mutable" || die "${fake_root}${hierarchy} is not mutable"
+check_usual_suspects_after_merge "${fake_root}" "${hierarchy}" e h u
+test -f "${ext_data_path}/now-is-mutable" || die "now-is-mutable is not stored in expected location"
+test -f "${real_ext_dir}/now-is-mutable" || die "now-is-mutable is not stored in expected location"
+
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" unmerge
+
+check_usual_suspects_after_unmerge "${fake_root}" "${hierarchy}" h
+test -f "${ext_data_path}/now-is-mutable" || die "now-is-mutable disappeared from writable storage after unmerge"
+test -f "${real_ext_dir}/now-is-mutable" || die "now-is-mutable disappeared from writable storage after unmerge"
+test ! -f "${fake_root}${hierarchy}/now-is-mutable" || die "now-is-mutable did not disappear from hierarchy after unmerge"
+
+
+#
+# /var/lib/extensions.mutable/… is a symlink to the hierarchy itself, auto-mutability
+#
+# for this to work, hierarchy must be mutable
+#
+# mutable merged
+#
+
+
+fake_root=${fake_roots_dir}/mutable-self-upper
+hierarchy=/usr
+
+prep_root "${fake_root}" "${hierarchy}"
+gen_os_release "${fake_root}"
+gen_test_ext_image "${fake_root}" "${hierarchy}"
+
+# generate extension writable data
+ext_data_path=$(hierarchy_ext_mut_path "${fake_root}" "${hierarchy}")
+real_ext_dir="${fake_root}${hierarchy}"
+prep_ext_mut "${real_ext_dir}"
+ln -sfTr "${real_ext_dir}" "${ext_data_path}"
+
+# prepare writable hierarchy
+touch "${fake_root}${hierarchy}/preexisting-file-in-hierarchy"
+
+# run systemd-sysext
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" --mutable=auto merge
+
+touch "${fake_root}${hierarchy}/now-is-mutable" || die "${fake_root}${hierarchy} is not mutable"
+check_usual_suspects_after_merge "${fake_root}" "${hierarchy}" e h u
+test -f "${ext_data_path}/now-is-mutable" || die "now-is-mutable is not stored in expected location"
+test -f "${real_ext_dir}/now-is-mutable" || die "now-is-mutable is not stored in expected location"
+
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" unmerge
+
+check_usual_suspects_after_unmerge "${fake_root}" "${hierarchy}" h u
+test -f "${ext_data_path}/now-is-mutable" || die "now-is-mutable disappeared from writable storage after unmerge"
+test -f "${real_ext_dir}/now-is-mutable" || die "now-is-mutable disappeared from writable storage after unmerge"
+
+
+#
+# /var/lib/extensions.mutable/… is a symlink to the hierarchy itself, which is
+# read-only, auto-mutability
+#
+# expecting a failure here
+#
+
+
+fake_root=${fake_roots_dir}/failure-self-upper-ro
+hierarchy=/usr
+
+prep_root "${fake_root}" "${hierarchy}"
+gen_os_release "${fake_root}"
+gen_test_ext_image "${fake_root}" "${hierarchy}"
+
+# generate extension writable data
+ext_data_path=$(hierarchy_ext_mut_path "${fake_root}" "${hierarchy}")
+real_ext_dir="${fake_root}${hierarchy}"
+prep_ext_mut "${real_ext_dir}"
+ln -sfTr "${real_ext_dir}" "${ext_data_path}"
+
+prep_ro_hierarchy "${fake_root}" "${hierarchy}"
+
+# run systemd-sysext
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" --mutable=auto merge || die "expected merge to fail"
+
+
+#
+# /var/lib/extensions.mutable/… is a dangling symlink, auto-mutability
+#
+# read-only merged
+#
+
+
+fake_root=${fake_roots_dir}/read-only-mutable-dangling-symlink
+hierarchy=/usr
+
+prep_root "${fake_root}" "${hierarchy}"
+gen_os_release "${fake_root}"
+gen_test_ext_image "${fake_root}" "${hierarchy}"
+
+ext_data_path=$(hierarchy_ext_mut_path "${fake_root}" "${hierarchy}")
+ln -sfTr "/should/not/exist/" "${ext_data_path}"
+
+prep_ro_hierarchy "${fake_root}" "${hierarchy}"
+
+touch "${fake_root}${hierarchy}/should-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+
+# run systemd-sysext
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" --mutable=auto merge
+
+touch "${fake_root}${hierarchy}/should-still-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+
+check_usual_suspects_after_merge "${fake_root}" "${hierarchy}" e h
+
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" unmerge
+
+check_usual_suspects_after_unmerge "${fake_root}" "${hierarchy}" h
+
+
+#
+# /var/lib/extensions.mutable/… exists, but it's ignored, mutability disabled explicitly
+#
+# read-only merged
+#
+
+
+fake_root=${fake_roots_dir}/disabled
+hierarchy=/usr
+
+prep_root "${fake_root}" "${hierarchy}"
+gen_os_release "${fake_root}"
+gen_test_ext_image "${fake_root}" "${hierarchy}"
+
+ext_data_path=$(hierarchy_ext_mut_path "${fake_root}" "${hierarchy}")
+prep_ext_mut "${ext_data_path}"
+
+prep_ro_hierarchy "${fake_root}" "${hierarchy}"
+
+touch "${fake_root}${hierarchy}/should-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+
+# run systemd-sysext
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" --mutable=no merge
+
+touch "${fake_root}${hierarchy}/should-still-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+
+check_usual_suspects_after_merge "${fake_root}" "${hierarchy}" e h
+
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" unmerge
+
+check_usual_suspects_after_unmerge "${fake_root}" "${hierarchy}" h
+
+
+#
+# /var/lib/extensions.mutable/… exists, but it's imported instead
+#
+# read-only merged
+#
+
+
+fake_root=${fake_roots_dir}/imported
+hierarchy=/usr
+
+prep_root "${fake_root}" "${hierarchy}"
+gen_os_release "${fake_root}"
+gen_test_ext_image "${fake_root}" "${hierarchy}"
+
+ext_data_path=$(hierarchy_ext_mut_path "${fake_root}" "${hierarchy}")
+prep_ext_mut "${ext_data_path}"
+
+prep_ro_hierarchy "${fake_root}" "${hierarchy}"
+
+touch "${fake_root}${hierarchy}/should-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+
+# run systemd-sysext
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" --mutable=import merge
+
+touch "${fake_root}${hierarchy}/should-still-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+
+check_usual_suspects_after_merge "${fake_root}" "${hierarchy}" e h u
+
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" unmerge
+
+check_usual_suspects_after_unmerge "${fake_root}" "${hierarchy}" h
+
+
+#
+# /var/lib/extensions.mutable/… does not exist, but mutability is enabled
+# explicitly
+#
+# mutable merged
+#
+
+
+fake_root=${fake_roots_dir}/enabled
+hierarchy=/usr
+
+prep_root "${fake_root}" "${hierarchy}"
+gen_os_release "${fake_root}"
+gen_test_ext_image "${fake_root}" "${hierarchy}"
+
+ext_data_path=$(hierarchy_ext_mut_path "${fake_root}" "${hierarchy}")
+
+prep_ro_hierarchy "${fake_root}" "${hierarchy}"
+
+touch "${fake_root}${hierarchy}/should-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+
+test ! -d "${ext_data_path}" || die "extensions.mutable should not exist"
+
+# run systemd-sysext
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" --mutable=yes merge
+
+test -d "${ext_data_path}" || die "extensions.mutable should exist now"
+touch "${fake_root}${hierarchy}/now-is-mutable" || die "${fake_root}${hierarchy} is not mutable"
+check_usual_suspects_after_merge "${fake_root}" "${hierarchy}" e h
+test -f "${ext_data_path}/now-is-mutable" || die "now-is-mutable is not stored in expected location"
+
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" unmerge
+
+check_usual_suspects_after_unmerge "${fake_root}" "${hierarchy}" h
+test -f "${ext_data_path}/now-is-mutable" || die "now-is-mutable disappeared from writable storage after unmerge"
+test ! -f "${fake_root}${hierarchy}/now-is-mutable" || die "now-is-mutable did not disappear from hierarchy after unmerge"
+
+
+#
+# /var/lib/extensions.mutable/… does not exist, auto-mutability
+#
+# read-only merged
+#
+
+
+fake_root=${fake_roots_dir}/simple-read-only-explicit
+hierarchy=/usr
+
+prep_root "${fake_root}" "${hierarchy}"
+gen_os_release "${fake_root}"
+gen_test_ext_image "${fake_root}" "${hierarchy}"
+
+prep_ro_hierarchy "${fake_root}" "${hierarchy}"
+
+touch "${fake_root}${hierarchy}/should-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+
+# run systemd-sysext
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" --mutable=auto merge
+
+touch "${fake_root}${hierarchy}/should-still-fail-on-read-only-fs" && die "${fake_root}${hierarchy} is not read-only"
+check_usual_suspects_after_merge "${fake_root}" "${hierarchy}" e h
+
+SYSTEMD_SYSEXT_HIERARCHIES="${hierarchy}" systemd-sysext --root="${fake_root}" unmerge
+
+check_usual_suspects_after_unmerge "${fake_root}" "${hierarchy}" h
+
+
+#
+# done
+#
+
+
touch /testok