HALF ECHO LEARN / REFERENCE COLLECTION

v262.2

src/nspawn/nspawn.c

300,508 bytesGit mode 100644Download original fileLGPL-2.1-or-later

Git blob: 1c687382baea8193c023ec86840c44042195bef9

Lines 6001–6878 of 6,878

                 * kernel's initialization should still be valid during runtime — at least if PID 1 is systemd. Note
                 * that PID 1 changes a number of other resource limits during early initialization which is why we
                 * don't read the other limits from PID 1 but prefer the static table above. */
        };
 
        int rl, r;
 
        for (rl = 0; rl < _RLIMIT_MAX; rl++) {
                /* Let's only fill in what the user hasn't explicitly configured anyway */
                if ((arg_settings_mask & (SETTING_RLIMIT_FIRST << rl)) == 0) {
                        const struct rlimit *v;
                        struct rlimit buffer;
 
                        if (IN_SET(rl, RLIMIT_NPROC, RLIMIT_SIGPENDING)) {
                                /* For these two let's read the limits off PID 1. See above for an explanation. */
 
                                r = pid_getrlimit(1, rl, &buffer);
                                if (r < 0)
                                        return log_error_errno(r, "Failed to read resource limit RLIMIT_%s of PID 1: %m", rlimit_to_string(rl));
 
                                v = &buffer;
                        } else if (rl == RLIMIT_NOFILE) {
                                /* We nowadays bump RLIMIT_NOFILE's hard limit early in PID 1 for all
                                 * userspace. Given that nspawn containers are often run without our PID 1,
                                 * let's grant the containers a raised RLIMIT_NOFILE hard limit by default,
                                 * so that container userspace gets similar resources as host userspace
                                 * gets. */
                                buffer = kernel_defaults[rl];
                                buffer.rlim_max = MIN((rlim_t) read_nr_open(), (rlim_t) HIGH_RLIMIT_NOFILE);
                                v = &buffer;
                        } else
                                v = kernel_defaults + rl;
 
                        arg_rlimit[rl] = newdup(struct rlimit, v, 1);
                        if (!arg_rlimit[rl])
                                return log_oom();
                }
 
                if (DEBUG_LOGGING) {
                        _cleanup_free_ char *k = NULL;
 
                        (void) rlimit_format(arg_rlimit[rl], &k);
                        log_debug("Setting RLIMIT_%s to %s.", rlimit_to_string(rl), k);
                }
        }
 
        return 0;
}
 
static int cant_be_in_netns(void) {
        int r;
 
        /* Check if we are in the same netns as udev. If we aren't, then device monitoring (and thus waiting
         * for loopback block devices) won't work, and we will hang. Detect this case and exit early with a
         * nice message. */
 
        if (!arg_image) /* only matters if --image= us used, i.e. we actually need to use loopback devices */
                return 0;
 
        if (arg_userns_mode == USER_NAMESPACE_MANAGED)
                return 0;
 
        if (!udev_available())
                return log_error_errno(SYNTHETIC_ERRNO(EOPNOTSUPP),
                                       "Sorry, but --image= requires that systemd-udevd is available.");
 
        r = namespace_is_init(NAMESPACE_NET);
        if (ERRNO_IS_NEG_PRIVILEGE(r)) {
                log_debug_errno(r, "Failed to check if we are in the main network namespace, assuming so, ignoring: %m");
                return 0;
        }
        if (r < 0)
                return log_error_errno(r, "Failed to check if we are in the main network namespace: %m");
        if (r == 0)
                return log_error_errno(SYNTHETIC_ERRNO(EOPNOTSUPP),
                                       "Sorry, but --image= is only supported in the main network namespace, since we need access to udev/AF_NETLINK.");
        return 0;
}
 
static void cleanup_propagation_and_export_directories(const char *runtime_dir) {
        _cleanup_free_ char *p = NULL;
 
        if (!runtime_dir || arg_userns_mode == USER_NAMESPACE_MANAGED)
                return;
 
        p = path_join("/run/systemd/nspawn/propagate", arg_machine);
        if (p)
                (void) rm_rf(p, REMOVE_ROOT);
 
        free(p);
        p = path_join(runtime_dir, "unix-export");
        if (p) {
                (void) umount2(p, MNT_DETACH|UMOUNT_NOFOLLOW);
                (void) rmdir(p);
        }
}
 
static int do_cleanup(void) {
        int r;
 
        if (arg_ephemeral)
                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Cannot specify --ephemeral with --cleanup.");
 
        r = determine_names();
        if (r < 0)
                return r;
 
        _cleanup_free_ char *runtime_dir = NULL;
        r = runtime_directory_resolve(arg_runtime_scope, "systemd/nspawn", arg_machine, &runtime_dir);
        if (r < 0)
                return r;
 
        cleanup_propagation_and_export_directories(runtime_dir);
        return 0;
}
 
static int acquire_listen_fdnames(FDSet *fds, char ***ret) {
        _cleanup_strv_free_ char **l = NULL;
        const char *e;
        int r;
 
        assert(ret);
 
        e = getenv("LISTEN_FDNAMES");
        if (!e || fdset_isempty(fds)) {
                *ret = NULL;
                return 0;
        }
 
        /* Split the same way sd_listen_fds_with_names() does, so that we end up with the same names. */
        r = strv_split_full(&l, e, ":", EXTRACT_DONT_COALESCE_SEPARATORS);
        if (r == -ENOMEM)
                return log_oom();
        if (r < 0) {
                log_warning_errno(r, "Failed to parse $LISTEN_FDNAMES, ignoring: %m");
                *ret = NULL;
                return 0;
        }
 
        if ((size_t) r != fdset_size(fds)) {
                log_warning("Number of entries in $LISTEN_FDNAMES (%i) doesn't match number of passed file descriptors (%u), ignoring.",
                            r, fdset_size(fds));
                *ret = NULL;
                return 0;
        }
 
        /* forward_fd_store() prefixes the names of the payload's fd store entries with "payload-" when
         * pushing them upstream. Strip it again when they are passed back to us, so that the payload gets
         * them back under the name it stored them with. Leave the names alone if we wouldn't forward fd
         * store entries, as they can't have come from the payload then. */
        bool strip_prefix = fd_store_forwarding_enabled();
 
        STRV_FOREACH(n, l) {
                if (!fdname_is_valid(*n)) {
                        _cleanup_free_ char *esc = cescape(*n);
                        log_warning("$LISTEN_FDNAMES contains invalid file descriptor name '%s', ignoring.", strna(esc));
                        *ret = NULL;
                        return 0;
                }
 
                const char *name = (strip_prefix ? startswith(*n, "payload-") : NULL) ?: *n;
 
                /* The payload's sd_listen_fds_with_names() will unescape the names again, hence escape
                 * them here. */
                char *esc = shell_escape(name, ":");
                if (!esc)
                        return log_oom();
 
                free_and_replace(*n, esc);
        }
 
        *ret = TAKE_PTR(l);
        return 0;
}
 
static int run(int argc, char *argv[]) {
        bool remove_image = false, veth_created = false;
        _cleanup_close_ int master = -EBADF, userns_fd = -EBADF, mount_fd = -EBADF;
        _cleanup_fdset_free_ FDSet *fds = NULL;
        _cleanup_strv_free_ char **fdnames = NULL;
        int r, ret = EXIT_SUCCESS;
        char veth_name[IFNAMSIZ] = "";
        struct ExposeArgs expose_args = {};
        _cleanup_(release_lock_file) LockFile tree_global_lock = LOCK_FILE_INIT, tree_local_lock = LOCK_FILE_INIT;
        _cleanup_(rmdir_and_freep) char *rootdir = NULL;
        _cleanup_(rm_rf_subvolume_and_freep) char *snapshot_dir = NULL;
        _cleanup_(loop_device_unrefp) LoopDevice *loop = NULL;
        _cleanup_(dissected_image_unrefp) DissectedImage *dissected_image = NULL;
        _cleanup_(mstack_freep) MStack *mstack = NULL;
        _cleanup_(sd_netlink_unrefp) sd_netlink *nfnl = NULL;
        _cleanup_(pidref_done) PidRef pid = PIDREF_NULL;
        _cleanup_(sd_varlink_unrefp) sd_varlink *nsresource_link = NULL, *mountfsd_link = NULL;
        _cleanup_(rm_rf_physical_and_freep) char *runtime_dir = NULL;
        _cleanup_(fork_notify_terminate) PidRef journal_remote_pidref = PIDREF_NULL;
 
        log_setup();
 
        r = parse_argv(argc, argv);
        if (r <= 0)
                goto finish;
 
        if (arg_cleanup)
                return do_cleanup();
 
        LIBACL_NOTE(recommended);
        LIBBLKID_NOTE(recommended);
        LIBCRYPTO_NOTE(recommended);
        LIBCRYPTSETUP_NOTE(suggested);
        LIBMOUNT_NOTE(recommended);
        LIBSECCOMP_NOTE(recommended);
        LIBSELINUX_NOTE(recommended);
        (void) dlopen_cryptsetup(LOG_DEBUG);
        (void) dlopen_libacl(LOG_DEBUG);
        (void) dlopen_libblkid(LOG_DEBUG);
        (void) dlopen_libcrypto(LOG_WARNING);
        (void) dlopen_libmount(LOG_DEBUG);
        (void) dlopen_libseccomp(LOG_DEBUG);
        (void) dlopen_libselinux(LOG_DEBUG);
 
        r = cg_has_legacy();
        if (r < 0)
                goto finish;
        if (r > 0) {
                r = log_error_errno(SYNTHETIC_ERRNO(EPROTO),
                                    "Detected host uses legacy cgroup v1 hierarchy, refusing.");
                goto finish;
        }
 
        r = cant_be_in_netns();
        if (r < 0)
                goto finish;
 
        r = initialize_rlimits();
        if (r < 0)
                goto finish;
 
        r = load_oci_bundle();
        if (r < 0)
                goto finish;
 
        r = pick_paths();
        if (r < 0)
                goto finish;
 
        r = determine_names();
        if (r < 0)
                goto finish;
 
        r = load_settings();
        if (r < 0)
                goto finish;
 
        if (!FLAGS_SET(arg_settings_mask, SETTING_RESTRICT_ADDRESS_FAMILIES) && !arg_restrict_address_families)
                log_notice("Note: in a future version of systemd-nspawn the default set of permitted socket address"
                           " families will be restricted to AF_INET, AF_INET6 and AF_UNIX."
                           " Use --restrict-address-families= to configure the set of permitted socket address"
                           " families, or set RestrictAddressFamilies= in a .nspawn file.");
 
        /* If we're not unsharing the network namespace and are unsharing the user namespace, we won't have
         * permissions to bind ports in the container, so let's drop the CAP_NET_BIND_SERVICE capability to
         * indicate that. */
        if (!arg_private_network && arg_userns_mode != USER_NAMESPACE_NO)
                arg_caps_retain &= ~(UINT64_C(1) << CAP_NET_BIND_SERVICE);
 
        r = verify_arguments();
        if (r < 0)
                goto finish;
 
        r = split_boot_parameters();
        if (r < 0)
                goto finish;
 
        r = resolve_network_interface_names(arg_network_interfaces);
        if (r < 0)
                goto finish;
 
        r = verify_network_interfaces_initialized();
        if (r < 0)
                goto finish;
 
        /* Ignore SIGPIPE here, because we use splice() on the ptyfwd stuff and that will generate SIGPIPE if
         * the result is closed. Note that the container payload child will reset signal mask+handler anyway,
         * so just turning this off here means we only turn it off in nspawn itself, not any children. */
        (void) ignore_signals(SIGPIPE);
 
        r = fdset_new_listen_fds(&fds, /* unset= */ false);
        if (r < 0) {
                log_error_errno(r, "Failed to collect file descriptors: %m");
                goto finish;
        }
 
        r = acquire_listen_fdnames(fds, &fdnames);
        if (r < 0)
                goto finish;
 
        /* The "default" umask. This is appropriate for most file and directory
        * operations performed by nspawn, and is the umask that will be used for
        * the child. Functions like copy_devnodes() change the umask temporarily. */
        umask(0022);
 
        if (arg_console_mode < 0)
                arg_console_mode = isatty_safe(STDIN_FILENO) && isatty_safe(STDOUT_FILENO) ?
                                   CONSOLE_INTERACTIVE : CONSOLE_READ_ONLY;
 
        if (arg_console_mode == CONSOLE_AUTOPIPE) {
                if (isatty_safe(STDIN_FILENO) && isatty_safe(STDOUT_FILENO))
                        arg_console_mode = CONSOLE_INTERACTIVE;
                else
                        arg_console_mode = CONSOLE_PIPE;
        }
 
        if (arg_console_mode == CONSOLE_PIPE) /* if we pass STDERR on to the container, don't add our own logs into it too */
                arg_quiet = true;
 
        polkit_agent_open();
 
        if (arg_userns_mode == USER_NAMESPACE_MANAGED) {
                /* Let's allocate a 64K userns first, if managed mode is chosen */
 
                r = nsresource_connect(&nsresource_link);
                if (r < 0) {
                        log_error_errno(r, "Failed to connect to nsresourced: %m");
                        goto finish;
                }
 
                r = mountfsd_connect(&mountfsd_link);
                if (r < 0) {
                        log_error_errno(r, "Failed to connect to mountfsd: %m");
                        goto finish;
                }
 
                _cleanup_free_ char *userns_name = NULL;
                if (asprintf(&userns_name, "nspawn-" PID_FMT "-%s", getpid_cached(), arg_machine) < 0) {
                        r = log_oom();
                        goto finish;
                }
 
                userns_fd = nsresource_allocate_userns_full(
                                nsresource_link,
                                userns_name,
                                NSRESOURCE_UIDS_64K,
                                arg_delegate_container_ranges);
                if (userns_fd < 0) {
                        r = log_error_errno(userns_fd, "Failed to allocate user namespace with 64K users: %m");
                        goto finish;
                }
 
                r = userns_get_base_uid(userns_fd, &arg_uid_shift, /* ret_gid= */ NULL);
                if (r < 0) {
                        log_error_errno(r, "Failed to determine UID shift from userns: %m");
                        goto finish;
                }
 
                arg_uid_range = NSRESOURCE_UIDS_64K;
        }
 
        if (arg_directory) {
                assert(!arg_image);
 
                /* Safety precaution: let's not allow running images from the live host OS image, as long as
                 * /var from the host will propagate into container dynamically (because bad things happen if
                 * two systems write to the same /var). Let's allow it for the special cases where /var is
                 * either copied (i.e. --ephemeral) or replaced (i.e. --volatile=yes|state). */
                if (path_equal(arg_directory, "/") && !(arg_ephemeral || IN_SET(arg_volatile_mode, VOLATILE_YES, VOLATILE_STATE))) {
                        r = log_error_errno(SYNTHETIC_ERRNO(EINVAL),
                                            "Spawning container on root directory is not supported. Consider using --ephemeral, --volatile=yes or --volatile=state.");
                        goto finish;
                }
 
                if (arg_ephemeral) {
                        r = chase_and_update(&arg_directory, 0);
                        if (r < 0)
                                goto finish;
 
                        r = create_ephemeral_snapshot(
                                        arg_directory,
                                        arg_runtime_scope,
                                        arg_read_only,
                                        &tree_global_lock,
                                        &tree_local_lock,
                                        &snapshot_dir);
                        if (r < 0) {
                                log_error_errno(r, "Failed to create ephemeral snapshot: %m");
                                goto finish;
                        }
 
                        r = free_and_strdup(&arg_directory, snapshot_dir);
                        if (r < 0) {
                                log_oom();
                                goto finish;
                        }
                } else {
                        r = chase_and_update(&arg_directory, arg_template ? CHASE_NONEXISTENT : 0);
                        if (r < 0)
                                goto finish;
 
                        r = image_path_lock(
                                        arg_runtime_scope,
                                        arg_directory,
                                        (arg_read_only ? LOCK_SH : LOCK_EX) | LOCK_NB,
                                        arg_runtime_scope == RUNTIME_SCOPE_SYSTEM ? &tree_global_lock : NULL,
                                        &tree_local_lock);
                        if (r == -EBUSY) {
                                log_error_errno(r, "Directory tree %s is currently busy.", arg_directory);
                                goto finish;
                        }
                        if (r < 0) {
                                log_error_errno(r, "Failed to lock %s: %m", arg_directory);
                                goto finish;
                        }
 
                        if (arg_template) {
                                r = chase_and_update(&arg_template, 0);
                                if (r < 0)
                                        goto finish;
 
                                {
                                        BLOCK_SIGNALS(SIGINT);
                                        r = btrfs_subvol_snapshot_at(AT_FDCWD, arg_template, AT_FDCWD, arg_directory,
                                                                     (arg_read_only ? BTRFS_SNAPSHOT_READ_ONLY : 0) |
                                                                     BTRFS_SNAPSHOT_FALLBACK_COPY |
                                                                     BTRFS_SNAPSHOT_FALLBACK_DIRECTORY |
                                                                     BTRFS_SNAPSHOT_FALLBACK_IMMUTABLE |
                                                                     BTRFS_SNAPSHOT_RECURSIVE |
                                                                     BTRFS_SNAPSHOT_QUOTA |
                                                                     BTRFS_SNAPSHOT_SIGINT);
                                }
                                if (r == -EEXIST)
                                        log_full(arg_quiet ? LOG_DEBUG : LOG_INFO,
                                                 "Directory %s already exists, not populating from template %s.", arg_directory, arg_template);
                                else if (r == -EINTR) {
                                        log_error_errno(r, "Interrupted while copying file system tree to %s, removed again.", arg_directory);
                                        goto finish;
                                } else if (r < 0) {
                                        log_error_errno(r, "Couldn't create snapshot %s from %s: %m", arg_directory, arg_template);
                                        goto finish;
                                } else
                                        log_full(arg_quiet ? LOG_DEBUG : LOG_INFO,
                                                 "Populated %s from template %s.", arg_directory, arg_template);
                        }
                }
 
                if (arg_start_mode == START_BOOT) {
                        _cleanup_free_ char *b = NULL;
                        const char *p;
                        int check_os_release, is_os_tree;
 
                        if (arg_pivot_root_new) {
                                b = path_join(arg_directory, arg_pivot_root_new);
                                if (!b) {
                                        r = log_oom();
                                        goto finish;
                                }
 
                                p = b;
                        } else
                                p = arg_directory;
 
                        check_os_release = getenv_bool("SYSTEMD_NSPAWN_CHECK_OS_RELEASE");
                        if (check_os_release < 0 && check_os_release != -ENXIO) {
                                r = log_error_errno(check_os_release, "Failed to parse $SYSTEMD_NSPAWN_CHECK_OS_RELEASE: %m");
                                goto finish;
                        }
 
                        is_os_tree = path_is_os_tree(p);
                        if (is_os_tree == 0 && check_os_release == 0)
                                log_debug("Directory %s is missing an os-release file, continuing anyway.", p);
                        else if (is_os_tree <= 0) {
                                r = log_error_errno(SYNTHETIC_ERRNO(EINVAL),
                                                    "Directory %s doesn't look like an OS root directory (os-release file is missing). Refusing.", p);
                                goto finish;
                        }
                } else {
                        _cleanup_free_ char *p = NULL;
 
                        if (arg_pivot_root_new)
                                p = path_join(arg_directory, arg_pivot_root_new, "/usr/");
                        else
                                p = path_join(arg_directory, "/usr/");
                        if (!p) {
                                r = log_oom();
                                goto finish;
                        }
 
                        r = access_nofollow(p, F_OK);
                        if (r == -ENOENT) {
                                log_error_errno(r, "Directory %s doesn't look like it has an OS tree (/usr/ directory is missing). Refusing.", arg_directory);
                                goto finish;
                        } else if (r < 0) {
                                log_error_errno(r, "Unable to determine if %s looks like it has an OS tree (i.e. whether /usr/ exists): %m", arg_directory);
                                goto finish;
                        }
                }
 
                if (userns_fd >= 0) {
                        r = mountfsd_mount_directory(
                                        mountfsd_link,
                                        arg_directory,
                                        userns_fd,
                                        determine_dissect_image_flags(),
                                        &mount_fd);
                        if (r < 0) {
                                log_error_errno(r, "Failed to mount directory via mountfsd: %m");
                                goto finish;
                        }
                }
 
        } else if (arg_image) {
                DissectImageFlags dissect_image_flags =
                        determine_dissect_image_flags();
 
                assert(!arg_template);
 
                r = chase_and_update(&arg_image, 0);
                if (r < 0)
                        goto finish;
 
                if (arg_ephemeral)  {
                        _cleanup_free_ char *np = NULL;
 
                        r = tempfn_random(arg_image, "machine.", &np);
                        if (r < 0) {
                                log_error_errno(r, "Failed to generate name for image snapshot: %m");
                                goto finish;
                        }
 
                        /* Always take an exclusive lock on our own ephemeral copy. */
                        r = image_path_lock(
                                        arg_runtime_scope,
                                        np,
                                        LOCK_EX|LOCK_NB,
                                        arg_runtime_scope == RUNTIME_SCOPE_SYSTEM ? &tree_global_lock : NULL,
                                        &tree_local_lock);
                        if (r < 0) {
                                log_error_errno(r, "Failed to create image lock: %m");
                                goto finish;
                        }
 
                        {
                                BLOCK_SIGNALS(SIGINT);
                                r = copy_file(arg_image, np, O_EXCL, arg_read_only ? 0400 : 0600,
                                              COPY_CRTIME|COPY_SIGINT|COPY_NOCOW_AFTER);
                        }
                        if (r == -EINTR) {
                                log_error_errno(r, "Interrupted while copying image file to %s, removed again.", np);
                                goto finish;
                        }
                        if (r < 0) {
                                r = log_error_errno(r, "Failed to copy image file: %m");
                                goto finish;
                        }
 
                        free_and_replace(arg_image, np);
                        remove_image = true;
                } else {
                        r = image_path_lock(
                                        arg_runtime_scope,
                                        arg_image,
                                        (arg_read_only ? LOCK_SH : LOCK_EX) | LOCK_NB,
                                        arg_runtime_scope == RUNTIME_SCOPE_SYSTEM ? &tree_global_lock : NULL,
                                        &tree_local_lock);
                        if (r == -EBUSY) {
                                log_error_errno(r, "Disk image %s is currently busy.", arg_image);
                                goto finish;
                        }
                        if (r < 0) {
                                log_error_errno(r, "Failed to create image lock: %m");
                                goto finish;
                        }
 
                        r = verity_settings_load(
                                        &arg_verity_settings,
                                        arg_image, NULL, NULL);
                        if (r < 0) {
                                log_error_errno(r, "Failed to read verity artefacts for %s: %m", arg_image);
                                goto finish;
                        }
 
                        if (arg_verity_settings.data_path)
                                dissect_image_flags |= DISSECT_IMAGE_NO_PARTITION_TABLE;
                }
 
                if (arg_userns_mode != USER_NAMESPACE_MANAGED) {
                        r = loop_device_make_by_path(
                                        arg_image,
                                        arg_read_only ? O_RDONLY : -1,
                                        /* sector_size= */ UINT32_MAX,
                                        FLAGS_SET(dissect_image_flags, DISSECT_IMAGE_NO_PARTITION_TABLE) ? 0 : LO_FLAGS_PARTSCAN,
                                        LOCK_SH,
                                        &loop);
                        if (r < 0) {
                                log_error_errno(r, "Failed to set up loopback block device: %m");
                                goto finish;
                        }
 
                        r = dissect_loop_device_and_warn(
                                        loop,
                                        &arg_verity_settings,
                                        /* mount_options= */ NULL,
                                        arg_image_policy ?: &image_policy_container,
                                        /* image_filter= */ NULL,
                                        dissect_image_flags,
                                        &dissected_image);
                        if (r == -ENOPKG) {
                                /* dissected_image_and_warn() already printed a brief error message. Extend on that with more details */
                                log_notice("Note that the disk image needs to\n"
                                           "    a) either contain only a single MBR partition of type 0x83 that is marked bootable\n"
                                           "    b) or contain a single GPT partition of type 0FC63DAF-8483-4772-8E79-3D69D8477DE4\n"
                                           "    c) or follow https://uapi-group.org/specifications/specs/discoverable_partitions_specification\n"
                                           "    d) or contain a file system without a partition table\n"
                                           "in order to be bootable with systemd-nspawn.");
                                goto finish;
                        }
                        if (r < 0)
                                goto finish;
 
                        r = dissected_image_load_verity_sig_partition(
                                        dissected_image,
                                        loop->fd,
                                        &arg_verity_settings);
                        if (r < 0) {
                                log_error_errno(r, "Failed to load Verity signature partition: %m");
                                goto finish;
                        }
 
                        r = dissected_image_guess_verity_roothash(
                                        dissected_image,
                                        &arg_verity_settings);
                        if (r < 0) {
                                log_error_errno(r, "Failed to guess Verity root hash: %m");
                                goto finish;
                        }
 
                        if (dissected_image->has_verity && !iovec_is_set(&arg_verity_settings.root_hash))
                                log_notice("Note: image %s contains verity information, but no root hash specified and no embedded "
                                           "root hash signature found! Proceeding without integrity checking.", arg_image);
 
                        r = dissected_image_decrypt_interactively(
                                        dissected_image,
                                        NULL,
                                        &arg_verity_settings,
                                        arg_image_policy ?: &image_policy_container,
                                        dissect_image_flags);
                        if (r < 0)
                                goto finish;
                } else {
                        r = mountfsd_mount_image(
                                        mountfsd_link,
                                        arg_image,
                                        userns_fd,
                                        /* options= */ NULL,
                                        arg_image_policy,
                                        &arg_verity_settings,
                                        dissect_image_flags,
                                        &dissected_image);
                        if (r < 0) {
                                log_error_errno(r, "Failed to mount image via mountfsd: %m");
                                goto finish;
                        }
                }
 
                /* Now that we mounted the image, let's try to remove it again, if it is ephemeral */
                if (remove_image && unlink(arg_image) >= 0)
                        remove_image = false;
 
                if (arg_architecture < 0)
                        arg_architecture = dissected_image_architecture(dissected_image);
 
        } else if (arg_mstack) {
                assert(!arg_template);
                assert(!arg_ephemeral);
 
                r = chase_and_update(&arg_mstack, CHASE_MUST_BE_DIRECTORY);
                if (r < 0)
                        goto finish;
 
                if (!IN_SET(arg_userns_mode, USER_NAMESPACE_NO, USER_NAMESPACE_MANAGED))
                        return log_error_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), "--mstack= requires managed user namespacing, or user namespace turned off.");
 
                MStackFlags mstack_flags = arg_read_only ? MSTACK_RDONLY : 0;
                r = mstack_load(arg_mstack,
                                /* dir_fd= */ -EBADF,
                                &mstack);
                if (r < 0) {
                        log_error_errno(r, "Failed to load .mstack/ directory '%s': %m", arg_mstack);
                        goto finish;
                }
 
                r = mstack_open_images(
                                mstack,
                                mountfsd_link,
                                userns_fd,
                                arg_image_policy,
                                /* image_filter= */ NULL,
                                mstack_flags);
                if (r < 0) {
                        log_error_errno(r, "Failed to open .mstack/ layer images '%s': %m", arg_mstack);
                        goto finish;
                }
        } else
                assert_not_reached();
 
        r = runtime_directory_make(
                        arg_runtime_scope,
                        "systemd/nspawn",
                        arg_machine,
                        &runtime_dir);
        if (r < 0) {
                log_error_errno(r, "Failed to create runtime directory: %m");
                goto finish;
        }
 
        /* Create a temporary place to mount stuff. */
        r = mkdtemp_malloc("/tmp/nspawn-root-XXXXXX", &rootdir);
        if (r < 0) {
                log_error_errno(r, "Failed to create temporary directory: %m");
                goto finish;
        }
 
        r = custom_mount_prepare_all(rootdir, arg_custom_mounts, arg_n_custom_mounts);
        if (r < 0)
                goto finish;
 
        mountfsd_link = sd_varlink_unref(mountfsd_link);
        nsresource_link = sd_varlink_unref(nsresource_link);
 
        if (!arg_quiet) {
                const char *t = arg_mstack ?: arg_image ?: arg_directory;
                _cleanup_free_ char *u = NULL;
                (void) terminal_urlify_path(t, t, &u);
 
                log_info("%s %sSpawning container %s on %s.%s",
                         glyph(GLYPH_LIGHT_SHADE), ansi_grey(), arg_machine, u ?: t, ansi_normal());
 
                if (arg_console_mode == CONSOLE_INTERACTIVE)
                        log_info("%s %sPress %sCtrl-]%s three times within 1s to kill container; two times followed by %sr%s\n"
                                 "%s %sto reboot container; two times followed by %sp%s to poweroff container.%s",
                                 glyph(GLYPH_LIGHT_SHADE), ansi_grey(), ansi_highlight(), ansi_grey(), ansi_highlight(), ansi_normal(),
                                 glyph(GLYPH_LIGHT_SHADE), ansi_grey(), ansi_highlight(), ansi_grey(), ansi_normal());
        }
 
        assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGCHLD, SIGWINCH, SIGTERM, SIGINT, SIGRTMIN+18) >= 0);
 
        r = make_reaper_process(true);
        if (r < 0) {
                log_error_errno(r, "Failed to become subreaper: %m");
                goto finish;
        }
 
        if (arg_expose_ports) {
                r = sd_nfnl_socket_open(&nfnl);
                if (r < 0) {
                        log_error_errno(r, "Cannot expose configured ports, failed to initialize nftables: %m");
                        goto finish;
                }
                expose_args.nfnl = nfnl;
        }
 
        if (arg_forward_journal) {
                _cleanup_free_ char *socket_path = path_join(runtime_dir, "journal-remote-socket");
                if (!socket_path) {
                        r = log_oom();
                        goto finish;
                }
 
                union sockaddr_union sa;
                r = sockaddr_un_set_path(&sa.un, socket_path);
                if (r < 0) {
                        log_error_errno(r, "Failed to set AF_UNIX path to '%s': %m", socket_path);
                        goto finish;
                }
 
                (void) sockaddr_un_unlink(&sa.un);
 
                r = fork_journal_remote(
                                socket_path,
                                arg_forward_journal,
                                arg_forward_journal_max_use,
                                arg_forward_journal_keep_free,
                                arg_forward_journal_max_file_size,
                                arg_forward_journal_max_files,
                                &journal_remote_pidref);
                if (r < 0)
                        goto finish;
 
                CustomMount *cm = custom_mount_add(&arg_custom_mounts, &arg_n_custom_mounts, CUSTOM_MOUNT_BIND);
                if (!cm) {
                        r = log_oom();
                        goto finish;
                }
 
                cm->source = TAKE_PTR(socket_path);
                cm->read_only = true;
                cm->destination = strdup(NSPAWN_JOURNAL_SOCKET_PATH);
                if (!cm->destination) {
                        r = log_oom();
                        goto finish;
                }
 
                r = machine_credential_add(&arg_credentials, "journal.forward_to_socket", NSPAWN_JOURNAL_SOCKET_PATH, SIZE_MAX);
                if (r == -EEXIST) {
                        log_error_errno(r, "Credential 'journal.forward_to_socket' already set via --set-credential=, refusing --forward-journal=.");
                        goto finish;
                }
                if (r < 0) {
                        log_error_errno(r, "Failed to add 'journal.forward_to_socket' credential: %m");
                        goto finish;
                }
        }
 
        for (;;) {
                r = run_container(
                                runtime_dir,
                                rootdir,
                                mount_fd,
                                dissected_image,
                                mstack,
                                userns_fd,
                                fds,
                                fdnames,
                                veth_name,
                                &veth_created,
                                &expose_args, &master,
                                &pid, &ret);
                if (r <= 0)
                        break;
        }
 
finish:
        (void) sd_notify(false,
                         r == 0 && ret == EXIT_FORCE_RESTART ? "STOPPING=1\nSTATUS=Restarting..." :
                                                               "STOPPING=1\nSTATUS=Terminating...");
 
        if (pidref_is_set(&pid))
                (void) pidref_kill(&pid, SIGKILL);
 
        /* Try to flush whatever is still queued in the pty */
        if (master >= 0) {
                (void) copy_bytes(master, STDOUT_FILENO, UINT64_MAX, 0);
                master = safe_close(master);
        }
 
        if (pidref_is_set(&pid)) {
                (void) pidref_wait_for_terminate(&pid, NULL);
                pidref_done(&pid);
        }
 
        pager_close();
 
        if (remove_image && arg_image) {
                if (unlink(arg_image) < 0)
                        log_warning_errno(errno, "Can't remove image file '%s', ignoring: %m", arg_image);
        }
 
        cleanup_propagation_and_export_directories(runtime_dir);
 
        expose_port_flush(nfnl, arg_expose_ports, AF_INET,  &expose_args.address4);
        expose_port_flush(nfnl, arg_expose_ports, AF_INET6, &expose_args.address6);
 
        if (arg_userns_mode != USER_NAMESPACE_MANAGED) {
                if (veth_created)
                        (void) remove_veth_links(veth_name, arg_network_veth_extra);
                (void) remove_bridge(arg_network_zone);
        }
 
        custom_mount_free_all(arg_custom_mounts, arg_n_custom_mounts);
        expose_port_free_all(arg_expose_ports);
        rlimit_free_all(arg_rlimit);
        device_node_array_free(arg_extra_nodes, arg_n_extra_nodes);
 
        if (r < 0)
                return r;
 
        return ret;
}
 
DEFINE_MAIN_FUNCTION_WITH_POSITIVE_FAILURE(run);