HALF ECHO LEARN / REFERENCE COLLECTION

v262.2

src/core/cgroup.c

179,588 bytesGit mode 100644Download original fileLGPL-2.1-or-later

Git blob: 543d1ac8e3c43a5af0861e2f1a038485663554eb

Lines 3001–4000 of 4,696

 
        LIST_REMOVE(cgroup_empty_queue, u->manager->cgroup_empty_queue, u);
        u->in_cgroup_empty_queue = false;
}
 
int unit_check_oomd_kill(Unit *u) {
        _cleanup_free_ char *value = NULL;
        bool increased;
        uint64_t n = 0;
        int r;
 
        assert(u);
 
        CGroupRuntime *crt = unit_get_cgroup_runtime(u);
        if (!crt || !crt->cgroup_path)
                return 0;
 
        r = cg_get_xattr(crt->cgroup_path, "user.oomd_ooms", &value, /* ret_size= */ NULL);
        if (r < 0 && !ERRNO_IS_XATTR_ABSENT(r))
                return r;
 
        if (!isempty(value)) {
                 r = safe_atou64(value, &n);
                 if (r < 0)
                         return r;
        }
 
        increased = n > crt->managed_oom_kill_last;
        crt->managed_oom_kill_last = n;
 
        if (!increased)
                return 0;
 
        n = 0;
        value = mfree(value);
        r = cg_get_xattr(crt->cgroup_path, "user.oomd_kill", &value, /* ret_size= */ NULL);
        if (r >= 0 && !isempty(value))
                (void) safe_atou64(value, &n);
 
        if (n > 0)
                log_unit_struct(u, LOG_NOTICE,
                                LOG_MESSAGE_ID(SD_MESSAGE_UNIT_OOMD_KILL_STR),
                                LOG_UNIT_INVOCATION_ID(u),
                                LOG_UNIT_MESSAGE(u, "systemd-oomd killed %"PRIu64" process(es) in this unit.", n),
                                LOG_ITEM("N_PROCESSES=%" PRIu64, n));
        else
                log_unit_struct(u, LOG_NOTICE,
                                LOG_MESSAGE_ID(SD_MESSAGE_UNIT_OOMD_KILL_STR),
                                LOG_UNIT_INVOCATION_ID(u),
                                LOG_UNIT_MESSAGE(u, "systemd-oomd killed some process(es) in this unit."));
 
        unit_notify_cgroup_oom(u, /* managed_oom= */ true);
 
        return 1;
}
 
int unit_check_oom(Unit *u) {
        bool increased;
        uint64_t c = 0;
        int r;
 
        CGroupRuntime *crt = unit_get_cgroup_runtime(u);
        if (!crt || !crt->cgroup_path)
                return 0;
 
        CGroupContext *ctx = ASSERT_PTR(unit_get_cgroup_context(u));
 
        /* If memory.oom.group=1, then look up the oom_group_kill field, which reports how many times the
         * kernel killed every process recursively in this cgroup and its descendants, similar to
         * systemd-oomd. Because the memory.events.local file was only introduced in kernel 5.12, we fall
         * back to reading oom_kill if we can't find the file or field. */
 
        if (ctx->memory_oom_group) {
                r = cg_get_keyed_attribute_uint64(
                                crt->cgroup_path,
                                "memory.events.local",
                                "oom_group_kill",
                                &c);
                if (r < 0 && !IN_SET(r, -ENOENT, -ENXIO))
                        return log_unit_debug_errno(u, r, "Failed to read oom_group_kill field of memory.events.local cgroup attribute, ignoring: %m");
        }
 
        if (!ctx->memory_oom_group || r < 0) {
                r = cg_get_keyed_attribute_uint64(
                                crt->cgroup_path,
                                "memory.events",
                                "oom_kill",
                                &c);
                if (r < 0 && !IN_SET(r, -ENOENT, -ENXIO))
                        return log_unit_debug_errno(u, r, "Failed to read oom_kill field of memory.events cgroup attribute: %m");
        }
 
        increased = c > crt->oom_kill_last;
        crt->oom_kill_last = c;
 
        if (!increased)
                return 0;
 
        log_unit_struct(u, LOG_NOTICE,
                        LOG_MESSAGE_ID(SD_MESSAGE_UNIT_OUT_OF_MEMORY_STR),
                        LOG_UNIT_INVOCATION_ID(u),
                        LOG_UNIT_MESSAGE(u, "The kernel OOM killer killed some processes in this unit."));
 
        unit_notify_cgroup_oom(u, /* managed_oom= */ false);
 
        return 1;
}
 
static int on_cgroup_oom_event(sd_event_source *s, void *userdata) {
        Manager *m = ASSERT_PTR(userdata);
        Unit *u;
        int r;
 
        assert(s);
 
        u = m->cgroup_oom_queue;
        if (!u)
                return 0;
 
        assert(u->in_cgroup_oom_queue);
        u->in_cgroup_oom_queue = false;
        LIST_REMOVE(cgroup_oom_queue, m->cgroup_oom_queue, u);
 
        if (m->cgroup_oom_queue) {
                /* More stuff queued, let's make sure we remain enabled */
                r = sd_event_source_set_enabled(s, SD_EVENT_ONESHOT);
                if (r < 0)
                        log_debug_errno(r, "Failed to reenable cgroup oom event source, ignoring: %m");
        }
 
        (void) unit_check_oom(u);
        unit_add_to_gc_queue(u);
 
        return 0;
}
 
static void unit_add_to_cgroup_oom_queue(Unit *u) {
        int r;
 
        assert(u);
 
        if (u->in_cgroup_oom_queue)
                return;
 
        CGroupRuntime *crt = unit_get_cgroup_runtime(u);
        if (!crt || !crt->cgroup_path)
                return;
 
        LIST_PREPEND(cgroup_oom_queue, u->manager->cgroup_oom_queue, u);
        u->in_cgroup_oom_queue = true;
 
        /* Trigger the defer event */
        if (!u->manager->cgroup_oom_event_source) {
                _cleanup_(sd_event_source_unrefp) sd_event_source *s = NULL;
 
                r = sd_event_add_defer(u->manager->event, &s, on_cgroup_oom_event, u->manager);
                if (r < 0) {
                        log_error_errno(r, "Failed to create cgroup oom event source: %m");
                        return;
                }
 
                r = sd_event_source_set_priority(s, EVENT_PRIORITY_CGROUP_OOM);
                if (r < 0) {
                        log_error_errno(r, "Failed to set priority of cgroup oom event source: %m");
                        return;
                }
 
                (void) sd_event_source_set_description(s, "cgroup-oom");
                u->manager->cgroup_oom_event_source = TAKE_PTR(s);
        }
 
        r = sd_event_source_set_enabled(u->manager->cgroup_oom_event_source, SD_EVENT_ONESHOT);
        if (r < 0)
                log_error_errno(r, "Failed to enable cgroup oom event source: %m");
}
 
static int unit_check_cgroup_events(Unit *u) {
        char *values[2] = {};
        int r;
 
        assert(u);
 
        CGroupRuntime *crt = unit_get_cgroup_runtime(u);
        if (!crt || !crt->cgroup_path)
                return 0;
 
        r = cg_get_keyed_attribute(
                        crt->cgroup_path,
                        "cgroup.events",
                        STRV_MAKE("populated", "frozen"),
                        values);
        if (r < 0)
                return r;
 
        /* The cgroup.events notifications can be merged together so act as we saw the given state for the
         * first time. The functions we call to handle given state are idempotent, which makes them
         * effectively remember the previous state. */
        if (streq(values[0], "1"))
                unit_remove_from_cgroup_empty_queue(u);
        else
                unit_add_to_cgroup_empty_queue(u);
 
        /* Disregard freezer state changes due to operations not initiated by us.
         * See: https://github.com/systemd/systemd/pull/13512/files#r416469963 and
         *      https://github.com/systemd/systemd/pull/13512#issuecomment-573007207 */
        if (IN_SET(u->freezer_state, FREEZER_FREEZING, FREEZER_FREEZING_BY_PARENT, FREEZER_THAWING))
                unit_freezer_complete(u, streq(values[1], "0") ? FREEZER_RUNNING : FREEZER_FROZEN);
 
        free_many_charp(values, ELEMENTSOF(values));
        return 0;
}
 
static int on_cgroup_inotify_event(sd_event_source *s, int fd, uint32_t revents, void *userdata) {
        Manager *m = ASSERT_PTR(userdata);
 
        assert(s);
        assert(fd >= 0);
 
        for (;;) {
                union inotify_event_buffer buffer;
                ssize_t l;
 
                l = read(fd, &buffer, sizeof(buffer));
                if (l < 0) {
                        if (ERRNO_IS_TRANSIENT(errno))
                                return 0;
 
                        return log_error_errno(errno, "Failed to read control group inotify events: %m");
                }
 
                FOREACH_INOTIFY_EVENT_WARN(e, buffer, l) {
                        Unit *u;
 
                        if (e->wd < 0)
                                /* Queue overflow has no watch descriptor */
                                continue;
 
                        if (e->mask & IN_IGNORED)
                                /* The watch was just removed */
                                continue;
 
                        /* Note that inotify might deliver events for a watch even after it was removed,
                         * because it was queued before the removal. Let's ignore this here safely. */
 
                        u = hashmap_get(m->cgroup_control_inotify_wd_unit, INT_TO_PTR(e->wd));
                        if (u)
                                unit_check_cgroup_events(u);
 
                        u = hashmap_get(m->cgroup_memory_inotify_wd_unit, INT_TO_PTR(e->wd));
                        if (u)
                                unit_add_to_cgroup_oom_queue(u);
                }
        }
}
 
static int cg_bpf_mask_supported(CGroupMask *ret) {
        CGroupMask mask = 0;
        int r;
 
        assert(ret);
 
        /* BPF-based firewall, device access control, and pinned foreign prog */
        if (bpf_program_supported() > 0)
                mask |= CGROUP_MASK_BPF_FIREWALL |
                        CGROUP_MASK_BPF_DEVICES |
                        CGROUP_MASK_BPF_FOREIGN;
 
        /* BPF-based bind{4|6} hooks */
        r = bpf_socket_bind_supported();
        if (r < 0)
                return r;
        if (r > 0)
                mask |= CGROUP_MASK_BPF_SOCKET_BIND;
 
        /* BPF-based cgroup_skb/{egress|ingress} hooks */
        r = bpf_restrict_ifaces_supported();
        if (r < 0)
                return r;
        if (r > 0)
                mask |= CGROUP_MASK_BPF_RESTRICT_NETWORK_INTERFACES;
 
        /* BPF-based cgroup/sock_create hooks */
        r = bpf_bind_network_interface_supported();
        if (r < 0)
                return r;
        if (r > 0)
                mask |= CGROUP_MASK_BPF_BIND_NETWORK_INTERFACE;
 
        *ret = mask;
        return 0;
}
 
int manager_setup_cgroup(Manager *m) {
        int r;
 
        assert(m);
 
        /* 1. Determine hierarchy */
        m->cgroup_root = mfree(m->cgroup_root);
        r = cg_pid_get_path(0, &m->cgroup_root);
        if (r < 0)
                return log_error_errno(r, "Cannot determine cgroup we are running in: %m");
 
        /* Chop off the init scope, if we are already located in it */
        char *e = endswith(m->cgroup_root, "/" SPECIAL_INIT_SCOPE);
        if (e)
                *e = 0;
 
        /* And make sure to store away the root value without trailing slash, even for the root dir, so that we can
         * easily prepend it everywhere. */
        delete_trailing_chars(m->cgroup_root, "/");
 
        /* 2. Pin the cgroupfs mount, so that it cannot be unmounted */
        safe_close(m->pin_cgroupfs_fd);
        m->pin_cgroupfs_fd = open("/sys/fs/cgroup", O_PATH|O_CLOEXEC|O_DIRECTORY);
        if (m->pin_cgroupfs_fd < 0)
                return log_error_errno(errno, "Failed to pin cgroup hierarchy: %m");
 
        /* 3. Allocate cgroup empty defer event source */
        m->cgroup_empty_event_source = sd_event_source_disable_unref(m->cgroup_empty_event_source);
        r = sd_event_add_defer(m->event, &m->cgroup_empty_event_source, on_cgroup_empty_event, m);
        if (r < 0)
                return log_error_errno(r, "Failed to create cgroup empty event source: %m");
 
        /* Schedule cgroup empty checks early, but after having processed service notification messages or
         * SIGCHLD signals, so that a cgroup running empty is always just the last safety net of
         * notification, and we collected the metadata the notification and SIGCHLD stuff offers first. */
        r = sd_event_source_set_priority(m->cgroup_empty_event_source, EVENT_PRIORITY_CGROUP_EMPTY);
        if (r < 0)
                return log_error_errno(r, "Failed to set priority of cgroup empty event source: %m");
 
        r = sd_event_source_set_enabled(m->cgroup_empty_event_source, SD_EVENT_OFF);
        if (r < 0)
                return log_error_errno(r, "Failed to disable cgroup empty event source: %m");
 
        (void) sd_event_source_set_description(m->cgroup_empty_event_source, "cgroup-empty");
 
        /* 4. Install cgroup empty event notifier inotify object */
        m->cgroup_inotify_event_source = sd_event_source_disable_unref(m->cgroup_inotify_event_source);
        safe_close(m->cgroup_inotify_fd);
 
        m->cgroup_inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
        if (m->cgroup_inotify_fd < 0)
                return log_error_errno(errno, "Failed to create control group inotify object: %m");
 
        r = sd_event_add_io(m->event, &m->cgroup_inotify_event_source, m->cgroup_inotify_fd, EPOLLIN, on_cgroup_inotify_event, m);
        if (r < 0)
                return log_error_errno(r, "Failed to watch control group inotify object: %m");
 
        /* Process cgroup empty notifications early. Note that when this event is dispatched it'll
         * just add the unit to a cgroup empty queue, hence let's run earlier than that. Also see
         * handling of cgroup agent notifications, for the classic cgroup hierarchy support. */
        r = sd_event_source_set_priority(m->cgroup_inotify_event_source, EVENT_PRIORITY_CGROUP_INOTIFY);
        if (r < 0)
                return log_error_errno(r, "Failed to set priority of inotify event source: %m");
 
        (void) sd_event_source_set_description(m->cgroup_inotify_event_source, "cgroup-inotify");
 
        /* 5. Make sure we are in the special "init.scope" unit in the root slice. */
        const char *scope_path = strjoina(m->cgroup_root, "/" SPECIAL_INIT_SCOPE);
        r = cg_create_and_attach(scope_path, /* pid= */ 0);
        if (r >= 0) {
                /* Also, move all other userspace processes remaining in the root cgroup into that scope. */
                r = cg_migrate(m->cgroup_root, scope_path, 0);
                if (r < 0)
                        log_warning_errno(r, "Couldn't move remaining userspace processes, ignoring: %m");
 
        } else if (!MANAGER_IS_TEST_RUN(m))
                return log_error_errno(r, "Failed to create %s control group: %m", scope_path);
 
        /* 6. Figure out which controllers are supported */
        r = cg_mask_supported_subtree(m->cgroup_root, &m->cgroup_supported);
        if (r < 0)
                return log_error_errno(r, "Failed to determine supported controllers: %m");
 
        /* 7. Figure out which bpf-based pseudo-controllers are supported */
        CGroupMask mask;
        r = cg_bpf_mask_supported(&mask);
        if (r < 0)
                return log_error_errno(r, "Failed to determine supported bpf-based pseudo-controllers: %m");
        m->cgroup_supported |= mask;
 
        /* 8. Log which controllers are supported */
        for (CGroupController c = 0; c < _CGROUP_CONTROLLER_MAX; c++)
                log_debug("Controller '%s' supported: %s", cgroup_controller_to_string(c),
                          yes_no(m->cgroup_supported & CGROUP_CONTROLLER_TO_MASK(c)));
 
        return 0;
}
 
void manager_shutdown_cgroup(Manager *m, bool delete) {
        assert(m);
 
        /* We can't really delete the group, since we are in it. But
         * let's trim it. */
        if (delete && m->cgroup_root && !FLAGS_SET(m->test_run_flags, MANAGER_TEST_RUN_MINIMAL))
                (void) cg_trim(m->cgroup_root, false);
 
        m->cgroup_empty_event_source = sd_event_source_disable_unref(m->cgroup_empty_event_source);
 
        m->cgroup_control_inotify_wd_unit = hashmap_free(m->cgroup_control_inotify_wd_unit);
        m->cgroup_memory_inotify_wd_unit = hashmap_free(m->cgroup_memory_inotify_wd_unit);
 
        m->cgroup_inotify_event_source = sd_event_source_disable_unref(m->cgroup_inotify_event_source);
        m->cgroup_inotify_fd = safe_close(m->cgroup_inotify_fd);
 
        m->pin_cgroupfs_fd = safe_close(m->pin_cgroupfs_fd);
 
        m->cgroup_root = mfree(m->cgroup_root);
}
 
Unit* manager_get_unit_by_cgroup(Manager *m, const char *cgroup) {
        char *p;
        Unit *u;
 
        assert(m);
        assert(cgroup);
 
        u = hashmap_get(m->cgroup_unit, cgroup);
        if (u)
                return u;
 
        p = strdupa_safe(cgroup);
        for (;;) {
                char *e;
 
                e = strrchr(p, '/');
                if (!e || e == p)
                        return NULL; /* reached cgroup root? return NULL and possibly fall back to manager_get_unit_by_pidref_watching() */
 
                *e = 0;
 
                u = hashmap_get(m->cgroup_unit, p);
                if (u)
                        return u;
        }
}
 
Unit* manager_get_unit_by_pidref_cgroup(Manager *m, const PidRef *pid) {
        _cleanup_free_ char *cgroup = NULL;
 
        assert(m);
 
        if (cg_pidref_get_path(pid, &cgroup) < 0)
                return NULL;
 
        return manager_get_unit_by_cgroup(m, cgroup);
}
 
Unit* manager_get_unit_by_pidref_watching(Manager *m, const PidRef *pid) {
        Unit *u, **array;
 
        assert(m);
 
        if (!pidref_is_set(pid))
                return NULL;
 
        u = hashmap_get(m->watch_pids, pid);
        if (u)
                return u;
 
        array = hashmap_get(m->watch_pids_more, pid);
        if (array)
                return array[0];
 
        return NULL;
}
 
Unit* manager_get_unit_by_pidref(Manager *m, PidRef *pid) {
        Unit *u;
 
        assert(m);
 
        /* Note that a process might be owned by multiple units, we return only one here, which is good
         * enough for most cases, though not strictly correct. We prefer the one reported by cgroup
         * membership, as that's the most relevant one as children of the process will be assigned to that
         * one, too, before all else. */
 
        if (!pidref_is_set(pid))
                return NULL;
 
        if (pidref_is_self(pid))
                return hashmap_get(m->units, SPECIAL_INIT_SCOPE);
        if (pid->pid == 1)
                return NULL;
 
        u = manager_get_unit_by_pidref_cgroup(m, pid);
        if (u)
                return u;
 
        u = manager_get_unit_by_pidref_watching(m, pid);
        if (u)
                return u;
 
        return NULL;
}
 
int unit_get_memory_available(Unit *u, uint64_t *ret) {
        uint64_t available = UINT64_MAX, current = 0;
 
        assert(u);
        assert(ret);
 
        /* If data from cgroups can be accessed, try to find out how much more memory a unit can
         * claim before hitting the configured cgroup limits (if any). Consider both MemoryHigh
         * and MemoryMax, and also any slice the unit might be nested below. */
 
        do {
                uint64_t unit_available, unit_limit = UINT64_MAX;
                CGroupContext *unit_context;
 
                /* No point in continuing if we can't go any lower */
                if (available == 0)
                        break;
 
                unit_context = unit_get_cgroup_context(u);
                if (!unit_context)
                        return -ENODATA;
 
                (void) unit_get_memory_accounting(u, CGROUP_MEMORY_CURRENT, &current);
                /* in case of error, previous current propagates as lower bound */
 
                if (unit_has_name(u, SPECIAL_ROOT_SLICE))
                        unit_limit = physical_memory();
                else if (unit_context->memory_max == UINT64_MAX && unit_context->memory_high == UINT64_MAX)
                        continue;
                unit_limit = MIN3(unit_limit, unit_context->memory_max, unit_context->memory_high);
 
                unit_available = LESS_BY(unit_limit, current);
                available = MIN(unit_available, available);
        } while ((u = UNIT_GET_SLICE(u)));
 
        *ret = available;
 
        return 0;
}
 
int unit_get_memory_accounting(Unit *u, CGroupMemoryAccountingMetric metric, uint64_t *ret) {
 
        static const char* const attributes_table[_CGROUP_MEMORY_ACCOUNTING_METRIC_MAX] = {
                [CGROUP_MEMORY_CURRENT]       = "memory.current",
                [CGROUP_MEMORY_PEAK]          = "memory.peak",
                [CGROUP_MEMORY_SWAP_CURRENT]  = "memory.swap.current",
                [CGROUP_MEMORY_SWAP_PEAK]     = "memory.swap.peak",
                [CGROUP_MEMORY_ZSWAP_CURRENT] = "memory.zswap.current",
        };
 
        uint64_t bytes;
        bool updated = false;
        int r;
 
        assert(u);
        assert(metric >= 0);
        assert(metric < _CGROUP_MEMORY_ACCOUNTING_METRIC_MAX);
 
        if (!UNIT_CGROUP_BOOL(u, memory_accounting))
                return -ENODATA;
 
        /* The root cgroup doesn't expose this information. */
        if (unit_has_host_root_cgroup(u)) {
                /* System-wide memory usage can be acquired from /proc/ */
                if (metric == CGROUP_MEMORY_CURRENT)
                        return procfs_memory_get_used(ret);
 
                return -ENODATA;
        }
 
        CGroupRuntime *crt = unit_get_cgroup_runtime(u);
        if (!crt)
                return -ENODATA;
        if (!crt->cgroup_path)
                /* If the cgroup is already gone, we try to find the last cached value. */
                goto finish;
 
        if (!FLAGS_SET(crt->cgroup_realized_mask, CGROUP_MASK_MEMORY))
                return -ENODATA;
 
        r = cg_get_attribute_as_uint64(crt->cgroup_path, attributes_table[metric], &bytes);
        if (r < 0 && r != -ENODATA)
                return r;
        updated = r >= 0;
 
finish:
        if (metric <= _CGROUP_MEMORY_ACCOUNTING_METRIC_CACHED_LAST) {
                uint64_t *last = &crt->memory_accounting_last[metric];
 
                if (updated)
                        *last = bytes;
                else if (*last != UINT64_MAX)
                        bytes = *last;
                else
                        return -ENODATA;
 
        } else if (!updated)
                return -ENODATA;
 
        if (ret)
                *ret = bytes;
 
        return 0;
}
 
int unit_get_tasks_current(Unit *u, uint64_t *ret) {
        assert(u);
        assert(ret);
 
        if (!UNIT_CGROUP_BOOL(u, tasks_accounting))
                return -ENODATA;
 
        CGroupRuntime *crt = unit_get_cgroup_runtime(u);
        if (!crt || !crt->cgroup_path)
                return -ENODATA;
 
        /* The root cgroup doesn't expose this information, let's get it from /proc instead */
        if (unit_has_host_root_cgroup(u))
                return procfs_tasks_get_current(ret);
 
        if ((crt->cgroup_realized_mask & CGROUP_MASK_PIDS) == 0)
                return -ENODATA;
 
        return cg_get_attribute_as_uint64(crt->cgroup_path, "pids.current", ret);
}
 
static int unit_get_cpu_usage_raw(const Unit *u, const CGroupRuntime *crt, nsec_t *ret) {
        int r;
 
        assert(u);
        assert(crt);
        assert(ret);
 
        if (!crt->cgroup_path)
                return -ENODATA;
 
        /* The root cgroup doesn't expose this information, let's get it from /proc instead */
        if (unit_has_host_root_cgroup(u))
                return procfs_cpu_get_usage(ret);
 
        uint64_t us;
 
        r = cg_get_keyed_attribute_uint64(crt->cgroup_path, "cpu.stat", "usage_usec", &us);
        if (r < 0)
                return r;
 
        *ret = us * NSEC_PER_USEC;
 
        return 0;
}
 
int unit_get_cpu_usage(Unit *u, nsec_t *ret) {
        nsec_t ns;
        int r;
 
        assert(u);
 
        /* Retrieve the current CPU usage counter. This will subtract the CPU counter taken when the unit was
         * started. If the cgroup has been removed already, returns the last cached value. To cache the value, simply
         * call this function with a NULL return value. */
 
        CGroupRuntime *crt = unit_get_cgroup_runtime(u);
        if (!crt)
                return -ENODATA;
 
        r = unit_get_cpu_usage_raw(u, crt, &ns);
        if (r == -ENODATA && crt->cpu_usage_last != NSEC_INFINITY) {
                /* If we can't get the CPU usage anymore (because the cgroup was already removed, for example), use our
                 * cached value. */
 
                if (ret)
                        *ret = crt->cpu_usage_last;
                return 0;
        }
        if (r < 0)
                return r;
 
        if (ns > crt->cpu_usage_base)
                ns -= crt->cpu_usage_base;
        else
                ns = 0;
 
        crt->cpu_usage_last = ns;
        if (ret)
                *ret = ns;
 
        return 0;
}
 
int unit_get_ip_accounting(
                Unit *u,
                CGroupIPAccountingMetric metric,
                uint64_t *ret) {
 
        uint64_t value;
        int fd, r;
 
        assert(u);
        assert(metric >= 0);
        assert(metric < _CGROUP_IP_ACCOUNTING_METRIC_MAX);
        assert(ret);
 
        if (!UNIT_CGROUP_BOOL(u, ip_accounting))
                return -ENODATA;
 
        CGroupRuntime *crt = unit_get_cgroup_runtime(u);
        if (!crt)
                return -ENODATA;
 
        fd = IN_SET(metric, CGROUP_IP_INGRESS_BYTES, CGROUP_IP_INGRESS_PACKETS) ?
                crt->ip_accounting_ingress_map_fd :
                crt->ip_accounting_egress_map_fd;
        if (fd < 0)
                return -ENODATA;
 
        if (IN_SET(metric, CGROUP_IP_INGRESS_BYTES, CGROUP_IP_EGRESS_BYTES))
                r = bpf_firewall_read_accounting(fd, &value, NULL);
        else
                r = bpf_firewall_read_accounting(fd, NULL, &value);
        if (r < 0)
                return r;
 
        /* Add in additional metrics from a previous runtime. Note that when reexecing/reloading the daemon we compile
         * all BPF programs and maps anew, but serialize the old counters. When deserializing we store them in the
         * ip_accounting_extra[] field, and add them in here transparently. */
 
        *ret = value + crt->ip_accounting_extra[metric];
 
        return r;
}
 
static uint64_t unit_get_effective_limit_one(Unit *u, CGroupLimitType type) {
        CGroupContext *cc;
 
        assert(u);
        assert(UNIT_HAS_CGROUP_CONTEXT(u));
 
        if (unit_has_name(u, SPECIAL_ROOT_SLICE))
                switch (type) {
                        case CGROUP_LIMIT_MEMORY_MAX:
                        case CGROUP_LIMIT_MEMORY_HIGH:
                                return physical_memory();
                        case CGROUP_LIMIT_TASKS_MAX:
                                return system_tasks_max();
                        default:
                                assert_not_reached();
                }
 
        cc = ASSERT_PTR(unit_get_cgroup_context(u));
        switch (type) {
                case CGROUP_LIMIT_MEMORY_MAX:
                        return cc->memory_max;
                case CGROUP_LIMIT_MEMORY_HIGH:
                        return cc->memory_high;
                case CGROUP_LIMIT_TASKS_MAX:
                        return cgroup_tasks_max_resolve(&cc->tasks_max);
                default:
                        assert_not_reached();
        }
}
 
int unit_get_effective_limit(Unit *u, CGroupLimitType type, uint64_t *ret) {
        uint64_t infimum;
 
        assert(u);
        assert(ret);
        assert(type >= 0);
        assert(type < _CGROUP_LIMIT_TYPE_MAX);
 
        if (!UNIT_HAS_CGROUP_CONTEXT(u))
                return -EINVAL;
 
        infimum = unit_get_effective_limit_one(u, type);
        for (Unit *slice = UNIT_GET_SLICE(u); slice; slice = UNIT_GET_SLICE(slice))
                infimum = MIN(infimum, unit_get_effective_limit_one(slice, type));
 
        *ret = infimum;
        return 0;
}
 
static int unit_get_io_accounting_raw(
                const Unit *u,
                const CGroupRuntime *crt,
                uint64_t ret[static _CGROUP_IO_ACCOUNTING_METRIC_MAX]) {
 
        static const char* const field_names[_CGROUP_IO_ACCOUNTING_METRIC_MAX] = {
                [CGROUP_IO_READ_BYTES]       = "rbytes=",
                [CGROUP_IO_WRITE_BYTES]      = "wbytes=",
                [CGROUP_IO_READ_OPERATIONS]  = "rios=",
                [CGROUP_IO_WRITE_OPERATIONS] = "wios=",
        };
 
        uint64_t acc[_CGROUP_IO_ACCOUNTING_METRIC_MAX] = {};
        _cleanup_free_ char *path = NULL;
        _cleanup_fclose_ FILE *f = NULL;
        int r;
 
        assert(u);
        assert(crt);
 
        if (!crt->cgroup_path)
                return -ENODATA;
 
        if (unit_has_host_root_cgroup(u))
                return -ENODATA; /* TODO: return useful data for the top-level cgroup */
 
        if (!FLAGS_SET(crt->cgroup_realized_mask, CGROUP_MASK_IO))
                return -ENODATA;
 
        r = cg_get_path(crt->cgroup_path, "io.stat", &path);
        if (r < 0)
                return r;
 
        f = fopen(path, "re");
        if (!f)
                return -errno;
 
        for (;;) {
                _cleanup_free_ char *line = NULL;
                const char *p;
 
                r = read_line(f, LONG_LINE_MAX, &line);
                if (r < 0)
                        return r;
                if (r == 0)
                        break;
 
                p = line;
                p += strcspn(p, WHITESPACE); /* Skip over device major/minor */
                p += strspn(p, WHITESPACE);  /* Skip over following whitespace */
 
                for (;;) {
                        _cleanup_free_ char *word = NULL;
 
                        r = extract_first_word(&p, &word, NULL, EXTRACT_RETAIN_ESCAPE);
                        if (r < 0)
                                return r;
                        if (r == 0)
                                break;
 
                        for (CGroupIOAccountingMetric i = 0; i < _CGROUP_IO_ACCOUNTING_METRIC_MAX; i++) {
                                const char *x;
 
                                x = startswith(word, field_names[i]);
                                if (x) {
                                        uint64_t w;
 
                                        r = safe_atou64(x, &w);
                                        if (r < 0)
                                                return r;
 
                                        /* Sum up the stats of all devices */
                                        acc[i] += w;
                                        break;
                                }
                        }
                }
        }
 
        memcpy(ret, acc, sizeof(acc));
        return 0;
}
 
int unit_get_io_accounting(
                Unit *u,
                CGroupIOAccountingMetric metric,
                uint64_t *ret) {
 
        uint64_t raw[_CGROUP_IO_ACCOUNTING_METRIC_MAX];
        int r;
 
        /*
         * Retrieve an IO counter, subtracting the value of the counter value at the time the unit was started.
         * If ret == NULL and metric == _<...>_INVALID, no return value is expected (refresh the caches only).
         */
 
        assert(u);
        assert(metric >= 0 || (!ret && metric == _CGROUP_IO_ACCOUNTING_METRIC_INVALID));
        assert(metric < _CGROUP_IO_ACCOUNTING_METRIC_MAX);
 
        if (!UNIT_CGROUP_BOOL(u, io_accounting))
                return -ENODATA;
 
        CGroupRuntime *crt = unit_get_cgroup_runtime(u);
        if (!crt)
                return -ENODATA;
 
        r = unit_get_io_accounting_raw(u, crt, raw);
        if (r == -ENODATA && metric >= 0 && crt->io_accounting_last[metric] != UINT64_MAX)
                goto done;
        if (r < 0)
                return r;
 
        for (CGroupIOAccountingMetric i = 0; i < _CGROUP_IO_ACCOUNTING_METRIC_MAX; i++) {
                /* Saturated subtraction */
                if (raw[i] > crt->io_accounting_base[i])
                        crt->io_accounting_last[i] = raw[i] - crt->io_accounting_base[i];
                else
                        crt->io_accounting_last[i] = 0;
        }
 
done:
        if (ret)
                *ret = crt->io_accounting_last[metric];
 
        return 0;
}
 
static int unit_reset_cpu_accounting(Unit *unit, CGroupRuntime *crt) {
        int r;
 
        assert(crt);
 
        crt->cpu_usage_base = 0;
        crt->cpu_usage_last = NSEC_INFINITY;
 
        if (unit) {
                r = unit_get_cpu_usage_raw(unit, crt, &crt->cpu_usage_base);
                if (r < 0 && r != -ENODATA)
                        return r;
        }
 
        return 0;
}
 
static int unit_reset_io_accounting(Unit *unit, CGroupRuntime *crt) {
        int r;
 
        assert(crt);
 
        zero(crt->io_accounting_base);
        FOREACH_ELEMENT(i, crt->io_accounting_last)
                *i = UINT64_MAX;
 
        if (unit) {
                r = unit_get_io_accounting_raw(unit, crt, crt->io_accounting_base);
                if (r < 0 && r != -ENODATA)
                        return r;
        }
 
        return 0;
}
 
static void cgroup_runtime_reset_memory_accounting_last(CGroupRuntime *crt) {
        assert(crt);
 
        FOREACH_ELEMENT(i, crt->memory_accounting_last)
                *i = UINT64_MAX;
}
 
static int cgroup_runtime_reset_ip_accounting(CGroupRuntime *crt) {
        int r = 0;
 
        assert(crt);
 
        if (crt->ip_accounting_ingress_map_fd >= 0)
                RET_GATHER(r, bpf_firewall_reset_accounting(crt->ip_accounting_ingress_map_fd));
 
        if (crt->ip_accounting_egress_map_fd >= 0)
                RET_GATHER(r, bpf_firewall_reset_accounting(crt->ip_accounting_egress_map_fd));
 
        zero(crt->ip_accounting_extra);
 
        return r;
}
 
int unit_reset_accounting(Unit *u) {
        int r = 0;
 
        assert(u);
 
        CGroupRuntime *crt = unit_get_cgroup_runtime(u);
        if (!crt)
                return 0;
 
        cgroup_runtime_reset_memory_accounting_last(crt);
        RET_GATHER(r, unit_reset_cpu_accounting(u, crt));
        RET_GATHER(r, unit_reset_io_accounting(u, crt));
        RET_GATHER(r, cgroup_runtime_reset_ip_accounting(crt));
 
        return r;
}
 
bool unit_invalidate_cgroup(Unit *u, CGroupMask m) {
        assert(u);
 
        if (!UNIT_HAS_CGROUP_CONTEXT(u))
                return false;
 
        CGroupRuntime *crt = unit_get_cgroup_runtime(u);
        if (!crt)
                return false;
 
        /* If all controllers shall be invalidated, let's unconditionally submit the unit to realize queue.
         * We initialize the field to _CGROUP_MASK_ALL after all, and semantically it makes sense to use
         * it as a special signal to forcibly re-realize cgroup. */
        if (m != _CGROUP_MASK_ALL &&
            FLAGS_SET(crt->cgroup_invalidated_mask, m)) /* NOP? */
                return false;
 
        crt->cgroup_invalidated_mask |= m;
        unit_add_to_cgroup_realize_queue(u);