HALF ECHO LEARN / REFERENCE COLLECTION

v262.2

src/libsystemd/sd-event/sd-event.c

181,814 bytesGit mode 100644Download original fileLGPL-2.1-or-later

Git blob: 231ed955b540a6e71c82e47bc4d98bc301ac14ac

Lines 4001–5000 of 5,529

                                if (!inode_data) {
                                        event_inotify_data_drop(e, d, sz);
                                        continue;
                                }
 
                                /* The watch descriptor was removed by the kernel, let's drop it here too */
                                inode_data->wd = -1;
                        } else {
                                inode_data = hashmap_get(d->wd, INT_TO_PTR(d->buffer.ev.wd));
                                if (!inode_data) {
                                        event_inotify_data_drop(e, d, sz);
                                        continue;
                                }
                        }
 
                        /* Trigger all event sources that are interested in these events. Also trigger all event
                         * sources if IN_IGNORED or IN_UNMOUNT is set. */
                        LIST_FOREACH(inotify.by_inode_data, s, inode_data->event_sources) {
 
                                if (event_source_is_offline(s))
                                        continue;
 
                                if ((d->buffer.ev.mask & (IN_IGNORED|IN_UNMOUNT)) == 0 &&
                                    (s->inotify.mask & d->buffer.ev.mask & IN_ALL_EVENTS) == 0)
                                        continue;
 
                                r = source_set_pending(s, true);
                                if (r < 0)
                                        return r;
                        }
                }
 
                /* Something pending now? If so, let's finish. */
                if (d->n_pending > 0)
                        return 1;
 
                /* otherwise, drop the event and let's read more */
                event_inotify_data_drop(e, d, sz);
        }
 
        return 0;
}
 
static int process_inotify(sd_event *e) {
        int r, done = 0;
 
        assert(e);
 
        LIST_FOREACH(buffered, d, e->buffered_inotify_data_list) {
                r = event_inotify_data_process(e, d);
                if (r < 0)
                        return r;
                if (r > 0)
                        done++;
        }
 
        return done;
}
 
static int process_pressure(sd_event_source *s, uint32_t revents) {
        assert(s);
        assert(EVENT_SOURCE_IS_PRESSURE(s));
 
        if (s->pending)
                s->pressure.revents |= revents;
        else
                s->pressure.revents = revents;
 
        return source_set_pending(s, true);
}
 
static int source_pressure_write(sd_event_source *s) {
        ssize_t n;
        int r;
 
        assert(s);
        assert(EVENT_SOURCE_IS_PRESSURE(s));
 
        /* once we start writing, the buffer is locked, we allow no further changes. */
        s->pressure.locked = true;
 
        if (s->pressure.write_buffer_size > 0) {
                n = write(s->pressure.fd, s->pressure.write_buffer, s->pressure.write_buffer_size);
                if (n < 0) {
                        if (!ERRNO_IS_TRANSIENT(errno)) {
                                /* If kernel is built with CONFIG_PSI_DEFAULT_DISABLED it will expose PSI
                                 * files, but then generates EOPNOSUPP on read() and write() (instead of on
                                 * open()!). This sucks hard, since we can only detect this kind of failure
                                 * so late. Let's make the best of it, and turn off the event source like we
                                 * do for failed event source handlers. */
 
                                log_debug_errno(errno, "Writing pressure settings to kernel failed, disabling pressure event source: %m");
                                assert_se(sd_event_source_set_enabled(s, SD_EVENT_OFF) >= 0);
                                return 0;
                        }
 
                        n = 0;
                }
        } else
                n = 0;
 
        assert(n >= 0);
 
        if ((size_t) n == s->pressure.write_buffer_size) {
                s->pressure.write_buffer = mfree(s->pressure.write_buffer);
 
                if (n > 0) {
                        s->pressure.write_buffer_size = 0;
 
                        /* Update epoll events mask, since we have now written everything and don't care for EPOLLOUT anymore */
                        r = source_pressure_register(s, s->enabled);
                        if (r < 0)
                                return r;
                }
        } else if (n > 0) {
                _cleanup_free_ void *c = NULL;
 
                assert((size_t) n < s->pressure.write_buffer_size);
 
                c = memdup((uint8_t*) s->pressure.write_buffer + n, s->pressure.write_buffer_size - n);
                if (!c)
                        return -ENOMEM;
 
                free_and_replace(s->pressure.write_buffer, c);
                s->pressure.write_buffer_size -= n;
                return 1;
        }
 
        return 0;
}
 
static int source_pressure_initiate_dispatch(sd_event_source *s) {
        int r;
 
        assert(s);
        assert(EVENT_SOURCE_IS_PRESSURE(s));
 
        r = source_pressure_write(s);
        if (r < 0)
                return r;
        if (r > 0)
                return 1; /* if we wrote something, then don't continue with dispatching user dispatch
                           * function. Instead, shortcut it so that we wait for next EPOLLOUT immediately. */
 
        /* No pending incoming IO? Then let's not continue further */
        if ((s->pressure.revents & (EPOLLIN|EPOLLPRI)) == 0) {
 
                /* Treat IO errors on the notifier the same ways errors returned from a callback */
                if ((s->pressure.revents & (EPOLLHUP|EPOLLERR|EPOLLRDHUP)) != 0)
                        return -EIO;
 
                return 1; /* leave dispatch, we already processed everything */
        }
 
        if (s->pressure.revents & EPOLLIN) {
                uint8_t pipe_buf[PIPE_BUF];
                ssize_t n;
 
                /* If the fd is readable, then flush out anything that might be queued */
 
                n = read(s->pressure.fd, pipe_buf, sizeof(pipe_buf));
                if (n < 0 && !ERRNO_IS_TRANSIENT(errno))
                        return -errno;
        }
 
        return 0; /* go on, dispatch to user callback */
}
 
static int maybe_mark_post_sources_pending(EventSourceType t, sd_event *e) {
        sd_event_source *z;
        int r;
 
        if (t == SOURCE_POST)
                return 0;
 
        SET_FOREACH(z, e->post_sources) {
                if (event_source_is_offline(z))
                        continue;
 
                r = source_set_pending(z, true);
                if (r < 0)
                        return r;
        }
 
        return 0;
}
 
static int source_dispatch(sd_event_source *s) {
        EventSourceType saved_type;
        sd_event *saved_event;
        int r = 0;
 
        assert(s);
        assert(s->pending || s->type == SOURCE_EXIT);
 
        /* Save the event source type, here, so that we still know it after the event callback which might
         * invalidate the event. */
        saved_type = s->type;
 
        /* Similarly, store a reference to the event loop object, so that we can still access it after the
         * callback might have invalidated/disconnected the event source. */
        saved_event = s->event;
        PROTECT_EVENT(saved_event);
 
        /* Check if we hit the ratelimit for this event source, and if so, let's disable it. */
        assert(!s->ratelimited);
        if (!ratelimit_below(&s->rate_limit)) {
                r = event_source_enter_ratelimited(s);
                if (r < 0)
                        return r;
 
                return 1;
        }
 
        if (IN_SET(s->type, SOURCE_DEFER, SOURCE_EXIT)) {
                /* Make sure this event source is moved to the end of the priority list now. We do this here
                 * because defer and exit event sources are always pending from the moment they're added so
                 * the same logic in source_set_pending() is never triggered. */
                s->pending_iteration = s->event->iteration;
                event_source_pp_prioq_reshuffle(s);
        } else {
                r = source_set_pending(s, false);
                if (r < 0)
                        return r;
        }
 
        /* If we execute a non-post source, let's mark all post sources as pending. */
        r = maybe_mark_post_sources_pending(s->type, s->event);
        if (r < 0)
                return r;
 
        if (EVENT_SOURCE_IS_PRESSURE(s)) {
                r = source_pressure_initiate_dispatch(s);
                if (r == -EIO) /* handle EIO errors similar to callback errors */
                        goto finish;
                if (r < 0)
                        return r;
                if (r > 0) /* already handled */
                        return 1;
        }
 
        if (s->enabled == SD_EVENT_ONESHOT) {
                r = sd_event_source_set_enabled(s, SD_EVENT_OFF);
                if (r < 0)
                        return r;
        }
 
        s->dispatching = true;
 
        switch (s->type) {
 
        case SOURCE_IO:
                r = s->io.callback(s, s->io.fd, s->io.revents, s->userdata);
                break;
 
        case SOURCE_TIME_REALTIME:
        case SOURCE_TIME_BOOTTIME:
        case SOURCE_TIME_MONOTONIC:
        case SOURCE_TIME_REALTIME_ALARM:
        case SOURCE_TIME_BOOTTIME_ALARM:
                r = s->time.callback(s, s->time.next, s->userdata);
                break;
 
        case SOURCE_SIGNAL:
                r = s->signal.callback(s, &s->signal.siginfo, s->userdata);
                break;
 
        case SOURCE_CHILD: {
                bool zombie = SIGINFO_CODE_IS_DEAD(s->child.siginfo.si_code);
 
                r = s->child.callback(s, &s->child.siginfo, s->userdata);
 
                /* Now, reap the PID for good (unless WNOWAIT was specified by the caller). */
                if (zombie) {
                        (void) waitid(P_PIDFD, s->child.pidfd, &s->child.siginfo, WNOHANG|WEXITED|(s->child.options & WNOWAIT));
                        if (!FLAGS_SET(s->child.options, WNOWAIT))
                                s->child.waited = true;
                }
 
                break;
        }
 
        case SOURCE_DEFER:
                r = s->defer.callback(s, s->userdata);
                break;
 
        case SOURCE_POST:
                r = s->post.callback(s, s->userdata);
                break;
 
        case SOURCE_EXIT:
                r = s->exit.callback(s, s->userdata);
                break;
 
        case SOURCE_INOTIFY: {
                struct sd_event *e = s->event;
                InotifyData *d;
                size_t sz;
 
                assert(s->inotify.inode_data);
                assert_se(d = s->inotify.inode_data->inotify_data);
 
                assert(d->buffer_filled >= offsetof(struct inotify_event, name));
                sz = offsetof(struct inotify_event, name) + d->buffer.ev.len;
                assert(d->buffer_filled >= sz);
 
                /* If the inotify callback destroys the event source then this likely means we don't need to
                 * watch the inode anymore, and thus also won't need the inotify object anymore. But if we'd
                 * free it immediately, then we couldn't drop the event from the inotify event queue without
                 * memory corruption anymore, as below. Hence, let's not free it immediately, but mark it
                 * "busy" with a counter (which will ensure it's not GC'ed away prematurely). Let's then
                 * explicitly GC it after we are done dropping the inotify event from the buffer. */
                d->n_busy++;
                r = s->inotify.callback(s, &d->buffer.ev, s->userdata);
                d->n_busy--;
 
                /* When no event is pending anymore on this inotify object, then let's drop the event from
                 * the inotify event queue buffer. */
                if (d->n_pending == 0)
                        event_inotify_data_drop(e, d, sz);
 
                /* Now we don't want to access 'd' anymore, it's OK to GC now. */
                event_gc_inotify_data(e, d);
                break;
        }
 
        case SOURCE_MEMORY_PRESSURE:
        case SOURCE_CPU_PRESSURE:
        case SOURCE_IO_PRESSURE:
                r = s->pressure.callback(s, s->userdata);
                break;
 
        case SOURCE_WATCHDOG:
        case _SOURCE_EVENT_SOURCE_TYPE_MAX:
        case _SOURCE_EVENT_SOURCE_TYPE_INVALID:
                assert_not_reached();
        }
 
        s->dispatching = false;
 
finish:
        if (r < 0) {
                log_debug_errno(r, "Event source %s (type %s) returned error, %s: %m",
                                strna(s->description),
                                event_source_type_to_string(saved_type),
                                s->exit_on_failure ? "exiting" : "disabling");
 
                if (s->exit_on_failure)
                        (void) sd_event_exit(saved_event, r);
        }
 
        if (s->n_ref == 0)
                source_free(s);
        else if (r < 0)
                assert_se(sd_event_source_set_enabled(s, SD_EVENT_OFF) >= 0);
 
        /* More post sources might have been added while executing the callback, let's make sure
         * those are marked pending as well. */
        r = maybe_mark_post_sources_pending(saved_type, saved_event);
        if (r < 0)
                return r;
 
        return 1;
}
 
static int event_prepare(sd_event *e) {
        int r;
 
        assert(e);
 
        for (;;) {
                sd_event_source *s;
 
                s = prioq_peek(e->prepare);
                if (!s || s->prepare_iteration == e->iteration || event_source_is_offline(s))
                        break;
 
                s->prepare_iteration = e->iteration;
                prioq_reshuffle(e->prepare, s, &s->prepare_index);
 
                assert(s->prepare);
                s->dispatching = true;
                r = s->prepare(s, s->userdata);
                s->dispatching = false;
 
                if (r < 0) {
                        log_debug_errno(r, "Prepare callback of event source %s (type %s) returned error, %s: %m",
                                        strna(s->description),
                                        event_source_type_to_string(s->type),
                                        s->exit_on_failure ? "exiting" : "disabling");
 
                        if (s->exit_on_failure)
                                (void) sd_event_exit(e, r);
                }
 
                if (s->n_ref == 0)
                        source_free(s);
                else if (r < 0)
                        assert_se(sd_event_source_set_enabled(s, SD_EVENT_OFF) >= 0);
        }
 
        return 0;
}
 
static int dispatch_exit(sd_event *e) {
        sd_event_source *p;
        int r;
 
        assert(e);
 
        p = prioq_peek(e->exit);
        assert(!p || p->type == SOURCE_EXIT);
 
        if (!p || event_source_is_offline(p)) {
                e->state = SD_EVENT_FINISHED;
                return 0;
        }
 
        PROTECT_EVENT(e);
        e->iteration++;
        e->state = SD_EVENT_EXITING;
        r = source_dispatch(p);
        e->state = SD_EVENT_INITIAL;
        return r;
}
 
static sd_event_source* event_next_pending(sd_event *e) {
        sd_event_source *p;
 
        assert(e);
 
        p = prioq_peek(e->pending);
        if (!p)
                return NULL;
 
        if (event_source_is_offline(p))
                return NULL;
 
        return p;
}
 
static int arm_watchdog(sd_event *e) {
        struct itimerspec its = {};
        usec_t t;
 
        assert(e);
        assert(e->watchdog_fd >= 0);
 
        t = sleep_between(e,
                          usec_add(e->watchdog_last, (e->watchdog_period / 2)),
                          usec_add(e->watchdog_last, (e->watchdog_period * 3 / 4)));
 
        timespec_store(&its.it_value, t);
 
        /* Make sure we never set the watchdog to 0, which tells the
         * kernel to disable it. */
        if (its.it_value.tv_sec == 0 && its.it_value.tv_nsec == 0)
                its.it_value.tv_nsec = 1;
 
        return RET_NERRNO(timerfd_settime(e->watchdog_fd, TFD_TIMER_ABSTIME, &its, NULL));
}
 
static int process_watchdog(sd_event *e) {
        assert(e);
 
        if (!e->watchdog)
                return 0;
 
        /* Don't notify watchdog too often */
        if (e->watchdog_last + e->watchdog_period / 4 > e->timestamp.monotonic)
                return 0;
 
        sd_notify(false, "WATCHDOG=1");
        e->watchdog_last = e->timestamp.monotonic;
 
        return arm_watchdog(e);
}
 
static void event_close_inode_data_fds(sd_event *e) {
        InodeData *d;
 
        assert(e);
 
        /* Close the fds pointing to the inodes to watch now. We need to close them as they might otherwise pin
         * filesystems. But we can't close them right-away as we need them as long as the user still wants to make
         * adjustments to the event source, such as changing the priority (which requires us to remove and re-add a watch
         * for the inode). Hence, let's close them when entering the first iteration after they were added, as a
         * compromise. */
 
        while ((d = e->inode_data_to_close_list)) {
                assert(d->fd >= 0);
                d->fd = safe_close(d->fd);
 
                LIST_REMOVE(to_close, e->inode_data_to_close_list, d);
        }
}
 
static int event_pressure_write_list(sd_event *e) {
        int r;
 
        assert(e);
 
        for (;;) {
                sd_event_source *s;
 
                s = LIST_POP(pressure.write_list, e->pressure_write_list);
                if (!s)
                        break;
 
                assert(EVENT_SOURCE_IS_PRESSURE(s));
                assert(s->pressure.write_buffer_size > 0);
                s->pressure.in_write_list = false;
 
                r = source_pressure_write(s);
                if (r < 0)
                        return r;
        }
 
        return 0;
}
 
static bool event_loop_idle(sd_event *e) {
        assert(e);
 
        LIST_FOREACH(sources, s, e->sources) {
                /* Exit sources only trigger on exit, so whether they're enabled or not doesn't matter when
                 * we're deciding if the event loop is idle or not. */
                if (s->type == SOURCE_EXIT)
                        continue;
 
                if (s->enabled == SD_EVENT_OFF)
                        continue;
 
                /* Post event sources always need another active event source to become pending. */
                if (s->type == SOURCE_POST && !s->pending)
                        continue;
 
                return false;
        }
 
        return true;
}
 
_public_ int sd_event_prepare(sd_event *e) {
        int r;
 
        assert_return(e, -EINVAL);
        assert_return(e = event_resolve(e), -ENOPKG);
        assert_return(!event_origin_changed(e), -ECHILD);
        assert_return(e->state != SD_EVENT_FINISHED, -ESTALE);
        assert_return(e->state == SD_EVENT_INITIAL, -EBUSY);
 
        /* Let's check that if we are a default event loop we are executed in the correct thread. We only do
         * this check here once, since gettid() is typically not cached, and thus want to minimize
         * syscalls */
        assert_return(!e->default_event_ptr || e->tid == gettid(), -EREMOTEIO);
 
        /* Make sure that none of the preparation callbacks ends up freeing the event source under our feet */
        PROTECT_EVENT(e);
 
        e->ratelimit_expired = false;
 
        if (!e->exit_requested && e->exit_on_idle && event_loop_idle(e))
                (void) sd_event_exit(e, 0);
 
        if (e->exit_requested)
                goto pending;
 
        e->iteration++;
 
        e->state = SD_EVENT_PREPARING;
        r = event_prepare(e);
        e->state = SD_EVENT_INITIAL;
        if (r < 0)
                return r;
 
        r = event_pressure_write_list(e);
        if (r < 0)
                return r;
 
        r = event_arm_timer(e, &e->realtime);
        if (r < 0)
                return r;
 
        r = event_arm_timer(e, &e->boottime);
        if (r < 0)
                return r;
 
        r = event_arm_timer(e, &e->monotonic);
        if (r < 0)
                return r;
 
        r = event_arm_timer(e, &e->realtime_alarm);
        if (r < 0)
                return r;
 
        r = event_arm_timer(e, &e->boottime_alarm);
        if (r < 0)
                return r;
 
        event_close_inode_data_fds(e);
 
        if (event_next_pending(e) || e->need_process_child || e->buffered_inotify_data_list)
                goto pending;
 
        e->state = SD_EVENT_ARMED;
 
        return 0;
 
pending:
        e->state = SD_EVENT_ARMED;
        r = sd_event_wait(e, 0);
        if (r == 0)
                e->state = SD_EVENT_ARMED;
 
        return r;
}
 
static int epoll_wait_usec(
                int fd,
                struct epoll_event *events,
                int maxevents,
                usec_t timeout) {
 
        static bool epoll_pwait2_absent = false;
        int r, msec;
 
        /* A wrapper that uses missing_epoll_pwait2(), which directly calls epoll_pwait2 syscall. */
 
        if (!epoll_pwait2_absent && timeout != USEC_INFINITY) {
                r = missing_epoll_pwait2(
                                fd,
                                events,
                                maxevents,
                                TIMESPEC_STORE(timeout),
                                /* sigmask= */ NULL);
                if (r >= 0)
                        return r;
                if (!ERRNO_IS_NOT_SUPPORTED(errno) && !ERRNO_IS_PRIVILEGE(errno))
                        return -errno; /* Only fallback to old epoll_wait() if the syscall is masked or not
                                        * supported. */
 
                epoll_pwait2_absent = true;
        }
 
        if (timeout == USEC_INFINITY)
                msec = -1;
        else {
                usec_t k;
 
                k = DIV_ROUND_UP(timeout, USEC_PER_MSEC);
                if (k >= INT_MAX)
                        msec = INT_MAX; /* Saturate */
                else
                        msec = (int) k;
        }
 
        return RET_NERRNO(epoll_wait(fd, events, maxevents, msec));
}
 
static int process_epoll(sd_event *e, usec_t timeout, int64_t threshold, int64_t *ret_min_priority) {
        size_t n_event_queue, m, n_event_max;
        int64_t min_priority = threshold;
        bool something_new = false;
        int r;
 
        assert(e);
        assert(ret_min_priority);
 
        n_event_queue = MAX(e->n_sources, 1u);
        if (!GREEDY_REALLOC(e->event_queue, n_event_queue))
                return -ENOMEM;
 
        n_event_max = MALLOC_ELEMENTSOF(e->event_queue);
 
        /* If we still have inotify data buffered, then query the other fds, but don't wait on it */
        if (e->buffered_inotify_data_list)
                timeout = 0;
 
        for (;;) {
                r = epoll_wait_usec(
                                e->epoll_fd,
                                e->event_queue,
                                n_event_max,
                                timeout);
                if (r < 0)
                        return r;
 
                m = (size_t) r;
                msan_unpoison(e->event_queue, m * sizeof(struct epoll_event));
 
                if (m < n_event_max)
                        break;
 
                if (n_event_max >= n_event_queue * 10)
                        break;
 
                if (!GREEDY_REALLOC(e->event_queue, n_event_max + n_event_queue))
                        return -ENOMEM;
 
                n_event_max = MALLOC_ELEMENTSOF(e->event_queue);
                timeout = 0;
        }
 
        /* Set timestamp only when this is called first time. */
        if (threshold == INT64_MAX)
                triple_timestamp_now(&e->timestamp);
 
        FOREACH_ARRAY(i, e->event_queue, m) {
 
                if (i->data.ptr == INT_TO_PTR(SOURCE_WATCHDOG))
                        r = flush_timer(e, e->watchdog_fd, i->events, NULL);
                else {
                        WakeupType *t = ASSERT_PTR(i->data.ptr);
 
                        switch (*t) {
 
                        case WAKEUP_EVENT_SOURCE: {
                                sd_event_source *s = i->data.ptr;
 
                                if (s->priority > threshold)
                                        continue;
 
                                min_priority = MIN(min_priority, s->priority);
 
                                switch (s->type) {
 
                                case SOURCE_IO:
                                        r = process_io(e, s, i->events);
                                        break;
 
                                case SOURCE_CHILD:
                                        r = process_pidfd(e, s, i->events);
                                        break;
 
                                case SOURCE_MEMORY_PRESSURE:
                                case SOURCE_CPU_PRESSURE:
                                case SOURCE_IO_PRESSURE:
                                        r = process_pressure(s, i->events);
                                        break;
 
                                default:
                                        assert_not_reached();
                                }
 
                                break;
                        }
 
                        case WAKEUP_CLOCK_DATA: {
                                struct clock_data *d = i->data.ptr;
 
                                r = flush_timer(e, d->fd, i->events, &d->next);
                                break;
                        }
 
                        case WAKEUP_SIGNAL_DATA:
                                r = process_signal(e, i->data.ptr, i->events, &min_priority);
                                break;
 
                        case WAKEUP_INOTIFY_DATA:
                                r = event_inotify_data_read(e, i->data.ptr, i->events, threshold);
                                break;
 
                        default:
                                assert_not_reached();
                        }
                }
                if (r < 0)
                        return r;
                if (r > 0)
                        something_new = true;
        }
 
        *ret_min_priority = min_priority;
        return something_new;
}
 
_public_ int sd_event_wait(sd_event *e, uint64_t timeout) {
        int r;
 
        assert_return(e, -EINVAL);
        assert_return(e = event_resolve(e), -ENOPKG);
        assert_return(!event_origin_changed(e), -ECHILD);
        assert_return(e->state != SD_EVENT_FINISHED, -ESTALE);
        assert_return(e->state == SD_EVENT_ARMED, -EBUSY);
 
        if (e->exit_requested) {
                e->state = SD_EVENT_PENDING;
                return 1;
        }
 
        for (int64_t threshold = INT64_MAX; ; threshold--) {
                int64_t epoll_min_priority, child_min_priority;
 
                /* There may be a possibility that new epoll (especially IO) and child events are
                 * triggered just after process_epoll() call but before process_child(), and the new IO
                 * events may have higher priority than the child events. To salvage these events,
                 * let's call epoll_wait() again, but accepts only events with higher priority than the
                 * previous. See issue https://github.com/systemd/systemd/issues/18190 and comments
                 * https://github.com/systemd/systemd/pull/18750#issuecomment-785801085
                 * https://github.com/systemd/systemd/pull/18922#issuecomment-792825226 */
 
                r = process_epoll(e, timeout, threshold, &epoll_min_priority);
                if (r == -EINTR) {
                        e->state = SD_EVENT_PENDING;
                        return 1;
                }
                if (r < 0)
                        goto finish;
                if (r == 0 && threshold < INT64_MAX)
                        /* No new epoll event. */
                        break;
 
                r = process_child(e, threshold, &child_min_priority);
                if (r < 0)
                        goto finish;
                if (r == 0)
                        /* No new child event. */
                        break;
 
                threshold = MIN(epoll_min_priority, child_min_priority);
                if (threshold == INT64_MIN)
                        break;
 
                timeout = 0;
        }
 
        r = process_watchdog(e);
        if (r < 0)
                goto finish;
 
        r = process_inotify(e);
        if (r < 0)
                goto finish;
 
        r = process_timer(e, e->timestamp.realtime, &e->realtime);
        if (r < 0)
                goto finish;
 
        r = process_timer(e, e->timestamp.boottime, &e->boottime);
        if (r < 0)
                goto finish;
        else if (r == 1)
                e->ratelimit_expired = true;
 
        r = process_timer(e, e->timestamp.realtime, &e->realtime_alarm);
        if (r < 0)
                goto finish;
 
        r = process_timer(e, e->timestamp.boottime, &e->boottime_alarm);
        if (r < 0)
                goto finish;
 
        r = process_timer(e, e->timestamp.monotonic, &e->monotonic);
        if (r < 0)
                goto finish;
 
        if (e->ratelimit_expired) {
                /* Ratelimit expiry callback was called. Let's postpone processing pending sources in order
                 * to evaluate (in the next iteration) also sources that were potentially re-enabled by the
                 * callback. Report this as pending rather than returning 0, as the caller would block
                 * otherwise. sd_event_dispatch() then returns right away.
                 *
                 * Wondering why we treat only the CLOCK_BOOTTIME invocation of process_timer() differently?
                 * Once event source is ratelimited we essentially transform it into a CLOCK_BOOTTIME timer
                 * hence ratelimit expiry callback is never called for any other timer type.
                 *
                 * Note that we don't short-circuit earlier because all timer prioqs must be fully processed
                 * as their timerfds may have already been flushed by process_epoll(), so skipping them would
                 * leave those sources unprocessed and the timerfds un-rearmed. */
                e->state = SD_EVENT_PENDING;
                return 1;
        }
 
        if (event_next_pending(e)) {
                e->state = SD_EVENT_PENDING;
                return 1;
        }
 
        r = 0;
 
finish:
        e->state = SD_EVENT_INITIAL;
 
        return r;
}
 
_public_ int sd_event_dispatch(sd_event *e) {
        sd_event_source *p;
        int r;
 
        assert_return(e, -EINVAL);
        assert_return(e = event_resolve(e), -ENOPKG);
        assert_return(!event_origin_changed(e), -ECHILD);
        assert_return(e->state != SD_EVENT_FINISHED, -ESTALE);
        assert_return(e->state == SD_EVENT_PENDING, -EBUSY);
 
        if (e->ratelimit_expired) {
                /* A ratelimit expiry callback was called in this iteration, don't dispatch anything else. */
                e->state = SD_EVENT_INITIAL;
                return 1;
        }
 
        if (e->exit_requested)
                return dispatch_exit(e);
 
        p = event_next_pending(e);
        if (p) {
                PROTECT_EVENT(e);
 
                e->state = SD_EVENT_RUNNING;
                r = source_dispatch(p);
                e->state = SD_EVENT_INITIAL;
                return r;
        }
 
        e->state = SD_EVENT_INITIAL;
 
        return 1;
}
 
static void event_log_delays(sd_event *e) {
        char b[ELEMENTSOF(e->delays) * DECIMAL_STR_MAX(unsigned) + 1], *p;
        size_t l;
 
        p = b;
        l = sizeof(b);
        FOREACH_ELEMENT(delay, e->delays) {
                l = strpcpyf(&p, l, "%u ", *delay);
                *delay = 0;
        }
        log_debug("Event loop iterations: %s", b);
}
 
_public_ int sd_event_run(sd_event *e, uint64_t timeout) {
        int r;
 
        assert_return(e, -EINVAL);
        assert_return(e = event_resolve(e), -ENOPKG);
        assert_return(!event_origin_changed(e), -ECHILD);
        assert_return(e->state != SD_EVENT_FINISHED, -ESTALE);
        assert_return(e->state == SD_EVENT_INITIAL, -EBUSY);
 
        /* When running on a fiber, delegate to the suspending implementation. Note that the
         * profile_delays accounting below is intentionally skipped on that path: the suspending variant
         * drives the event loop via sd_event_prepare()/sd_event_wait()/sd_event_dispatch() itself, which
         * are the same primitives profile_delays tracks when called directly. */
        if (sd_fiber_is_running())
                return event_run_suspend(e, timeout);
 
        if (e->profile_delays && e->last_run_usec != 0) {
                usec_t this_run;
                unsigned l;
 
                this_run = now(CLOCK_MONOTONIC);
 
                l = log2u64(this_run - e->last_run_usec);
                assert(l < ELEMENTSOF(e->delays));
                e->delays[l]++;
 
                if (this_run - e->last_log_usec >= 5*USEC_PER_SEC) {
                        event_log_delays(e);
                        e->last_log_usec = this_run;
                }
        }
 
        /* Make sure that none of the preparation callbacks ends up freeing the event source under our feet */
        PROTECT_EVENT(e);
 
        r = sd_event_prepare(e);
        if (r == 0)
                /* There was nothing? Then wait... */
                r = sd_event_wait(e, timeout);
 
        if (e->profile_delays)
                e->last_run_usec = now(CLOCK_MONOTONIC);
 
        if (r > 0) {
                /* There's something now, then let's dispatch it */
                r = sd_event_dispatch(e);
                if (r < 0)
                        return r;
 
                return 1;
        }
 
        return r;
}
 
_public_ int sd_event_loop(sd_event *e) {
        int r;
 
        assert_return(e, -EINVAL);
        assert_return(e = event_resolve(e), -ENOPKG);
        assert_return(!event_origin_changed(e), -ECHILD);
        assert_return(e->state == SD_EVENT_INITIAL, -EBUSY);
 
        PROTECT_EVENT(e);
 
        while (e->state != SD_EVENT_FINISHED) {