[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-changelog] [xen-unstable] of SIGCHLD. The application can tell us whether it wants to own
# HG changeset patch # User Ian Jackson <Ian.Jackson@xxxxxxxxxxxxx> # Date 1336759136 -3600 # Node ID e3371fc765b1b9bb63af122ea3d42a6901702e18 # Parent 4e7a03e27a12e76e6a65f1e45ec991c600119ba1 of SIGCHLD. The application can tell us whether it wants to own SIGCHLD or not; if it does, it has to tell us about deaths of our children. Currently there are no callers in libxl which use these facilities. All code in libxl which forks needs to be converted and libxl_fork needse to be be abolished. Signed-off-by: Ian Jackson <ian.jackson@xxxxxxxxxxxxx> Acked-by: Ian Campbell <ian.campbell@xxxxxxxxxx> Committed-by: Ian Jackson <Ian.Jackson@xxxxxxxxxxxxx> --- diff -r 4e7a03e27a12 -r e3371fc765b1 tools/libxl/libxl.c --- a/tools/libxl/libxl.c Fri May 11 18:58:55 2012 +0100 +++ b/tools/libxl/libxl.c Fri May 11 18:58:56 2012 +0100 @@ -39,7 +39,7 @@ int libxl_ctx_alloc(libxl_ctx **pctx, in memset(ctx, 0, sizeof(libxl_ctx)); ctx->lg = lg; - /* First initialise pointers (cannot fail) */ + /* First initialise pointers etc. (cannot fail) */ LIBXL_TAILQ_INIT(&ctx->occurred); @@ -58,6 +58,11 @@ int libxl_ctx_alloc(libxl_ctx **pctx, in LIBXL_TAILQ_INIT(&ctx->death_list); libxl__ev_xswatch_init(&ctx->death_watch); + ctx->childproc_hooks = &libxl__childproc_default_hooks; + ctx->childproc_user = 0; + + ctx->sigchld_selfpipe[0] = -1; + /* The mutex is special because we can't idempotently destroy it */ if (libxl__init_recursive_mutex(ctx, &ctx->lock) < 0) { @@ -160,6 +165,16 @@ int libxl_ctx_free(libxl_ctx *ctx) discard_events(&ctx->occurred); + /* If we have outstanding children, then the application inherits + * them; we wish the application good luck with understanding + * this if and when it reaps them. */ + libxl__sigchld_removehandler(ctx); + + if (ctx->sigchld_selfpipe[0] >= 0) { + close(ctx->sigchld_selfpipe[0]); + close(ctx->sigchld_selfpipe[1]); + } + pthread_mutex_destroy(&ctx->lock); GC_FREE; diff -r 4e7a03e27a12 -r e3371fc765b1 tools/libxl/libxl.h --- a/tools/libxl/libxl.h Fri May 11 18:58:55 2012 +0100 +++ b/tools/libxl/libxl.h Fri May 11 18:58:56 2012 +0100 @@ -380,6 +380,7 @@ enum { ERROR_NOT_READY = -11, ERROR_OSEVENT_REG_FAIL = -12, ERROR_BUFFERFULL = -13, + ERROR_UNKNOWN_CHILD = -14, }; diff -r 4e7a03e27a12 -r e3371fc765b1 tools/libxl/libxl_event.c --- a/tools/libxl/libxl_event.c Fri May 11 18:58:55 2012 +0100 +++ b/tools/libxl/libxl_event.c Fri May 11 18:58:56 2012 +0100 @@ -627,6 +627,10 @@ static int beforepoll_internal(libxl__gc \ REQUIRE_FD(poller->wakeup_pipe[0], POLLIN, BODY); \ \ + int selfpipe = libxl__fork_selfpipe_active(CTX); \ + if (selfpipe >= 0) \ + REQUIRE_FD(selfpipe, POLLIN, BODY); \ + \ }while(0) #define REQUIRE_FD(req_fd_, req_events_, BODY) do{ \ @@ -766,10 +770,11 @@ static void afterpoll_internal(libxl__eg int nfds, const struct pollfd *fds, struct timeval now) { + /* May make callbacks into the application for child processes. + * ctx must be locked exactly once */ EGC_GC; libxl__ev_fd *efd; - LIBXL_LIST_FOREACH(efd, &CTX->efds, entry) { if (!efd->events) continue; @@ -780,11 +785,16 @@ static void afterpoll_internal(libxl__eg } if (afterpoll_check_fd(poller,fds,nfds, poller->wakeup_pipe[0],POLLIN)) { - char buf[256]; - int r = read(poller->wakeup_pipe[0], buf, sizeof(buf)); - if (r < 0) - if (errno != EINTR && errno != EWOULDBLOCK) - LIBXL__EVENT_DISASTER(egc, "read wakeup", errno, 0); + int e = libxl__self_pipe_eatall(poller->wakeup_pipe[0]); + if (e) LIBXL__EVENT_DISASTER(egc, "read wakeup", e, 0); + } + + int selfpipe = libxl__fork_selfpipe_active(CTX); + if (selfpipe >= 0 && + afterpoll_check_fd(poller,fds,nfds, selfpipe, POLLIN)) { + int e = libxl__self_pipe_eatall(selfpipe); + if (e) LIBXL__EVENT_DISASTER(egc, "read sigchld pipe", e, 0); + libxl__fork_selfpipe_woken(egc); } for (;;) { @@ -1082,16 +1092,37 @@ void libxl__poller_put(libxl_ctx *ctx, l void libxl__poller_wakeup(libxl__egc *egc, libxl__poller *p) { + int e = libxl__self_pipe_wakeup(p->wakeup_pipe[1]); + if (e) LIBXL__EVENT_DISASTER(egc, "cannot poke watch pipe", e, 0); +} + +int libxl__self_pipe_wakeup(int fd) +{ static const char buf[1] = ""; for (;;) { - int r = write(p->wakeup_pipe[1], buf, 1); - if (r==1) return; + int r = write(fd, buf, 1); + if (r==1) return 0; assert(r==-1); if (errno == EINTR) continue; - if (errno == EWOULDBLOCK) return; - LIBXL__EVENT_DISASTER(egc, "cannot poke watch pipe", errno, 0); - return; + if (errno == EWOULDBLOCK) return 0; + assert(errno); + return errno; + } +} + +int libxl__self_pipe_eatall(int fd) +{ + char buf[256]; + for (;;) { + int r = read(fd, buf, sizeof(buf)); + if (r == sizeof(buf)) continue; + if (r >= 0) return 0; + assert(r == -1); + if (errno == EINTR) continue; + if (errno == EWOULDBLOCK) return 0; + assert(errno); + return errno; } } diff -r 4e7a03e27a12 -r e3371fc765b1 tools/libxl/libxl_event.h --- a/tools/libxl/libxl_event.h Fri May 11 18:58:55 2012 +0100 +++ b/tools/libxl/libxl_event.h Fri May 11 18:58:56 2012 +0100 @@ -163,11 +163,6 @@ void libxl_event_register_callbacks(libx * After libxl_ctx_free, all corresponding evgen handles become * invalid and must no longer be passed to evdisable. * - * Events enabled with evenable prior to a fork and libxl_ctx_postfork - * are no longer generated after the fork/postfork; however the evgen - * structures are still valid and must be passed to evdisable if the - * memory they use should not be leaked. - * * Applications should ensure that they eventually retrieve every * event using libxl_event_check or libxl_event_wait, since events * which occur but are not retreived by the application will be queued @@ -372,6 +367,148 @@ void libxl_osevent_occurred_fd(libxl_ctx void libxl_osevent_occurred_timeout(libxl_ctx *ctx, void *for_libxl); +/*======================================================================*/ + +/* + * Subprocess handling. + * + * Unfortunately the POSIX interface makes this very awkward. + * + * There are two possible arrangements for collecting statuses from + * wait/waitpid. + * + * For naive programs: + * + * libxl will keep a SIGCHLD handler installed whenever it has an + * active (unreaped) child. It will reap all children with + * wait(); any children it does not recognise will be passed to + * the application via an optional callback (and will result in + * logged warnings if no callback is provided or the callback + * denies responsibility for the child). + * + * libxl may have children whenever: + * + * - libxl is performing an operation which can be made + * asynchronous; ie one taking a libxl_asyncop_how, even + * if NULL is passed indicating that the operation is + * synchronous; or + * + * - events of any kind are being generated, as requested + * by libxl_evenable_.... + * + * A multithreaded application which is naive in this sense may + * block SIGCHLD on some of its threads, but there must be at + * least one thread that has SIGCHLD unblocked. libxl will not + * modify the blocking flag for SIGCHLD (except that it may create + * internal service threads with all signals blocked). + * + * A naive program must only have at any one time only + * one libxl context which might have children. + * + * For programs which run their own children alongside libxl's: + * + * A program which does this must call libxl_childproc_setmode. + * There are two options: + * + * libxl_sigchld_owner_mainloop: + * The application must install a SIGCHLD handler and reap (at + * least) all of libxl's children and pass their exit status + * to libxl by calling libxl_childproc_exited. + * + * libxl_sigchld_owner_libxl_always: + * The application expects libxl to reap all of its children, + * and provides a callback to be notified of their exit + * statues. + * + * An application which fails to call setmode, or which passes 0 for + * hooks, while it uses any libxl operation which might + * create or use child processes (see above): + * - Must not have any child processes running. + * - Must not install a SIGCHLD handler. + * - Must not reap any children. + */ + + +typedef enum { + /* libxl owns SIGCHLD whenever it has a child. */ + libxl_sigchld_owner_libxl, + + /* Application promises to call libxl_childproc_exited but NOT + * from within a signal handler. libxl will not itself arrange to + * (un)block or catch SIGCHLD. */ + libxl_sigchld_owner_mainloop, + + /* libxl owns SIGCHLD all the time, and the application is + * relying on libxl's event loop for reaping its own children. */ + libxl_sigchld_owner_libxl_always, +} libxl_sigchld_owner; + +typedef struct { + libxl_sigchld_owner chldowner; + + /* All of these are optional: */ + + /* Called by libxl instead of fork. Should behave exactly like + * fork, including setting errno etc. May NOT reenter into libxl. + * Application may use this to discover pids of libxl's children, + * for example. + */ + pid_t (*fork_replacement)(void *user); + + /* With libxl_sigchld_owner_libxl, called by libxl when it has + * reaped a pid. (Not permitted with _owner_mainloop.) + * + * Should return 0 if the child was recognised by the application + * (or if the application does not keep those kind of records), + * ERROR_UNKNOWN_CHILD if the application knows that the child is not + * the application's; if it returns another error code it is a + * disaster as described for libxl_event_register_callbacks. + * (libxl will report unexpected children to its error log.) + * + * If not supplied, the application is assumed not to start + * any children of its own. + * + * This function is NOT called from within the signal handler. + * Rather it will be called from inside a libxl's event handling + * code and thus only when libxl is running, for example from + * within libxl_event_wait. (libxl uses the self-pipe trick + * to implement this.) + * + * childproc_exited_callback may call back into libxl, but it + * is best to avoid making long-running libxl calls as that might + * stall the calling event loop while the nested operation + * completes. + */ + int (*reaped_callback)(pid_t, int status, void *user); +} libxl_childproc_hooks; + +/* hooks may be 0 in which is equivalent to &{ libxl_sigchld_owner_libxl, 0, 0 } + * + * May not be called when libxl might have any child processes, or the + * behaviour is undefined. So it is best to call this at + * initialisation. + */ +void libxl_childproc_setmode(libxl_ctx *ctx, const libxl_childproc_hooks *hooks, + void *user); + +/* + * This function is for an application which owns SIGCHLD and which + * therefore reaps all of the process's children. + * + * May be called only by an application which has called setmode with + * chldowner == libxl_sigchld_owner_mainloop. If pid was a process started + * by this instance of libxl, returns 0 after doing whatever + * processing is appropriate. Otherwise silently returns + * ERROR_UNKNOWN_CHILD. No other error returns are possible. + * + * May NOT be called from within a signal handler which might + * interrupt any libxl operation. The application will almost + * certainly need to use the self-pipe trick (or a working pselect or + * ppoll) to implement this. + */ +int libxl_childproc_reaped(libxl_ctx *ctx, pid_t, int status); + + /* * An application which initialises a libxl_ctx in a parent process * and then forks a child which does not quickly exec, must diff -r 4e7a03e27a12 -r e3371fc765b1 tools/libxl/libxl_fork.c --- a/tools/libxl/libxl_fork.c Fri May 11 18:58:55 2012 +0100 +++ b/tools/libxl/libxl_fork.c Fri May 11 18:58:56 2012 +0100 @@ -46,6 +46,12 @@ static int atfork_registered; static LIBXL_LIST_HEAD(, libxl__carefd) carefds = LIBXL_LIST_HEAD_INITIALIZER(carefds); +/* non-null iff installed, protected by no_forking */ +static libxl_ctx *sigchld_owner; +static struct sigaction sigchld_saved_action; + +static void sigchld_removehandler_core(void); + static void atfork_lock(void) { int r = pthread_mutex_lock(&no_forking); @@ -107,6 +113,7 @@ void libxl_postfork_child_noexec(libxl_c int r; atfork_lock(); + LIBXL_LIST_FOREACH_SAFE(cf, &carefds, entry, cf_tmp) { if (cf->fd >= 0) { r = close(cf->fd); @@ -118,6 +125,10 @@ void libxl_postfork_child_noexec(libxl_c free(cf); } LIBXL_LIST_INIT(&carefds); + + if (sigchld_owner) + sigchld_removehandler_core(); + atfork_unlock(); } @@ -141,6 +152,250 @@ int libxl__carefd_fd(const libxl__carefd } /* + * Actual child process handling + */ + +static void sigchld_handler(int signo) +{ + int e = libxl__self_pipe_wakeup(sigchld_owner->sigchld_selfpipe[1]); + assert(!e); /* errors are probably EBADF, very bad */ +} + +static void sigchld_removehandler_core(void) +{ + struct sigaction was; + int r; + + r = sigaction(SIGCHLD, &sigchld_saved_action, &was); + assert(!r); + assert(!(was.sa_flags & SA_SIGINFO)); + assert(was.sa_handler == sigchld_handler); + sigchld_owner = 0; +} + +void libxl__sigchld_removehandler(libxl_ctx *ctx) /* non-reentrant */ +{ + atfork_lock(); + if (sigchld_owner == ctx) + sigchld_removehandler_core(); + atfork_unlock(); +} + +int libxl__sigchld_installhandler(libxl_ctx *ctx) /* non-reentrant */ +{ + int r, rc; + + if (ctx->sigchld_selfpipe[0] < 0) { + r = pipe(ctx->sigchld_selfpipe); + if (r) { + ctx->sigchld_selfpipe[0] = -1; + LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, + "failed to create sigchld pipe"); + rc = ERROR_FAIL; + goto out; + } + } + + atfork_lock(); + if (sigchld_owner != ctx) { + struct sigaction ours; + + assert(!sigchld_owner); + sigchld_owner = ctx; + + memset(&ours,0,sizeof(ours)); + ours.sa_handler = sigchld_handler; + sigemptyset(&ours.sa_mask); + ours.sa_flags = SA_NOCLDSTOP | SA_RESTART; + r = sigaction(SIGCHLD, &ours, &sigchld_saved_action); + assert(!r); + + assert(((void)"application must negotiate with libxl about SIGCHLD", + !(sigchld_saved_action.sa_flags & SA_SIGINFO) && + (sigchld_saved_action.sa_handler == SIG_DFL || + sigchld_saved_action.sa_handler == SIG_IGN))); + } + atfork_unlock(); + + rc = 0; + out: + return rc; +} + +static int chldmode_ours(libxl_ctx *ctx) +{ + return ctx->childproc_hooks->chldowner == libxl_sigchld_owner_libxl; +} + +int libxl__fork_selfpipe_active(libxl_ctx *ctx) +{ + /* Returns the fd to read, or -1 */ + if (!chldmode_ours(ctx)) + return -1; + + if (LIBXL_LIST_EMPTY(&ctx->children)) + return -1; + + return ctx->sigchld_selfpipe[0]; +} + +static void perhaps_removehandler(libxl_ctx *ctx) +{ + if (LIBXL_LIST_EMPTY(&ctx->children) && + ctx->childproc_hooks->chldowner != libxl_sigchld_owner_libxl_always) + libxl__sigchld_removehandler(ctx); +} + +static int childproc_reaped(libxl__egc *egc, pid_t pid, int status) +{ + EGC_GC; + libxl__ev_child *ch; + + LIBXL_LIST_FOREACH(ch, &CTX->children, entry) + if (ch->pid == pid) + goto found; + + /* not found */ + return ERROR_UNKNOWN_CHILD; + + found: + LIBXL_LIST_REMOVE(ch, entry); + ch->pid = -1; + ch->callback(egc, ch, pid, status); + + perhaps_removehandler(CTX); + + return 0; +} + +int libxl_childproc_reaped(libxl_ctx *ctx, pid_t pid, int status) +{ + EGC_INIT(ctx); + CTX_LOCK; + int rc = childproc_reaped(egc, pid, status); + CTX_UNLOCK; + EGC_FREE; + return rc; +} + +void libxl__fork_selfpipe_woken(libxl__egc *egc) +{ + /* May make callbacks into the application for child processes. + * ctx must be locked EXACTLY ONCE */ + EGC_GC; + + while (chldmode_ours(CTX) /* in case the app changes the mode */) { + int status; + pid_t pid = waitpid(-1, &status, WNOHANG); + + if (pid == 0) return; + + if (pid == -1) { + if (errno == ECHILD) return; + if (errno == EINTR) continue; + LIBXL__EVENT_DISASTER(egc, "waitpid() failed", errno, 0); + return; + } + + int rc = childproc_reaped(egc, pid, status); + + if (rc) { + if (CTX->childproc_hooks->reaped_callback) { + CTX_UNLOCK; + rc = CTX->childproc_hooks->reaped_callback + (pid, status, CTX->childproc_user); + CTX_LOCK; + if (rc != 0 && rc != ERROR_UNKNOWN_CHILD) { + char disasterbuf[200]; + snprintf(disasterbuf, sizeof(disasterbuf), " reported by" + " libxl_childproc_hooks->reaped_callback" + " (for pid=%lu, status=%d; error code %d)", + (unsigned long)pid, status, rc); + LIBXL__EVENT_DISASTER(egc, disasterbuf, 0, 0); + return; + } + } else { + rc = ERROR_UNKNOWN_CHILD; + } + if (rc) + libxl_report_child_exitstatus(CTX, XTL_WARN, + "unknown child", (long)pid, status); + } + } +} + +pid_t libxl__ev_child_fork(libxl__gc *gc, libxl__ev_child *ch, + libxl__ev_child_callback *death) +{ + CTX_LOCK; + int rc; + + if (chldmode_ours(CTX)) { + rc = libxl__sigchld_installhandler(CTX); + if (rc) goto out; + } + + pid_t pid = + CTX->childproc_hooks->fork_replacement + ? CTX->childproc_hooks->fork_replacement(CTX->childproc_user) + : fork(); + if (pid == -1) { + LOGE(ERROR, "fork failed"); + rc = ERROR_FAIL; + goto out; + } + + if (!pid) { + /* woohoo! */ + return 0; /* Yes, CTX is left locked in the child. */ + } + + ch->pid = pid; + ch->callback = death; + LIBXL_LIST_INSERT_HEAD(&CTX->children, ch, entry); + rc = pid; + + out: + perhaps_removehandler(CTX); + CTX_UNLOCK; + return rc; +} + +void libxl_childproc_setmode(libxl_ctx *ctx, const libxl_childproc_hooks *hooks, + void *user) +{ + GC_INIT(ctx); + CTX_LOCK; + + assert(LIBXL_LIST_EMPTY(&CTX->children)); + + if (!hooks) + hooks = &libxl__childproc_default_hooks; + + ctx->childproc_hooks = hooks; + ctx->childproc_user = user; + + switch (ctx->childproc_hooks->chldowner) { + case libxl_sigchld_owner_mainloop: + case libxl_sigchld_owner_libxl: + libxl__sigchld_removehandler(ctx); + break; + case libxl_sigchld_owner_libxl_always: + libxl__sigchld_installhandler(ctx); + break; + default: + abort(); + } + + CTX_UNLOCK; + GC_FREE; +} + +const libxl_childproc_hooks libxl__childproc_default_hooks = { + libxl_sigchld_owner_libxl, 0, 0 +}; + +/* * Local variables: * mode: C * c-basic-offset: 4 diff -r 4e7a03e27a12 -r e3371fc765b1 tools/libxl/libxl_internal.h --- a/tools/libxl/libxl_internal.h Fri May 11 18:58:55 2012 +0100 +++ b/tools/libxl/libxl_internal.h Fri May 11 18:58:56 2012 +0100 @@ -198,6 +198,19 @@ _hidden libxl__ev_xswatch *libxl__watch_ int slotnum); +typedef struct libxl__ev_child libxl__ev_child; +typedef void libxl__ev_child_callback(libxl__egc *egc, libxl__ev_child*, + pid_t pid, int status); +struct libxl__ev_child { + /* caller should include this in their own struct */ + /* read-only for caller: */ + pid_t pid; /* -1 means unused ("unregistered", ie Idle) */ + libxl__ev_child_callback *callback; + /* remainder is private for libxl__ev_... */ + LIBXL_LIST_ENTRY(struct libxl__ev_child) entry; +}; + + /* * evgen structures, which are the state we use for generating * events for the caller. @@ -306,10 +319,14 @@ struct libxl__ctx { LIBXL_LIST_HEAD(, libxl_evgen_disk_eject) disk_eject_evgens; - /* for callers who reap children willy-nilly; caller must only - * set this after libxl_init and before any other call - or - * may leave them untouched */ + const libxl_childproc_hooks *childproc_hooks; + void *childproc_user; + int sigchld_selfpipe[2]; /* [0]==-1 means handler not installed */ + LIBXL_LIST_HEAD(, libxl__ev_child) children; + + /* This is obsolete and must be removed: */ int (*waitpid_instead)(pid_t pid, int *status, int flags); + libxl_version_info version_info; }; @@ -557,6 +574,36 @@ static inline int libxl__ev_xswatch_isre /* + * For making subprocesses. This is the only permitted mechanism for + * code in libxl to do so. + * + * In the parent, returns the pid, filling in childw_out. + * In the child, returns 0. + * If it fails, returns a libxl error (all of which are -ve). + * + * The child should go on to exec (or exit) soon. The child may not + * make any further calls to libxl infrastructure, except for memory + * allocation and logging. If the child needs to use xenstore it + * must open its own xs handle and use it directly, rather than via + * the libxl event machinery. + * + * The parent may signal the child but it must not reap it. That will + * be done by the event machinery. death may be NULL in which case + * the child is still reaped but its death is ignored. + * + * It is not possible to "deregister" the child death event source. + * It will generate exactly one event callback; until then the childw + * is Active and may not be reused. + */ +_hidden pid_t libxl__ev_child_fork(libxl__gc *gc, libxl__ev_child *childw_out, + libxl__ev_child_callback *death); +static inline void libxl__ev_child_init(libxl__ev_child *childw_out) + { childw_out->pid = -1; } +static inline int libxl__ev_child_inuse(libxl__ev_child *childw_out) + { return childw_out->pid >= 0; } + + +/* * Other event-handling support provided by the libxl event core to * the rest of libxl. */ @@ -610,6 +657,15 @@ _hidden void libxl__poller_put(libxl_ctx * ctx must be locked. */ _hidden void libxl__poller_wakeup(libxl__egc *egc, libxl__poller *p); +/* Internal to fork and child reaping machinery */ +extern const libxl_childproc_hooks libxl__childproc_default_hooks; +int libxl__sigchld_installhandler(libxl_ctx *ctx); /* non-reentrant;logs errs */ +void libxl__sigchld_removehandler(libxl_ctx *ctx); /* non-reentrant */ +int libxl__fork_selfpipe_active(libxl_ctx *ctx); /* returns read fd or -1 */ +void libxl__fork_selfpipe_woken(libxl__egc *egc); +int libxl__self_pipe_wakeup(int fd); /* returns 0 or -1 setting errno */ +int libxl__self_pipe_eatall(int fd); /* returns 0 or -1 setting errno */ + _hidden int libxl__atfork_init(libxl_ctx *ctx); _______________________________________________ Xen-changelog mailing list Xen-changelog@xxxxxxxxxxxxx http://lists.xensource.com/xen-changelog
|
Lists.xenproject.org is hosted with RackSpace, monitoring our |