[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH 20/20] libxl: ao: Convert libxl_run_bootloader
Convert libxl_run_bootloader to an ao_how-taking function. It's implemented in terms of libxl__bootloader_run, which can be used internally. The resulting code is pretty much a rewrite. Significant changes include: - We provide and use a new asynchronous internal facility for pty creation, which runs openpty in a child for SIGCHLD etc. safety (see the comment at the top of libxl__openptys). - We direct the bootloader's results to a file, not a pipe. This makes it simpler to deal with as we don't have to read it concurrently along with everything else. - We now issue a warning if we find unexpected statements in the bootloader results. - The arrangements for buffering of bootloader input and output are completely changed. Now we have a fixed limit of 64k on output, and 4k on input, and discard the oldest data when this overflows (which it shouldn't). There is no timeout any more. - This is implemented using a new asynchronous internal data- copying facility. Signed-off-by: Ian Jackson <ian.jackson@xxxxxxxxxxxxx> --- tools/libxl/libxl.c | 4 + tools/libxl/libxl.h | 3 +- tools/libxl/libxl_bootloader.c | 957 ++++++++++++++++++++++++++-------------- tools/libxl/libxl_create.c | 8 +- tools/libxl/libxl_internal.h | 97 ++++ 5 files changed, 733 insertions(+), 336 deletions(-) diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c index 59992b6..9ad02b7 100644 --- a/tools/libxl/libxl.c +++ b/tools/libxl/libxl.c @@ -1748,6 +1748,10 @@ int libxl_device_disk_local_detach(libxl_ctx *ctx, libxl_device_disk *disk) * For other device types assume that the blktap2 process is * needed by the soon to be started domain and do nothing. */ + /* + * FIXME + * This appears to leak the disk in failure paths + */ return 0; } diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h index 50bae2f..c99b62d 100644 --- a/tools/libxl/libxl.h +++ b/tools/libxl/libxl.h @@ -415,7 +415,8 @@ int libxl_get_max_cpus(libxl_ctx *ctx); int libxl_run_bootloader(libxl_ctx *ctx, libxl_domain_build_info *info, libxl_device_disk *disk, - uint32_t domid); + uint32_t domid, + libxl_asyncop_how *ao_how); /* 0 means ERROR_ENOMEM, which we have logged */ diff --git a/tools/libxl/libxl_bootloader.c b/tools/libxl/libxl_bootloader.c index 2774062..725fa09 100644 --- a/tools/libxl/libxl_bootloader.c +++ b/tools/libxl/libxl_bootloader.c @@ -15,70 +15,219 @@ #include "libxl_osdeps.h" /* must come before any other headers */ #include <termios.h> +#include <utmp.h> #include "libxl_internal.h" -#define XENCONSOLED_BUF_SIZE 16 -#define BOOTLOADER_BUF_SIZE 4096 -#define BOOTLOADER_TIMEOUT 1 +#define BOOTLOADER_BUF_OUT 65536 +#define BOOTLOADER_BUF_IN 4096 -static char **make_bootloader_args(libxl__gc *gc, - libxl_domain_build_info *info, - uint32_t domid, - const char *fifo, char *disk) +static void bootloader_gotptys(libxl__egc *egc, libxl__openpty_state *op); +static void bootloader_keystrokes_copyfail(libxl__egc *egc, + libxl__datacopier_state *dc, int onwrite, int errnoval); +static void bootloader_display_copyfail(libxl__egc *egc, + libxl__datacopier_state *dc, int onwrite, int errnoval); +static void bootloader_finished(libxl__egc *egc, libxl__ev_child *child, + pid_t pid, int status); + +/*----- openpty -----*/ + +/* + * opens count (>0) ptys like count calls to openpty, and then + * calls back. On entry, all op[].master and op[].slave must be + * 0. On callback, either rc==0 and master and slave are non-0, + * or rc is a libxl error and they are both 0. If libxl__openpty + * returns non-0 no callback will happen and everything is left + * cleaned up. + */ + +/* implementation */ + +static void openpty_cleanup(libxl__openpty_state *op) +{ + int i; + + for (i=0; i<op->count; i++) { + libxl__openpty_result *res = &op->results[i]; + libxl__carefd_close(res->master); res->master = 0; + libxl__carefd_close(res->slave); res->slave = 0; + } +} + +static void openpty_exited(libxl__egc *egc, libxl__ev_child *child, + pid_t pid, int status) { + libxl__openpty_state *op = CONTAINER_OF(child, *op, child); + EGC_GC; + + if (status) { + /* Perhaps the child gave us the fds and then exited nonzero. + * Well that would be odd but we don't really care. */ + libxl_report_child_exitstatus(CTX, op->rc ? LIBXL__LOG_ERROR + : LIBXL__LOG_WARNING, + "openpty child", pid, status); + } + if (op->rc) + openpty_cleanup(op); + op->callback(egc, op); +} + +int libxl__openptys(libxl__gc *gc, libxl__openpty_state *op, + const struct termios *termp, + const struct winsize *winp) { + /* + * This is completely crazy. openpty calls grantpt which the spec + * says may fork, and may not be called with a SIGCHLD handler. + * Now our application may have a SIGCHLD handler so that's bad. + * We could perhaps block it but we'd need to block it on all + * threads. This is just Too Hard. + * + * So instead, we run openpty in a child process. That child + * process then of course has only our own thread and our own + * signal handlers. We pass the fds back. + * + * Since our only current caller actually wants two ptys, we + * support calling openpty multiple times for a single fork. + */ + int count = op->count; + int r, i, rc, sockets[2], ptyfds[count][2]; + libxl__carefd *for_child = 0; + pid_t pid = -1; + + for (i=0; i<count; i++) { + ptyfds[i][0] = ptyfds[i][1] = -1; + libxl__openpty_result *res = &op->results[i]; + assert(!res->master); + assert(!res->slave); + } + sockets[0] = sockets[1] = -1; /* 0 is for us, 1 for our child */ + + libxl__carefd_begin(); + r = socketpair(AF_UNIX, SOCK_STREAM, 0, sockets); + if (r) { sockets[0] = sockets[1] = -1; } + for_child = libxl__carefd_opened(CTX, sockets[1]); + if (r) { LOGE(ERROR,"socketpair failed"); rc = ERROR_FAIL; goto out; } + + pid = libxl__ev_child_fork(gc, &op->child, openpty_exited); + if (pid == -1) { + rc = ERROR_FAIL; + goto out; + } + + if (!pid) { + /* child */ + close(sockets[0]); + signal(SIGCHLD, SIG_DFL); + + for (i=0; i<count; i++) { + r = openpty(&ptyfds[i][0], &ptyfds[i][1], NULL, termp, winp); + if (r) { LOGE(ERROR,"openpty failed"); _exit(-1); } + } + rc = libxl__sendmsg_fds(gc, sockets[1], "",1, + 2*count, &ptyfds[0][0], "ptys"); + if (rc) { LOGE(ERROR,"sendmsg to parent failed"); _exit(-1); } + _exit(0); + } + + libxl__carefd_close(for_child); + for_child = 0; + + /* this should be fast so do it synchronously */ + + libxl__carefd_begin(); + char buf[1]; + rc = libxl__recvmsg_fds(gc, sockets[1], buf,1, + 2*count, &ptyfds[0][0], "ptys"); + if (!rc) { + for (i=0; i<count; i++) { + libxl__openpty_result *res = &op->results[i]; + res->master = libxl__carefd_record(CTX, ptyfds[i][0]); + res->slave = libxl__carefd_record(CTX, ptyfds[i][1]); + } + } + /* now the pty fds are in the carefds, if they were ever open */ + libxl__carefd_unlock(); + if (rc) + goto out; + + rc = 0; + + out: + if (sockets[0] >= 0) close(sockets[0]); + if (sockets[1] >= 0) close(sockets[1]); + libxl__carefd_close(for_child); + if (libxl__ev_child_inuse(&op->child)) { + op->rc = rc; + /* we will get a callback when the child dies */ + return 0; + } + + assert(rc); + openpty_cleanup(op); + return rc; +} + +/*----- bootloader arguments -----*/ + +static void bootloader_arg(libxl__bootloader_state *bl, const char *arg) { + assert(bl->nargs < bl->argsspace); + bl->args[bl->nargs++] = arg; +} + +static void make_bootloader_args(libxl__gc *gc, libxl__bootloader_state *bl) { - flexarray_t *args; - int nr = 0; + const libxl_domain_build_info *info = bl->info; + + bl->argsspace = 7 + libxl_string_list_length(&info->u.pv.bootloader_args); + + GCNEW_ARRAY(bl->args, bl->argsspace); - args = flexarray_make(1, 1); - if (!args) - return NULL; +#define ARG(arg) bootloader_arg(bl, (arg)) - flexarray_set(args, nr++, (char *)info->u.pv.bootloader); + ARG(info->u.pv.bootloader); if (info->u.pv.kernel.path) - flexarray_set(args, nr++, libxl__sprintf(gc, "--kernel=%s", - info->u.pv.kernel.path)); + ARG(libxl__sprintf(gc, "--kernel=%s", info->u.pv.kernel.path)); if (info->u.pv.ramdisk.path) - flexarray_set(args, nr++, libxl__sprintf(gc, "--ramdisk=%s", info->u.pv.ramdisk.path)); + ARG(libxl__sprintf(gc, "--ramdisk=%s", info->u.pv.ramdisk.path)); if (info->u.pv.cmdline && *info->u.pv.cmdline != '\0') - flexarray_set(args, nr++, libxl__sprintf(gc, "--args=%s", info->u.pv.cmdline)); + ARG(libxl__sprintf(gc, "--args=%s", info->u.pv.cmdline)); - flexarray_set(args, nr++, libxl__sprintf(gc, "--output=%s", fifo)); - flexarray_set(args, nr++, "--output-format=simple0"); - flexarray_set(args, nr++, libxl__sprintf(gc, "--output-directory=%s", "/var/run/libxl/")); + ARG(libxl__sprintf(gc, "--output=%s", bl->outputpath)); + ARG("--output-format=simple0"); + ARG(libxl__sprintf(gc, "--output-directory=%s", "/var/run/xen/")); if (info->u.pv.bootloader_args) { char **p = info->u.pv.bootloader_args; while (*p) { - flexarray_set(args, nr++, *p); + ARG(*p); p++; } } - flexarray_set(args, nr++, disk); + ARG(bl->diskpath); /* Sentinal for execv */ - flexarray_set(args, nr++, NULL); + ARG(NULL); - return (char **) flexarray_contents(args); /* Frees args */ +#undef ARG } -static int open_xenconsoled_pty(int *master, int *slave, char *slave_path, size_t slave_path_len) +/*----- synchronous subroutines -----*/ + +static int setup_xenconsoled_pty(libxl__egc *egc, libxl__bootloader_state *bl, + char *slave_path, size_t slave_path_len) { + EGC_GC; struct termios termattr; - int ret; - - ret = openpty(master, slave, NULL, NULL, NULL); - if (ret < 0) - return -1; - - ret = ttyname_r(*slave, slave_path, slave_path_len); - if (ret == -1) { - close(*master); - close(*slave); - *master = *slave = -1; - return -1; + int r, rc; + int slave = libxl__carefd_fd(bl->ptys[1].slave); + int master = libxl__carefd_fd(bl->ptys[1].master); + + r = ttyname_r(slave, slave_path, slave_path_len); + if (r == -1) { + LOGE(ERROR,"ttyname_r failed"); + rc = ERROR_FAIL; + goto out; } /* @@ -92,309 +241,360 @@ static int open_xenconsoled_pty(int *master, int *slave, char *slave_path, size_ * for it. */ #if !defined(__sun__) && !defined(__NetBSD__) - tcgetattr(*master, &termattr); + tcgetattr(master, &termattr); cfmakeraw(&termattr); - tcsetattr(*master, TCSANOW, &termattr); + tcsetattr(master, TCSANOW, &termattr); - close(*slave); - *slave = -1; + close(slave); + slave = -1; #else - tcgetattr(*slave, &termattr); + tcgetattr(slave, &termattr); cfmakeraw(&termattr); - tcsetattr(*slave, TCSANOW, &termattr); + tcsetattr(slave, TCSANOW, &termattr); #endif - fcntl(*master, F_SETFL, O_NDELAY); - fcntl(*master, F_SETFD, FD_CLOEXEC); - return 0; -} -static pid_t fork_exec_bootloader(int *master, const char *arg0, char **args) -{ - struct termios termattr; - pid_t pid = forkpty(master, NULL, NULL, NULL); - if (pid == -1) - return -1; - else if (pid == 0) { - setenv("TERM", "vt100", 1); - libxl__exec(-1, -1, -1, arg0, args); - return -1; - } + out: + return rc; +} - /* - * On Solaris, the master pty side does not have terminal semantics, - * so don't try to set any attributes, as it will fail. - */ -#if !defined(__sun__) - tcgetattr(*master, &termattr); - cfmakeraw(&termattr); - tcsetattr(*master, TCSANOW, &termattr); -#endif +static const char *bootloader_result_command(const char *buf, + const char *prefix, size_t prefixlen) { + if (strncmp(buf, prefix, prefixlen)) + return 0; - fcntl(*master, F_SETFL, O_NDELAY); + const char *rhs = buf + prefixlen; + if (!CTYPE(isspace,*rhs)) + return 0; - return pid; + while (CTYPE(isspace,*rhs)) + rhs++; + return rhs; } -/* - * filedescriptors: - * fifo_fd - bootstring output from the bootloader - * xenconsoled_fd - input/output from/to xenconsole - * bootloader_fd - input/output from/to pty that controls the bootloader - * The filedescriptors are NDELAY, so it's ok to try to read - * bigger chunks than may be available, to keep e.g. curses - * screen redraws in the bootloader efficient. xenconsoled_fd is the side that - * gets xenconsole input, which will be keystrokes, so a small number - * is sufficient. bootloader_fd is pygrub output, which will be curses screen - * updates, so a larger number (1024) is appropriate there. - * - * For writeable descriptors, only include them in the set for select - * if there is actual data to write, otherwise this would loop too fast, - * eating up CPU time. - */ -static char * bootloader_interact(libxl__gc *gc, int xenconsoled_fd, int bootloader_fd, int fifo_fd) +static int parse_bootloader_result(libxl__egc *egc, + libxl__bootloader_state *bl) { - int ret; - - size_t nr_out = 0, size_out = 0; - char *output = NULL; - struct timeval wait; - - /* input from xenconsole. read on xenconsoled_fd write to bootloader_fd */ - int xenconsoled_prod = 0, xenconsoled_cons = 0; - char xenconsoled_buf[XENCONSOLED_BUF_SIZE]; - /* output from bootloader. read on bootloader_fd write to xenconsoled_fd */ - int bootloader_prod = 0, bootloader_cons = 0; - char bootloader_buf[BOOTLOADER_BUF_SIZE]; - - while(1) { - fd_set wsel, rsel; - int nfds; - - /* Set timeout to 1s before starting to discard data */ - wait.tv_sec = BOOTLOADER_TIMEOUT; - wait.tv_usec = 0; - - /* Move buffers around to drop already consumed data */ - if (xenconsoled_cons > 0) { - xenconsoled_prod -= xenconsoled_cons; - memmove(xenconsoled_buf, &xenconsoled_buf[xenconsoled_cons], - xenconsoled_prod); - xenconsoled_cons = 0; + EGC_GC; + char buf[PATH_MAX]; + FILE *f = 0; + int rc = ERROR_FAIL; + libxl_domain_build_info *info = bl->info; + + f = fopen(bl->outputpath, "r"); + if (!f) { + LOGE(ERROR,"open bootloader output file %s", bl->outputpath); + goto out; + } + + for (;;) { + if (!fgets(buf, sizeof(buf), f)) { + if (feof(f)) break; + assert(ferror(f)); + LOGE(ERROR,"read bootloader output file %s", bl->outputpath); + goto out; } - if (bootloader_cons > 0) { - bootloader_prod -= bootloader_cons; - memmove(bootloader_buf, &bootloader_buf[bootloader_cons], - bootloader_prod); - bootloader_cons = 0; + int l = strlen(buf); + assert(l > 0); + if (buf[l-1] != '\n') { + LOG(WARN,"bootloader output contained long line `%.50s...'", buf); + int c; + while ((c=fgetc(f)) != EOF && c !='\n') + continue; + continue; } - FD_ZERO(&rsel); - FD_SET(fifo_fd, &rsel); - nfds = fifo_fd + 1; - if (xenconsoled_prod < XENCONSOLED_BUF_SIZE) { - /* The buffer is not full, try to read more data */ - FD_SET(xenconsoled_fd, &rsel); - nfds = xenconsoled_fd + 1 > nfds ? xenconsoled_fd + 1 : nfds; - } - if (bootloader_prod < BOOTLOADER_BUF_SIZE) { - /* The buffer is not full, try to read more data */ - FD_SET(bootloader_fd, &rsel); - nfds = bootloader_fd + 1 > nfds ? bootloader_fd + 1 : nfds; - } + const char *rhs; +#define COMMAND(s) ((rhs = bootloader_result_command(buf, s, sizeof(s)-1))) - FD_ZERO(&wsel); - if (bootloader_prod > 0) { - /* The buffer has data to consume */ - FD_SET(xenconsoled_fd, &wsel); - nfds = xenconsoled_fd + 1 > nfds ? xenconsoled_fd + 1 : nfds; - } - if (xenconsoled_prod > 0) { - /* The buffer has data to consume */ - FD_SET(bootloader_fd, &wsel); - nfds = bootloader_fd + 1 > nfds ? bootloader_fd + 1 : nfds; + if (COMMAND("kernel")) { + free(info->u.pv.kernel.path); + info->u.pv.kernel.path = libxl__strdup(NULL, rhs); + libxl__file_reference_map(&info->u.pv.kernel); + unlink(info->u.pv.kernel.path); + } else if (COMMAND("ramdisk")) { + free(info->u.pv.ramdisk.path); + info->u.pv.ramdisk.path = libxl__strdup(NULL, rhs); + libxl__file_reference_map(&info->u.pv.ramdisk); + unlink(info->u.pv.ramdisk.path); + } else if (COMMAND("args")) { + free(info->u.pv.cmdline); + info->u.pv.cmdline = libxl__strdup(NULL, rhs); + } else { + LOG(WARN, "unexpected output from bootloader: `%s'", buf); } + } + rc = 0; - if (xenconsoled_prod == XENCONSOLED_BUF_SIZE || - bootloader_prod == BOOTLOADER_BUF_SIZE) - ret = select(nfds, &rsel, &wsel, NULL, &wait); - else - ret = select(nfds, &rsel, &wsel, NULL, NULL); - if (ret < 0) { - if (errno == EINTR) - continue; - goto out_err; - } + out: + if (f) fclose(f); + return rc; +} + + +/*----- data copier -----*/ + +void libxl__datacopier_init(libxl__datacopier_state *dc) +{ + libxl__ev_fd_init(&dc->toread); + libxl__ev_fd_init(&dc->towrite); + LIBXL_TAILQ_INIT(&dc->bufs); +} + +void libxl__datacopier_kill(libxl__gc *gc, libxl__datacopier_state *dc) +{ + libxl__datacopier_buf *buf, *tbuf; + + libxl__ev_fd_deregister(gc, &dc->toread); + libxl__ev_fd_deregister(gc, &dc->towrite); + LIBXL_TAILQ_FOREACH_SAFE(buf, &dc->bufs, entry, tbuf) + free(buf); + LIBXL_TAILQ_INIT(&dc->bufs); +} - /* Input from xenconsole, read xenconsoled_fd, write bootloader_fd */ - if (ret == 0 && xenconsoled_prod == XENCONSOLED_BUF_SIZE) { - /* Drop the buffer */ - xenconsoled_prod = 0; - xenconsoled_cons = 0; - } else if (FD_ISSET(xenconsoled_fd, &rsel)) { - ret = read(xenconsoled_fd, &xenconsoled_buf[xenconsoled_prod], XENCONSOLED_BUF_SIZE - xenconsoled_prod); - if (ret < 0 && errno != EIO && errno != EAGAIN) - goto out_err; - if (ret > 0) - xenconsoled_prod += ret; +static void datacopier_callback(libxl__egc *egc, libxl__datacopier_state *dc, + int onwrite, int errnoval) +{ + EGC_GC; + libxl__datacopier_kill(gc, dc); + dc->callback(egc, dc, onwrite, errnoval); +} + +static void datacopier_writable(libxl__egc *egc, libxl__ev_fd *ev, + int fd, short events, short revents); + +static void datacopier_check_state(libxl__egc *egc, libxl__datacopier_state *dc) +{ + EGC_GC; + int rc; + + if (dc->used) { + if (!libxl__ev_fd_isregistered(&dc->towrite)) { + rc = libxl__ev_fd_register(gc, &dc->towrite, datacopier_writable, + dc->writefd, POLLOUT); + if (!rc) { + LOG(ERROR, "unable to establish write event" + " during copy (%s)", dc->what); + datacopier_callback(egc, dc, -1, 0); + return; + } } - if (FD_ISSET(bootloader_fd, &wsel)) { - ret = write(bootloader_fd, &xenconsoled_buf[xenconsoled_cons], xenconsoled_prod - xenconsoled_cons); - if (ret < 0 && errno != EIO && errno != EAGAIN) - goto out_err; - if (ret > 0) - xenconsoled_cons += ret; + } else if (!libxl__ev_fd_isregistered(&dc->toread)) { + /* we have had eof */ + libxl__datacopier_kill(gc, dc); + dc->callback(egc, dc, 0, 0); + return; + } else { + /* nothing buffered, but still reading */ + libxl__ev_fd_deregister(gc, &dc->towrite); + } +} + +static void datacopier_readable(libxl__egc *egc, libxl__ev_fd *ev, + int fd, short events, short revents) { + libxl__datacopier_state *dc = CONTAINER_OF(ev, *dc, toread); + EGC_GC; + + if (revents & ~POLLIN) { + LOG(ERROR, "unexpected poll event 0x%x (should be POLLIN)" + " during copy (%s)", revents, dc->what); + datacopier_callback(egc, dc, -1, 0); + return; + } + assert(revents & POLLIN); + for (;;) { + while (dc->used >= dc->maxsz) { + libxl__datacopier_buf *rm = LIBXL_TAILQ_FIRST(&dc->bufs); + dc->used -= rm->used; + assert(dc->used >= 0); + LIBXL_TAILQ_REMOVE(&dc->bufs, rm, entry); + free(rm); } - /* Input from bootloader, read bootloader_fd, write xenconsoled_fd */ - if (ret == 0 && bootloader_prod == BOOTLOADER_BUF_SIZE) { - /* Drop the buffer */ - bootloader_prod = 0; - bootloader_cons = 0; - } else if (FD_ISSET(bootloader_fd, &rsel)) { - ret = read(bootloader_fd, &bootloader_buf[bootloader_prod], BOOTLOADER_BUF_SIZE - bootloader_prod); - if (ret < 0 && errno != EIO && errno != EAGAIN) - goto out_err; - if (ret > 0) - bootloader_prod += ret; + libxl__datacopier_buf *buf = + LIBXL_TAILQ_LAST(&dc->bufs, libxl__datacopier_bufs); + if (buf->used >= sizeof(buf->buf)) { + buf = malloc(sizeof(*buf)); + if (!buf) libxl__alloc_failed(__func__, 1, sizeof(*buf)); + buf->used = 0; + LIBXL_TAILQ_INSERT_TAIL(&dc->bufs, buf, entry); + } + int r = read(ev->fd, + buf->buf + buf->used, + sizeof(buf->buf) - buf->used); + if (r < 0) { + if (errno == EINTR) continue; + if (errno == EWOULDBLOCK) break; + LOGE(ERROR, "error reading during copy (%s)", dc->what); + datacopier_callback(egc, dc, 0, errno); + return; } - if (FD_ISSET(xenconsoled_fd, &wsel)) { - ret = write(xenconsoled_fd, &bootloader_buf[bootloader_cons], bootloader_prod - bootloader_cons); - if (ret < 0 && errno != EIO && errno != EAGAIN) - goto out_err; - if (ret > 0) - bootloader_cons += ret; + if (r == 0) { + libxl__ev_fd_deregister(gc, &dc->toread); + break; } + buf->used += r; + dc->used += r; + assert(buf->used <= sizeof(buf->buf)); + } + datacopier_check_state(egc, dc); +} - if (FD_ISSET(fifo_fd, &rsel)) { - if (size_out - nr_out < 256) { - char *temp; - size_t new_size = size_out == 0 ? 32 : size_out * 2; - - temp = realloc(output, new_size); - if (temp == NULL) - goto out_err; - output = temp; - memset(output + size_out, 0, new_size - size_out); - size_out = new_size; - } +static void datacopier_writable(libxl__egc *egc, libxl__ev_fd *ev, + int fd, short events, short revents) { + EGC_GC; + libxl__datacopier_state *dc = CONTAINER_OF(ev, *dc, towrite); - ret = read(fifo_fd, output + nr_out, size_out - nr_out); - if (ret > 0) - nr_out += ret; - if (ret == 0) - break; + if (revents & ~POLLOUT) { + LOG(ERROR, "unexpected poll event 0x%x (should be POLLOUT)" + " during copy (%s)", revents, dc->what); + datacopier_callback(egc, dc, -1, 0); + return; + } + assert(revents & POLLOUT); + for (;;) { + libxl__datacopier_buf *buf = LIBXL_TAILQ_FIRST(&dc->bufs); + if (!buf) + break; + int r = write(ev->fd, buf->buf, buf->used); + if (r < 0) { + if (errno == EINTR) continue; + if (errno == EWOULDBLOCK) break; + LOGE(ERROR, "error writing during copy (%s)", dc->what); + datacopier_callback(egc, dc, 1, errno); + return; } + assert(r > 0); + assert(r <= buf->used); + buf->used -= r; + dc->used -= r; + assert(dc->used >= 0); + memmove(buf->buf, buf->buf+r, buf->used); } + datacopier_check_state(egc, dc); +} + +int libxl__datacopier_start(libxl__gc *gc, libxl__datacopier_state *dc) +{ + int rc; + + libxl__datacopier_init(dc); + + rc = libxl__ev_fd_register(gc, &dc->toread, datacopier_readable, + dc->readfd, POLLIN); + if (rc) goto out; - libxl__ptr_add(gc, output); - return output; + rc = libxl__ev_fd_register(gc, &dc->towrite, datacopier_writable, + dc->writefd, POLLOUT); + if (rc) goto out; + + return 0; -out_err: - free(output); - return NULL; + out: + libxl__datacopier_kill(gc, dc); + return rc; } -static void parse_bootloader_result(libxl__gc *gc, - libxl_domain_build_info *info, - const char *o) +/*----- init and cleanup -----*/ + +void libxl__bootloader_init(libxl__bootloader_state *bl) { - while (*o != '\0') { - if (strncmp("kernel ", o, strlen("kernel ")) == 0) { - free(info->u.pv.kernel.path); - info->u.pv.kernel.path = strdup(o + strlen("kernel ")); - libxl__file_reference_map(&info->u.pv.kernel); - unlink(info->u.pv.kernel.path); - } else if (strncmp("ramdisk ", o, strlen("ramdisk ")) == 0) { - free(info->u.pv.ramdisk.path); - info->u.pv.ramdisk.path = strdup(o + strlen("ramdisk ")); - libxl__file_reference_map(&info->u.pv.ramdisk); - unlink(info->u.pv.ramdisk.path); - } else if (strncmp("args ", o, strlen("args ")) == 0) { - free(info->u.pv.cmdline); - info->u.pv.cmdline = strdup(o + strlen("args ")); - } + bl->diskpath = NULL; + bl->ptys[0].master = bl->ptys[0].slave = 0; + bl->ptys[1].master = bl->ptys[1].slave = 0; + libxl__ev_child_init(&bl->child); + libxl__datacopier_init(&bl->keystrokes); + libxl__datacopier_init(&bl->display); +} - o = o + strlen(o) + 1; +static void bootloader_cleanup(libxl__egc *egc, libxl__bootloader_state *bl) +{ + libxl__ao *ao = bl->ao; + AO_GC; + int i; + + if (bl->diskpath) { + libxl_device_disk_local_detach(CTX, bl->disk); + free(bl->diskpath); + bl->diskpath = 0; + } + libxl__datacopier_kill(gc, &bl->keystrokes); + libxl__datacopier_kill(gc, &bl->display); + for (i=0; i<2; i++) { + libxl__carefd_close(bl->ptys[0].master); + libxl__carefd_close(bl->ptys[0].slave); } } -int libxl_run_bootloader(libxl_ctx *ctx, - libxl_domain_build_info *info, - libxl_device_disk *disk, - uint32_t domid) +static void bootloader_callback(libxl__egc *egc, libxl__bootloader_state *bl, + int rc) { - GC_INIT(ctx); - int ret, rc = 0; - char *fifo = NULL; - char *diskpath = NULL; - char **args = NULL; - - char tempdir_template[] = "/var/run/libxl/bl.XXXXXX"; - char *tempdir; - - char *dom_console_xs_path; - char dom_console_slave_tty_path[PATH_MAX]; - - int xenconsoled_fd = -1, xenconsoled_slave = -1; - int bootloader_fd = -1, fifo_fd = -1; + bootloader_cleanup(egc, bl); + bl->callback(egc, bl, rc); +} - int blrc; - pid_t pid; - char *blout; +/*----- main flow of control -----*/ - struct stat st_buf; +void libxl__bootloader_run(libxl__egc *egc, libxl__bootloader_state *bl) +{ + libxl__ao *ao = bl->ao; + AO_GC; + libxl_domain_build_info *info = bl->info; + uint32_t domid = bl->domid; + int rc, r; - rc = libxl__domain_build_info_setdefault(gc, info); - if (rc) goto out; + libxl__bootloader_init(bl); - if (info->type != LIBXL_DOMAIN_TYPE_PV || !info->u.pv.bootloader) + if (info->type != LIBXL_DOMAIN_TYPE_PV || !info->u.pv.bootloader) { + bl->callback(egc, bl, 0); + rc = 0; goto out; + } - rc = libxl__domain_build_info_setdefault(gc, info); - if (rc) goto out; - - rc = ERROR_INVAL; - if (!disk) - goto out; + bl->outputdir = GCSPRINTF(XEN_RUN_DIR "/bootloader.%"PRIu32".d", domid); + bl->outputpath = GCSPRINTF(XEN_RUN_DIR "/bootloader.%"PRIu32".out", domid); - rc = ERROR_FAIL; - ret = mkdir("/var/run/libxl/", S_IRWXU); - if (ret < 0 && errno != EEXIST) + for (;;) { + r = mkdir(bl->outputdir, 0600); + if (!r) break; + if (errno == EINTR) continue; + if (errno == EEXIST) break; + LOGE(ERROR, "failed to create bootloader dir %s", bl->outputdir); + rc = ERROR_FAIL; goto out; + } - ret = stat("/var/run/libxl/", &st_buf); - if (ret < 0) - goto out; + make_bootloader_args(gc, bl); - if (!S_ISDIR(st_buf.st_mode)) + bl->diskpath = libxl_device_disk_local_attach(CTX, bl->disk); + if (!bl->diskpath) { + rc = ERROR_FAIL; goto out; + } - tempdir = mkdtemp(tempdir_template); - if (tempdir == NULL) - goto out; + bl->openpty.callback = bootloader_gotptys; + bl->openpty.count = 2; + bl->openpty.results = bl->ptys; + rc = libxl__openptys(gc, &bl->openpty, 0,0); + if (rc) goto out; - ret = asprintf(&fifo, "%s/fifo", tempdir); - if (ret < 0) { - fifo = NULL; - goto out_close; - } + return; - ret = mkfifo(fifo, 0600); - if (ret < 0) { - goto out_close; - } + out: + assert(rc); + bootloader_callback(egc, bl, rc); +} - diskpath = libxl_device_disk_local_attach(ctx, disk); - if (!diskpath) { - goto out_close; - } +static void bootloader_gotptys(libxl__egc *egc, libxl__openpty_state *op) +{ + libxl__bootloader_state *bl = CONTAINER_OF(op, *bl, openpty); + EGC_GC; + int rc, r; - args = make_bootloader_args(gc, info, domid, fifo, diskpath); - if (args == NULL) { - rc = ERROR_NOMEM; - goto out_close; + if (bl->openpty.rc) { + rc = bl->openpty.rc; + goto out; } /* @@ -407,76 +607,167 @@ int libxl_run_bootloader(libxl_ctx *ctx, * where we copy characters between the two master fds, as well as * listening on the bootloader's fifo for the results. */ - ret = open_xenconsoled_pty(&xenconsoled_fd, &xenconsoled_slave, + + char *dom_console_xs_path; + char dom_console_slave_tty_path[PATH_MAX]; + rc = setup_xenconsoled_pty(egc, bl, &dom_console_slave_tty_path[0], sizeof(dom_console_slave_tty_path)); - if (ret < 0) { - goto out_close; - } + if (rc) goto out; + + char *dompath = libxl__xs_get_dompath(gc, bl->domid); + if (!dompath) { rc = ERROR_FAIL; goto out; } - dom_console_xs_path = libxl__sprintf(gc, "%s/console/tty", libxl__xs_get_dompath(gc, domid)); - libxl__xs_write(gc, XBT_NULL, dom_console_xs_path, "%s", dom_console_slave_tty_path); + dom_console_xs_path = GCSPRINTF("%s/console/tty", dompath); - pid = fork_exec_bootloader(&bootloader_fd, info->u.pv.bootloader, args); - if (pid < 0) { - goto out_close; + rc = libxl__xs_write(gc, XBT_NULL, dom_console_xs_path, "%s", + dom_console_slave_tty_path); + if (!rc) { + LOGE(ERROR,"xs write console path %s := %s failed", + dom_console_xs_path, dom_console_slave_tty_path); + rc = ERROR_FAIL; + goto out; } - while (1) { - if (waitpid(pid, &blrc, WNOHANG) == pid) - goto out_close; + int bootloader_master = libxl__carefd_fd(bl->ptys[0].master); + int xenconsole_master = libxl__carefd_fd(bl->ptys[1].master); - fifo_fd = open(fifo, O_RDONLY); - if (fifo_fd > -1) - break; + libxl_fd_set_nonblock(CTX, bootloader_master, 1); + libxl_fd_set_nonblock(CTX, xenconsole_master, 1); - if (errno == EINTR) - continue; + bl->keystrokes.readfd = xenconsole_master; + bl->keystrokes.writefd = bootloader_master; + bl->keystrokes.maxsz = BOOTLOADER_BUF_OUT; + bl->keystrokes.what = GCSPRINTF("bootloader input for %"PRIu32, bl->domid); + bl->keystrokes.callback = bootloader_keystrokes_copyfail; + rc = libxl__datacopier_start(gc, &bl->keystrokes); + if (rc) goto out; - goto out_close; - } + bl->display.readfd = bootloader_master; + bl->display.writefd = xenconsole_master; + bl->display.maxsz = BOOTLOADER_BUF_IN; + bl->display.what = GCSPRINTF("bootloader output for %"PRIu32, bl->domid); + bl->display.callback = bootloader_display_copyfail; + rc = libxl__datacopier_start(gc, &bl->display); + if (rc) goto out; - fcntl(fifo_fd, F_SETFL, O_NDELAY); + struct termios termattr; - blout = bootloader_interact(gc, xenconsoled_fd, bootloader_fd, fifo_fd); - if (blout == NULL) { - goto out_close; + pid_t pid = libxl__ev_child_fork(gc, &bl->child, bootloader_finished); + if (pid == -1) { + rc = ERROR_FAIL; + goto out; } - pid = waitpid(pid, &blrc, 0); - if (pid == -1 || (pid > 0 && WIFEXITED(blrc) && WEXITSTATUS(blrc) != 0)) { - goto out_close; + if (!pid) { + /* child */ + r = login_tty(libxl__carefd_fd(bl->ptys[0].slave)); + if (r) { LOGE(ERROR, "login_tty failed"); exit(-1); } + setenv("TERM", "vt100", 1); + libxl__exec(-1, -1, -1, bl->args[0], (char**)bl->args); + exit(-1); } - parse_bootloader_result(gc, info, blout); + /* parent */ + libxl__carefd_close(bl->ptys[0].slave); + bl->ptys[0].slave = 0; - rc = 0; -out_close: - if (diskpath) { - libxl_device_disk_local_detach(ctx, disk); - free(diskpath); + /* + * On Solaris, the master pty side does not have terminal semantics, + * so don't try to set any attributes, as it will fail. + */ +#if !defined(__sun__) + tcgetattr(bootloader_master, &termattr); + cfmakeraw(&termattr); + tcsetattr(bootloader_master, TCSANOW, &termattr); +#endif + + return; + + out: + bootloader_callback(egc, bl, rc); +} + +/* perhaps one of these will be called, but perhaps not */ +static void bootloader_copyfail(libxl__egc *egc, const char *which, + libxl__bootloader_state *bl, int onwrite, int errnoval) +{ + EGC_GC; + + if (!onwrite && !errnoval) + LOG(ERROR, "unexpected eof copying %s", which); + libxl__datacopier_kill(gc, &bl->keystrokes); + libxl__datacopier_kill(gc, &bl->display); + bl->rc = ERROR_FAIL; +} +static void bootloader_keystrokes_copyfail(libxl__egc *egc, + libxl__datacopier_state *dc, int onwrite, int errnoval) +{ + libxl__bootloader_state *bl = CONTAINER_OF(dc, *bl, keystrokes); + bootloader_copyfail(egc, "bootloader input", bl, onwrite, errnoval); +} +static void bootloader_display_copyfail(libxl__egc *egc, + libxl__datacopier_state *dc, int onwrite, int errnoval) +{ + libxl__bootloader_state *bl = CONTAINER_OF(dc, *bl, display); + bootloader_copyfail(egc, "bootloader output", bl, onwrite, errnoval); +} + +static void bootloader_finished(libxl__egc *egc, libxl__ev_child *child, + pid_t pid, int status) { + libxl__bootloader_state *bl = CONTAINER_OF(child, *bl, child); + EGC_GC; + int rc; + + libxl__datacopier_kill(gc, &bl->keystrokes); + libxl__datacopier_kill(gc, &bl->display); + + if (status) { + libxl_report_child_exitstatus(CTX, XTL_ERROR, "bootloader", + pid, status); + rc = ERROR_FAIL; + goto out; } - if (fifo_fd > -1) - close(fifo_fd); - if (bootloader_fd > -1) - close(bootloader_fd); - if (xenconsoled_fd > -1) - close(xenconsoled_fd); - if (xenconsoled_slave > -1) - close(xenconsoled_slave); - - if (fifo) { - unlink(fifo); - free(fifo); + + if (bl->rc) { + /* datacopier went wrong */ + rc = bl->rc; + goto out; } - rmdir(tempdir); + rc = parse_bootloader_result(egc, bl); + if (rc) goto out; - free(args); + rc = 0; -out: - GC_FREE; - return rc; + out: + bootloader_callback(egc, bl, rc); +} + +/*----- entrypoint for external callers -----*/ + +static void run_bootloader_done(libxl__egc *egc, + libxl__bootloader_state *st, int rc) +{ + libxl__ao_complete(egc, st->ao, rc); +} + +int libxl_run_bootloader(libxl_ctx *ctx, + libxl_domain_build_info *info, + libxl_device_disk *disk, + uint32_t domid, + libxl_asyncop_how *ao_how) +{ + AO_CREATE(ctx,domid,ao_how); + libxl__bootloader_state *bl; + + GCNEW(bl); + bl->ao = ao; + bl->callback = run_bootloader_done; + bl->info = info; + bl->disk = disk; + libxl__bootloader_run(egc, bl); + return AO_INPROGRESS; } /* diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c index 30dbc32..da461b6 100644 --- a/tools/libxl/libxl_create.c +++ b/tools/libxl/libxl_create.c @@ -559,8 +559,12 @@ static int do_domain_create(libxl__gc *gc, libxl_domain_config *d_config, if (ret) goto error_out; } - if ( restore_fd < 0 ) { - ret = libxl_run_bootloader(ctx, &d_config->b_info, d_config->num_disks > 0 ? &d_config->disks[0] : NULL, domid); + libxl_device_disk *bootdisk = + d_config->num_disks > 0 ? &d_config->disks[0] : NULL; + + if (restore_fd < 0 && bootdisk) { + ret = libxl_run_bootloader(ctx, &d_config->b_info, bootdisk, domid, + 0 /* fixme-ao */); if (ret) { LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "failed to run bootloader: %d", ret); diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h index d486af2..5b73db8 100644 --- a/tools/libxl/libxl_internal.h +++ b/tools/libxl/libxl_internal.h @@ -42,6 +42,7 @@ #include <sys/time.h> #include <sys/types.h> #include <sys/wait.h> +#include <sys/socket.h> #include <xs.h> #include <xenctrl.h> @@ -1433,6 +1434,102 @@ int libxl__carefd_close(libxl__carefd*); int libxl__carefd_fd(const libxl__carefd*); +/*----- openpty -----*/ + +typedef struct libxl__openpty_state libxl__openpty_state; +typedef struct libxl__openpty_result libxl__openpty_result; +typedef void libxl__openpty_callback(libxl__egc *egc, libxl__openpty_state *op); + +struct libxl__openpty_state { + /* caller must fill these in, and they must all remain valid */ + libxl__openpty_callback *callback; + int count; + libxl__openpty_result *results; /* actual size is count, out parameter */ + /* public, result, caller may only read in callback */ + int rc; + /* private for implementation */ + libxl__ev_child child; +}; + +struct libxl__openpty_result { + libxl__carefd *master, *slave; +}; + +int libxl__openptys(libxl__gc *gc, libxl__openpty_state *op, + const struct termios *termp, + const struct winsize *winp); + + +/*----- datacopier: copies data from one fd to another -----*/ + +typedef struct libxl__datacopier_state libxl__datacopier_state; +typedef struct libxl__datacopier_buf libxl__datacopier_buf; + +/* onwrite==1 means failure happend when writing, logged, errno is valid + * onwrite==0 means failure happend when reading + * errno==0 means we got eof and all data was written + * errno!=0 means we had a read error, logged + * onwrite==-1 means some other internal failure, errnoval not valid, logged + * in all cases copier is killed before calling this callback */ +typedef void libxl__datacopier_callback(libxl__egc *egc, + libxl__datacopier_state *dc, int onwrite, int errnoval); + +struct libxl__datacopier_buf { + /* private to datacopier */ + LIBXL_TAILQ_ENTRY(libxl__datacopier_buf) entry; + int used; + char buf[1000]; +}; + +struct libxl__datacopier_state { + /* caller must fill these in, and they must all remain valid */ + int readfd, writefd; + ssize_t maxsz; + const char *what; /* for error msgs */ + libxl__datacopier_callback *callback; + /* remaining fields are private to datacopier */ + libxl__ev_fd toread, towrite; + ssize_t used; + LIBXL_TAILQ_HEAD(libxl__datacopier_bufs, libxl__datacopier_buf) bufs; +}; + +_hidden void libxl__datacopier_init(libxl__datacopier_state *dc); +_hidden void libxl__datacopier_kill(libxl__gc *gc, libxl__datacopier_state *dc); +_hidden int libxl__datacopier_start(libxl__gc *gc, libxl__datacopier_state *dc); + + +/*----- bootloader -----*/ + +typedef struct libxl__bootloader_state libxl__bootloader_state; +typedef void libxl__run_bootloader_callback(libxl__egc*, + libxl__bootloader_state*, int rc); + +struct libxl__bootloader_state { + /* caller must fill these in, and they must all remain valid */ + libxl__ao *ao; + libxl__run_bootloader_callback *callback; + libxl_domain_build_info *info; /* u.pv.{kernel,ramdisk,cmdline} updated */ + libxl_device_disk *disk; + uint32_t domid; + /* private to libxl__run_bootloader */ + char *outputpath, *outputdir; + char *diskpath; /* not from gc, represents actually attached disk */ + libxl__openpty_state openpty; + libxl__openpty_result ptys[2]; /* [0] is for bootloader */ + libxl__ev_child child; + int nargs, argsspace; + const char **args; + libxl__datacopier_state keystrokes, display; + int rc; +}; + +_hidden void libxl__bootloader_init(libxl__bootloader_state *bl); + +/* Will definitely call st->callback, perhaps reentrantly. + * If callback is passed rc==0, will have updated st->info appropriately */ +_hidden void libxl__bootloader_run(libxl__egc*, libxl__bootloader_state *st); + + /* * Convenience macros. */ -- 1.7.2.5 _______________________________________________ Xen-devel mailing list Xen-devel@xxxxxxxxxxxxx http://lists.xen.org/xen-devel
|
Lists.xenproject.org is hosted with RackSpace, monitoring our |