Merge pull request #1618 from brauner/2017-06-06/kill_lxc_monitord

src/lxc/commands.c

@@ -43,7 +43,9 @@
 #include "commands.h"
 #include "console.h"
 #include "confile.h"
+#include "lxclock.h"
 #include "mainloop.h"
+#include "monitor.h"
 #include "af_unix.h"
 #include "config.h"
 
@@ -142,6 +144,7 @@ static const char *lxc_cmd_str(lxc_cmd_t cmd)
 		[LXC_CMD_GET_CONFIG_ITEM] = "get_config_item",
 		[LXC_CMD_GET_NAME]        = "get_name",
 		[LXC_CMD_GET_LXCPATH]     = "get_lxcpath",
+		[LXC_CMD_STATE_SERVER]    = "state_server",
 	};
 
 	if (cmd >= LXC_CMD_MAX)
@@ -283,7 +286,11 @@ static int lxc_cmd(const char *name, struct lxc_cmd_rr *cmd, int *stopped,
 	char path[sizeof(((struct sockaddr_un *)0)->sun_path)] = { 0 };
 	char *offset = &path[1];
 	size_t len;
-	int stay_connected = cmd->req.cmd == LXC_CMD_CONSOLE;
+	bool stay_connected = false;
+
+	if (cmd->req.cmd == LXC_CMD_CONSOLE ||
+	    cmd->req.cmd == LXC_CMD_STATE_SERVER)
+		stay_connected = true;
 
 	*stopped = 0;
 
@@ -297,12 +304,20 @@ static int lxc_cmd(const char *name, struct lxc_cmd_rr *cmd, int *stopped,
 		return -1;
 
 	sock = lxc_abstract_unix_connect(path);
+	TRACE("command %s tries to connect to \"@%s\"",
+	      lxc_cmd_str(cmd->req.cmd), offset);
 	if (sock < 0) {
-		if (errno == ECONNREFUSED)
+		if (errno == ECONNREFUSED) {
+			TRACE("command %s failed to connect to \"@%s\": %s",
+			      lxc_cmd_str(cmd->req.cmd), offset,
+			      strerror(errno));
 			*stopped = 1;
-		else
-			SYSERROR("Command %s failed to connect to \"@%s\".",
-				 lxc_cmd_str(cmd->req.cmd), offset);
+		} else {
+			SYSERROR("command %s failed to connect to \"@%s\": %s",
+				 lxc_cmd_str(cmd->req.cmd), offset,
+				 strerror(errno));
+		}
+
 		return -1;
 	}
 
@@ -462,20 +477,26 @@ char *lxc_cmd_get_cgroup_path(const char *name, const char *lxcpath,
 	};
 
 	ret = lxc_cmd(name, &cmd, &stopped, lxcpath, NULL);
-	if (ret < 0)
+	if (ret < 0) {
+		TRACE("command %s failed for container \"%s\": %s.",
+		      lxc_cmd_str(cmd.req.cmd), name, strerror(errno));
 		return NULL;
+	}
 
 	if (!ret) {
-		WARN("Container \"%s\" has stopped before sending its state.", name);
+		WARN("container \"%s\" has stopped before sending its state", name);
 		return NULL;
 	}
 
 	if (cmd.rsp.ret < 0 || cmd.rsp.datalen < 0) {
-		ERROR("Command %s failed for container \"%s\": %s.",
+		ERROR("command %s failed for container \"%s\": %s",
 		      lxc_cmd_str(cmd.req.cmd), name, strerror(-cmd.rsp.ret));
 		return NULL;
 	}
 
+	TRACE("command %s successful for container \"%s\"",
+	      lxc_cmd_str(cmd.req.cmd), name);
+
 	return cmd.rsp.data;
 }
 
@@ -567,7 +588,7 @@ out:
  *
  * Returns the state on success, < 0 on failure
  */
-lxc_state_t lxc_cmd_get_state(const char *name, const char *lxcpath)
+int lxc_cmd_get_state(const char *name, const char *lxcpath)
 {
 	int ret, stopped;
 	struct lxc_cmd_rr cmd = {
@@ -847,6 +868,145 @@ static int lxc_cmd_get_lxcpath_callback(int fd, struct lxc_cmd_req *req,
 	return lxc_cmd_rsp_send(fd, &rsp);
 }
 
+/*
+ * lxc_cmd_state_server: register a client fd in the handler list
+ *
+ * @name      : name of container to connect to
+ * @lxcpath   : the lxcpath in which the container is running
+ *
+ * Returns the lxcpath on success, NULL on failure.
+ */
+int lxc_cmd_state_server(const char *name, const char *lxcpath,
+			 lxc_state_t states[MAX_STATE])
+{
+	int stopped;
+	ssize_t ret;
+	int state = -1;
+	struct lxc_msg msg = {0};
+	struct lxc_cmd_rr cmd = {
+	    .req = {
+		.cmd     = LXC_CMD_STATE_SERVER,
+		.data    = states,
+		.datalen = (sizeof(lxc_state_t) * MAX_STATE)
+	    },
+	};
+
+	/* Lock the whole lxc_cmd_state_server_callback() call to ensure that
+	 * lxc_set_state() doesn't cause us to miss a state.
+	 */
+	process_lock();
+	/* Check if already in requested state. */
+	state = lxc_getstate(name, lxcpath);
+	if (state < 0) {
+		process_unlock();
+		TRACE("failed to retrieve state of container: %s",
+		      strerror(errno));
+		return -1;
+	} else if (states[state]) {
+		process_unlock();
+		TRACE("container is %s state", lxc_state2str(state));
+		return state;
+	}
+
+	if ((state == STARTING) && !states[RUNNING] && !states[STOPPING] && !states[STOPPED]) {
+		process_unlock();
+		TRACE("container is in %s state and caller requested to be "
+		      "informed about a previous state",
+		      lxc_state2str(state));
+		return state;
+	} else if ((state == RUNNING) && !states[STOPPING] && !states[STOPPED]) {
+		process_unlock();
+		TRACE("container is in %s state and caller requested to be "
+		      "informed about a previous state",
+		      lxc_state2str(state));
+		return state;
+	} else if ((state == STOPPING) && !states[STOPPED]) {
+		process_unlock();
+		TRACE("container is in %s state and caller requested to be "
+		      "informed about a previous state",
+		      lxc_state2str(state));
+		return state;
+	} else if ((state == STOPPED) || (state == ABORTING)) {
+		process_unlock();
+		TRACE("container is in %s state and caller requested to be "
+		      "informed about a previous state",
+		      lxc_state2str(state));
+		return state;
+	}
+
+	ret = lxc_cmd(name, &cmd, &stopped, lxcpath, NULL);
+	process_unlock();
+	if (ret < 0) {
+		ERROR("failed to execute command: %s", strerror(errno));
+		return -1;
+	}
+	/* We should now be guaranteed to get an answer from the state sending
+	 * function.
+	 */
+
+	if (cmd.rsp.ret < 0) {
+		ERROR("failed to receive socket fd");
+		return -1;
+	}
+
+again:
+	ret = recv(cmd.rsp.ret, &msg, sizeof(msg), 0);
+	if (ret < 0) {
+		if (errno == EINTR)
+			goto again;
+
+		ERROR("failed to receive message: %s", strerror(errno));
+		return -1;
+	}
+	if (ret == 0) {
+		ERROR("length of message was 0");
+		return -1;
+	}
+
+	TRACE("received state %s from state client %d",
+	      lxc_state2str(msg.value), cmd.rsp.ret);
+	return msg.value;
+}
+
+static int lxc_cmd_state_server_callback(int fd, struct lxc_cmd_req *req,
+					 struct lxc_handler *handler)
+{
+	struct lxc_cmd_rsp rsp = {0};
+	struct state_client *newclient;
+	struct lxc_list *tmplist;
+
+	if (req->datalen < 0) {
+		TRACE("requested datalen was < 0");
+		return -1;
+	}
+
+	if (!req->data) {
+		TRACE("no states requested");
+		return -1;
+	}
+
+	newclient = malloc(sizeof(*newclient));
+	if (!newclient)
+		return -1;
+
+	/* copy requested states */
+	memcpy(newclient->states, req->data, sizeof(newclient->states));
+	newclient->clientfd = fd;
+
+	tmplist = malloc(sizeof(*tmplist));
+	if (!tmplist) {
+		free(newclient);
+		return -1;
+	}
+
+	lxc_list_add_elem(tmplist, newclient);
+	lxc_list_add_tail(&handler->state_clients, tmplist);
+
+	TRACE("added state client %d to state client list", fd);
+
+	return lxc_cmd_rsp_send(fd, &rsp);
+}
+
 static int lxc_cmd_process(int fd, struct lxc_cmd_req *req,
 			   struct lxc_handler *handler)
 {
@@ -863,6 +1023,7 @@ static int lxc_cmd_process(int fd, struct lxc_cmd_req *req,
 		[LXC_CMD_GET_CONFIG_ITEM] = lxc_cmd_get_config_item_callback,
 		[LXC_CMD_GET_NAME]        = lxc_cmd_get_name_callback,
 		[LXC_CMD_GET_LXCPATH]     = lxc_cmd_get_lxcpath_callback,
+		[LXC_CMD_STATE_SERVER]    = lxc_cmd_state_server_callback,
 	};
 
 	if (req->cmd >= LXC_CMD_MAX) {

src/lxc/commands.h

@@ -43,6 +43,7 @@ typedef enum {
 	LXC_CMD_GET_CONFIG_ITEM,
 	LXC_CMD_GET_NAME,
 	LXC_CMD_GET_LXCPATH,
+	LXC_CMD_STATE_SERVER,
 	LXC_CMD_MAX,
 } lxc_cmd_t;
 
@@ -82,8 +83,10 @@ extern char *lxc_cmd_get_config_item(const char *name, const char *item, const c
 extern char *lxc_cmd_get_name(const char *hashed_sock);
 extern char *lxc_cmd_get_lxcpath(const char *hashed_sock);
 extern pid_t lxc_cmd_get_init_pid(const char *name, const char *lxcpath);
-extern lxc_state_t lxc_cmd_get_state(const char *name, const char *lxcpath);
+extern int lxc_cmd_get_state(const char *name, const char *lxcpath);
 extern int lxc_cmd_stop(const char *name, const char *lxcpath);
+extern int lxc_cmd_state_server(const char *name, const char *lxcpath,
+				lxc_state_t states[MAX_STATE]);
 
 struct lxc_epoll_descr;
 struct lxc_handler;

src/lxc/conf.c

@@ -3771,12 +3771,16 @@ int chown_mapped_root(char *path, struct lxc_conf *conf)
 		return 0;
 	}
 
-	// save the current gid of "path"
+	/* save the current gid of "path" */
 	if (stat(path, &sb) < 0) {
 		ERROR("Error stat %s", path);
 		return -1;
 	}
 
+	/* Update the path argument in case this was overlayfs. */
+	args1[sizeof(args1) / sizeof(args1[0]) - 2] = path;
+	args2[sizeof(args2) / sizeof(args2[0]) - 2] = path;
+
 	/*
 	 * A file has to be group-owned by a gid mapped into the
 	 * container, or the container won't be privileged over it.

src/lxc/criu.c

@@ -797,10 +797,13 @@ static void do_restore(struct lxc_container *c, int status_pipe, struct migrate_
 		close(fd);
 	}
 
-	handler = lxc_init(c->name, c->lxc_conf, c->config_path);
+	handler = lxc_init_handler(c->name, c->lxc_conf, c->config_path);
 	if (!handler)
 		goto out;
 
+	if (lxc_init(c->name, handler) < 0)
+		goto out;
+
 	if (!cgroup_init(handler)) {
 		ERROR("failed initing cgroups");
 		goto out_fini_handler;

src/lxc/execute.c

@@ -111,16 +111,15 @@ static struct lxc_operations execute_start_ops = {
 };
 
 int lxc_execute(const char *name, char *const argv[], int quiet,
-		struct lxc_conf *conf, const char *lxcpath, bool backgrounded)
+		struct lxc_handler *handler, const char *lxcpath,
+		bool backgrounded)
 {
-	struct execute_args args = {
-		.argv = argv,
-		.quiet = quiet
-	};
+	struct execute_args args = {.argv = argv, .quiet = quiet};
 
-	if (lxc_check_inherited(conf, false, -1))
+	if (lxc_check_inherited(handler->conf, false, handler->conf->maincmd_fd))
 		return -1;
 
-	conf->is_execute = 1;
-	return __lxc_start(name, conf, &execute_start_ops, &args, lxcpath, backgrounded);
+	handler->conf->is_execute = 1;
+	return __lxc_start(name, handler, &execute_start_ops, &args, lxcpath,
+			   backgrounded);
 }

src/lxc/lxc.h

@@ -36,6 +36,7 @@ extern "C" {
 struct lxc_msg;
 struct lxc_conf;
 struct lxc_arguments;
+struct lxc_handler;
 
 /**
  Following code is for liblxc.
@@ -51,8 +52,9 @@ struct lxc_arguments;
  * @backgrounded : whether or not the container is daemonized
  * Returns 0 on success, < 0 otherwise
  */
-extern int lxc_start(const char *name, char *const argv[], struct lxc_conf *conf,
-		     const char *lxcpath, bool backgrounded);
+extern int lxc_start(const char *name, char *const argv[],
+		     struct lxc_handler *handler, const char *lxcpath,
+		     bool backgrounded);
 
 /*
  * Start the specified command inside an application container
@@ -64,7 +66,7 @@ extern int lxc_start(const char *name, char *const argv[], struct lxc_conf *conf
  * Returns 0 on success, < 0 otherwise
  */
 extern int lxc_execute(const char *name, char *const argv[], int quiet,
-		       struct lxc_conf *conf, const char *lxcpath,
+		       struct lxc_handler *handler, const char *lxcpath,
 		       bool backgrounded);
 
 /*

src/lxc/lxccontainer.c

@@ -57,6 +57,7 @@
 #include "namespace.h"
 #include "network.h"
 #include "sync.h"
+#include "start.h"
 #include "state.h"
 #include "utils.h"
 #include "version.h"
@@ -715,6 +716,7 @@ static void free_init_cmd(char **argv)
 static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const argv[])
 {
 	int ret;
+	struct lxc_handler *handler;
 	struct lxc_conf *conf;
 	bool daemonize = false;
 	FILE *pid_fp = NULL;
@@ -731,21 +733,21 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 	/* If anything fails before we set error_num, we want an error in there */
 	c->error_num = 1;
 
-	/* container has been setup */
+	/* container has not been setup */
 	if (!c->lxc_conf)
 		return false;
 
-	if ((ret = ongoing_create(c)) < 0) {
+	ret = ongoing_create(c);
+	if (ret < 0) {
 		ERROR("Error checking for incomplete creation");
 		return false;
-	}
-	if (ret == 2) {
-		ERROR("Error: %s creation was not completed", c->name);
-		do_lxcapi_destroy(c);
-		return false;
 	} else if (ret == 1) {
 		ERROR("Error: creation of %s is ongoing", c->name);
 		return false;
+	} else if (ret == 2) {
+		ERROR("Error: %s creation was not completed", c->name);
+		do_lxcapi_destroy(c);
+		return false;
 	}
 
 	/* is this app meant to be run through lxcinit, as in lxc-execute? */
@@ -756,10 +758,18 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 		return false;
 	conf = c->lxc_conf;
 	daemonize = c->daemonize;
+
+	/* initialize handler */
+	handler = lxc_init_handler(c->name, conf, c->config_path);
 	container_mem_unlock(c);
+	if (!handler)
+		return false;
 
 	if (useinit) {
-		ret = lxc_execute(c->name, argv, 1, conf, c->config_path, daemonize);
+		TRACE("calling \"lxc_execute\"");
+		ret = lxc_execute(c->name, argv, 1, handler, c->config_path,
+				  daemonize);
+		c->error_num = ret;
 		return ret == 0 ? true : false;
 	}
 
@@ -780,17 +790,21 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 	*/
 	if (daemonize) {
 		char title[2048];
-		lxc_monitord_spawn(c->config_path);
+		pid_t pid;
 
-		pid_t pid = fork();
-		if (pid < 0)
+		pid = fork();
+		if (pid < 0) {
+			free_init_cmd(init_cmd);
+			lxc_free_handler(handler);
 			return false;
+		}
 
 		if (pid != 0) {
 			/* Set to NULL because we don't want father unlink
 			 * the PID file, child will do the free and unlink.
 			 */
 			c->pidfile = NULL;
+			close(handler->conf->maincmd_fd);
 			return wait_on_daemonized_start(c, pid);
 		}
 
@@ -815,7 +829,7 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 			SYSERROR("Error chdir()ing to /.");
 			exit(1);
 		}
-		lxc_check_inherited(conf, true, -1);
+		lxc_check_inherited(conf, true, handler->conf->maincmd_fd);
 		if (null_stdfds() < 0) {
 			ERROR("failed to close fds");
 			exit(1);
@@ -824,11 +838,12 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 	} else {
 		if (!am_single_threaded()) {
 			ERROR("Cannot start non-daemonized container when threaded");
+			lxc_free_handler(handler);
 			return false;
 		}
 	}
 
-	/* We need to write PID file after daeminize, so we always
+	/* We need to write PID file after daemonize, so we always
 	 * write the right PID.
 	 */
 	if (c->pidfile) {
@@ -836,6 +851,8 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 		if (pid_fp == NULL) {
 			SYSERROR("Failed to create pidfile '%s' for '%s'",
 				 c->pidfile, c->name);
+			free_init_cmd(init_cmd);
+			lxc_free_handler(handler);
 			if (daemonize)
 				exit(1);
 			return false;
@@ -845,6 +862,8 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 			SYSERROR("Failed to write '%s'", c->pidfile);
 			fclose(pid_fp);
 			pid_fp = NULL;
+			free_init_cmd(init_cmd);
+			lxc_free_handler(handler);
 			if (daemonize)
 				exit(1);
 			return false;
@@ -860,22 +879,34 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 	if (conf->monitor_unshare) {
 		if (unshare(CLONE_NEWNS)) {
 			SYSERROR("failed to unshare mount namespace");
+			free_init_cmd(init_cmd);
+			lxc_free_handler(handler);
 			return false;
 		}
 		if (mount(NULL, "/", NULL, MS_SLAVE|MS_REC, NULL)) {
 			SYSERROR("Failed to make / rslave at startup");
+			free_init_cmd(init_cmd);
+			lxc_free_handler(handler);
 			return false;
 		}
 	}
 
 reboot:
-	if (lxc_check_inherited(conf, daemonize, -1)) {
+	if (conf->reboot == 2) {
+		/* initialize handler */
+		handler = lxc_init_handler(c->name, conf, c->config_path);
+		if (!handler)
+			goto out;
+	}
+
+	if (lxc_check_inherited(conf, daemonize, handler->conf->maincmd_fd)) {
 		ERROR("Inherited fds found");
+		lxc_free_handler(handler);
 		ret = 1;
 		goto out;
 	}
 
-	ret = lxc_start(c->name, argv, conf, c->config_path, daemonize);
+	ret = lxc_start(c->name, argv, handler, c->config_path, daemonize);
 	c->error_num = ret;
 
 	if (conf->reboot == 1) {
@@ -890,13 +921,11 @@ out:
 		free(c->pidfile);
 		c->pidfile = NULL;
 	}
-
 	free_init_cmd(init_cmd);
 
 	if (daemonize)
-		exit (ret == 0 ? true : false);
-	else
-		return (ret == 0 ? true : false);
+		exit(ret == 0 ? true : false);
+	return (ret == 0 ? true : false);
 }
 
 static bool lxcapi_start(struct lxc_container *c, int useinit, char * const argv[])

src/lxc/start.c

@@ -341,10 +341,64 @@ static int signal_handler(int fd, uint32_t events, void *data,
 	return 1;
 }
 
-int lxc_set_state(const char *name, struct lxc_handler *handler, lxc_state_t state)
+int lxc_set_state(const char *name, struct lxc_handler *handler,
+		  lxc_state_t state)
 {
+	ssize_t ret;
+	struct lxc_list *cur, *next;
+	struct state_client *client;
+	struct lxc_msg msg = {.type = lxc_msg_state, .value = state};
+
+	process_lock();
+	/* Only set state under process lock held so that we don't cause
+	 * lxc_cmd_state_server() to miss a state.
+	 */
 	handler->state = state;
+	TRACE("set container state to %s", lxc_state2str(state));
+
+	if (lxc_list_empty(&handler->state_clients)) {
+		TRACE("no state clients registered");
+		process_unlock();
+		return 0;
+	}
+
+	strncpy(msg.name, name, sizeof(msg.name));
+	msg.name[sizeof(msg.name) - 1] = 0;
+
+	lxc_list_for_each_safe(cur, &handler->state_clients, next) {
+		client = cur->elem;
+
+		if (!client->states[state]) {
+			TRACE("state %s not registered for state client %d",
+			      lxc_state2str(state), client->clientfd);
+			continue;
+		}
+
+		TRACE("sending state %s to state client %d",
+		      lxc_state2str(state), client->clientfd);
+
+	again:
+		ret = send(client->clientfd, &msg, sizeof(msg), 0);
+		if (ret < 0) {
+			if (errno == EINTR)
+				goto again;
+
+			ERROR("failed to send message to client");
+		}
+
+		/* kick client from list */
+		close(client->clientfd);
+		lxc_list_del(cur);
+		free(cur->elem);
+		free(cur);
+	}
+	process_unlock();
+
+	/* This function will try to connect to the legacy lxc-monitord state
+	 * server and only exists for backwards compatibility.
+	 */
 	lxc_monitor_send_state(name, state, handler->lxcpath);
+
 	return 0;
 }
 
@@ -384,6 +438,7 @@ int lxc_poll(const char *name, struct lxc_handler *handler)
 			DEBUG("Not starting utmp handler as CAP_SYS_BOOT cannot be dropped without capabilities support.");
 		#endif
 	}
+	TRACE("lxc mainloop is ready");
 
 	return lxc_mainloop(&descr, -1);
 
@@ -396,14 +451,29 @@ out_sigfd:
 	return -1;
 }
 
-struct lxc_handler *lxc_init(const char *name, struct lxc_conf *conf, const char *lxcpath)
+void lxc_free_handler(struct lxc_handler *handler)
+{
+	if (handler->conf && handler->conf->maincmd_fd)
+		close(handler->conf->maincmd_fd);
+
+	if (handler->name)
+		free(handler->name);
+
+	handler->conf = NULL;
+	free(handler);
+}
+
+struct lxc_handler *lxc_init_handler(const char *name, struct lxc_conf *conf,
+				     const char *lxcpath)
 {
 	int i;
 	struct lxc_handler *handler;
 
 	handler = malloc(sizeof(*handler));
-	if (!handler)
+	if (!handler) {
+		ERROR("failed to allocate memory");
 		return NULL;
+	}
 
 	memset(handler, 0, sizeof(*handler));
 
@@ -411,31 +481,52 @@ struct lxc_handler *lxc_init(const char *name, struct lxc_conf *conf, const char
 	handler->conf = conf;
 	handler->lxcpath = lxcpath;
 	handler->pinfd = -1;
+	lxc_list_init(&handler->state_clients);
 
 	for (i = 0; i < LXC_NS_MAX; i++)
 		handler->nsfd[i] = -1;
 
-	lsm_init();
-
 	handler->name = strdup(name);
 	if (!handler->name) {
-		ERROR("Failed to allocate memory.");
-		goto out_free;
+		ERROR("failed to allocate memory");
+		goto on_error;
 	}
 
-	if (lxc_cmd_init(name, handler, lxcpath))
-		goto out_free_name;
+	if (lxc_cmd_init(name, handler, lxcpath)) {
+		ERROR("failed to set up command socket");
+		goto on_error;
+	}
+
+	TRACE("unix domain socket %d for command server is ready",
+	      handler->conf->maincmd_fd);
+
+	return handler;
+
+on_error:
+	lxc_free_handler(handler);
+
+	return NULL;
+}
+
+int lxc_init(const char *name, struct lxc_handler *handler)
+{
+	struct lxc_conf *conf = handler->conf;
+
+	lsm_init();
+	TRACE("initialized LSM");
 
 	if (lxc_read_seccomp_config(conf) != 0) {
 		ERROR("Failed loading seccomp policy.");
 		goto out_close_maincmd_fd;
 	}
+	TRACE("read seccomp policy");
 
 	/* Begin by setting the state to STARTING. */
 	if (lxc_set_state(name, handler, STARTING)) {
 		ERROR("Failed to set state for container \"%s\" to \"%s\".", name, lxc_state2str(STARTING));
 		goto out_close_maincmd_fd;
 	}
+	TRACE("set container state to \"STARTING\"");
 
 	/* Start of environment variable setup for hooks. */
 	if (name && setenv("LXC_NAME", name, 1))
@@ -460,10 +551,13 @@ struct lxc_handler *lxc_init(const char *name, struct lxc_conf *conf, const char
 		SYSERROR("Failed to set environment variable LXC_CGNS_AWARE=1.");
 	/* End of environment variable setup for hooks. */
 
+	TRACE("set environment variables");
+
 	if (run_lxc_hooks(name, "pre-start", conf, handler->lxcpath, NULL)) {
 		ERROR("Failed to run lxc.hook.pre-start for container \"%s\".", name);
 		goto out_aborting;
 	}
+	TRACE("ran pre-start hooks");
 
 	/* The signal fd has to be created before forking otherwise if the child
 	 * process exits before we setup the signal fd, the event will be lost
@@ -474,20 +568,23 @@ struct lxc_handler *lxc_init(const char *name, struct lxc_conf *conf, const char
 		ERROR("Failed to setup SIGCHLD fd handler.");
 		goto out_delete_tty;
 	}
+	TRACE("set up signal fd");
 
 	/* Do this after setting up signals since it might unblock SIGWINCH. */
 	if (lxc_console_create(conf)) {
 		ERROR("Failed to create console for container \"%s\".", name);
 		goto out_restore_sigmask;
 	}
+	TRACE("created console");
 
 	if (lxc_ttys_shift_ids(conf) < 0) {
 		ERROR("Failed to shift tty into container.");
 		goto out_restore_sigmask;
 	}
+	TRACE("shifted tty ids");
 
-	INFO("Container \"%s\" is initialized.", name);
-	return handler;
+	INFO("container \"%s\" is initialized", name);
+	return 0;
 
 out_restore_sigmask:
 	sigprocmask(SIG_SETMASK, &handler->oldmask, NULL);
@@ -498,19 +595,15 @@ out_aborting:
 out_close_maincmd_fd:
 	close(conf->maincmd_fd);
 	conf->maincmd_fd = -1;
-out_free_name:
-	free(handler->name);
-	handler->name = NULL;
-out_free:
-	free(handler);
-	return NULL;
+	return -1;
 }
 
 void lxc_fini(const char *name, struct lxc_handler *handler)
 {
 	int i, rc;
+	struct lxc_list *cur, *next;
 	pid_t self = getpid();
-	char *namespaces[LXC_NS_MAX+1];
+	char *namespaces[LXC_NS_MAX + 1];
 	size_t namespace_count = 0;
 
 	/* The STOPPING state is there for future cleanup code which can take
@@ -572,8 +665,23 @@ void lxc_fini(const char *name, struct lxc_handler *handler)
 
 	lxc_console_delete(&handler->conf->console);
 	lxc_delete_tty(&handler->conf->tty_info);
+
+	/* close the command socket */
 	close(handler->conf->maincmd_fd);
 	handler->conf->maincmd_fd = -1;
+
+	/* The command socket is now closed, no more state clients can register
+	 * themselves from now on. So free the list of state clients.
+	 */
+	lxc_list_for_each_safe(cur, &handler->state_clients, next) {
+		struct state_client *client = cur->elem;
+		/* close state client socket */
+		close(client->clientfd);
+		lxc_list_del(cur);
+		free(cur->elem);
+		free(cur);
+	}
+
 	free(handler->name);
 	if (handler->ttysock[0] != -1) {
 		close(handler->ttysock[0]);
@@ -1337,17 +1445,16 @@ out_abort:
 	return -1;
 }
 
-int __lxc_start(const char *name, struct lxc_conf *conf,
+int __lxc_start(const char *name, struct lxc_handler *handler,
 		struct lxc_operations* ops, void *data, const char *lxcpath,
 		bool backgrounded)
 {
-	struct lxc_handler *handler;
-	int err = -1;
 	int status;
+	int err = -1;
 	bool removed_all_netdevs = true;
+	struct lxc_conf *conf = handler->conf;
 
-	handler = lxc_init(name, conf, lxcpath);
-	if (!handler) {
+	if (lxc_init(name, handler) < 0) {
 		ERROR("Failed to initialize container \"%s\".", name);
 		return -1;
 	}
@@ -1494,15 +1601,15 @@ static struct lxc_operations start_ops = {
 	.post_start = post_start
 };
 
-int lxc_start(const char *name, char *const argv[], struct lxc_conf *conf,
+int lxc_start(const char *name, char *const argv[], struct lxc_handler *handler,
 	      const char *lxcpath, bool backgrounded)
 {
 	struct start_args start_arg = {
 		.argv = argv,
 	};
 
-	conf->need_utmp_watch = 1;
-	return __lxc_start(name, conf, &start_ops, &start_arg, lxcpath, backgrounded);
+	handler->conf->need_utmp_watch = 1;
+	return __lxc_start(name, handler, &start_ops, &start_arg, lxcpath, backgrounded);
 }
 
 static void lxc_destroy_container_on_signal(struct lxc_handler *handler,

src/lxc/start.h

@@ -27,21 +27,11 @@
 #include <sys/param.h>
 #include <stdbool.h>
 
+#include "conf.h"
 #include "config.h"
 #include "state.h"
 #include "namespace.h"
 
-struct lxc_conf;
-
-struct lxc_handler;
-
-struct lxc_operations {
-	int (*start)(struct lxc_handler *, void *);
-	int (*post_start)(struct lxc_handler *, void *);
-};
-
-struct cgroup_desc;
-
 struct lxc_handler {
 	pid_t pid;
 	char *name;
@@ -60,17 +50,31 @@ struct lxc_handler {
 	bool backgrounded; // indicates whether should we close std{in,out,err} on start
 	int nsfd[LXC_NS_MAX];
 	int netnsfd;
+	struct lxc_list state_clients;
 };
 
+struct lxc_operations {
+	int (*start)(struct lxc_handler *, void *);
+	int (*post_start)(struct lxc_handler *, void *);
+};
+
+struct state_client {
+	int clientfd;
+	lxc_state_t states[MAX_STATE];
+};
 
 extern int lxc_poll(const char *name, struct lxc_handler *handler);
 extern int lxc_set_state(const char *name, struct lxc_handler *handler, lxc_state_t state);
 extern void lxc_abort(const char *name, struct lxc_handler *handler);
-extern struct lxc_handler *lxc_init(const char *name, struct lxc_conf *, const char *);
+extern struct lxc_handler *lxc_init_handler(const char *name,
+					    struct lxc_conf *conf,
+					    const char *lxcpath);
+extern void lxc_free_handler(struct lxc_handler *handler);
+extern int lxc_init(const char *name, struct lxc_handler *handler);
 extern void lxc_fini(const char *name, struct lxc_handler *handler);
 
 extern int lxc_check_inherited(struct lxc_conf *conf, bool closeall, int fd_to_ignore);
-int __lxc_start(const char *, struct lxc_conf *, struct lxc_operations *,
+int __lxc_start(const char *, struct lxc_handler *, struct lxc_operations *,
 		void *, const char *, bool);
 
 extern void resolve_clone_flags(struct lxc_handler *handler);

src/lxc/state.c

@@ -79,7 +79,7 @@ lxc_state_t lxc_getstate(const char *name, const char *lxcpath)
 	return state;
 }
 
-static int fillwaitedstates(const char *strstates, int *states)
+static int fillwaitedstates(const char *strstates, lxc_state_t *states)
 {
 	char *token, *saveptr = NULL;
 	char *strstates_dup = strdup(strstates);
@@ -108,90 +108,21 @@ static int fillwaitedstates(const char *strstates, int *states)
 extern int lxc_wait(const char *lxcname, const char *states, int timeout,
 		    const char *lxcpath)
 {
-	struct lxc_msg msg;
-	int state, ret;
-	int s[MAX_STATE] = {0}, fd;
+	int state;
+	lxc_state_t s[MAX_STATE] = {0};
 
 	if (fillwaitedstates(states, s))
 		return -1;
 
-	if (lxc_monitord_spawn(lxcpath))
-		return -1;
-
-	fd = lxc_monitor_open(lxcpath);
-	if (fd < 0)
-		return -1;
-
-	/*
-	 * if container present,
-	 * then check if already in requested state
-	 */
-	ret = -1;
-	state = lxc_getstate(lxcname, lxcpath);
+	state = lxc_cmd_state_server(lxcname, lxcpath, s);
 	if (state < 0) {
-		goto out_close;
-	} else if ((state >= 0) && (s[state])) {
-		ret = 0;
-		goto out_close;
+		SYSERROR("failed to receive state from monitor");
+		return -1;
 	}
 
-	for (;;) {
-		int64_t elapsed_time, curtime = 0;
-		struct timespec tspec;
-		int stop = 0;
-		int retval;
-
-		if (timeout != -1) {
-			retval = clock_gettime(CLOCK_REALTIME, &tspec);
-			if (retval)
-				goto out_close;
-			curtime = tspec.tv_sec;
-		}
-		if (lxc_monitor_read_timeout(fd, &msg, timeout) < 0) {
-			/* try again if select interrupted by signal */
-			if (errno != EINTR)
-				goto out_close;
-		}
-
-		if (timeout != -1) {
-			retval = clock_gettime(CLOCK_REALTIME, &tspec);
-			if (retval)
-				goto out_close;
-			elapsed_time = tspec.tv_sec - curtime;
-			if (timeout - elapsed_time <= 0)
-				stop = 1;
-			timeout -= elapsed_time;
-		}
-
-		if (strcmp(lxcname, msg.name)) {
-			if (stop) {
-				ret = -2;
-				goto out_close;
-			}
-			continue;
-		}
-
-		switch (msg.type) {
-		case lxc_msg_state:
-			if (msg.value < 0 || msg.value >= MAX_STATE)
-				goto out_close;
-
-			if (s[msg.value]) {
-				ret = 0;
-				goto out_close;
-			}
-			break;
-		default:
-			if (stop) {
-				ret = -2;
-				goto out_close;
-			}
-			/* just ignore garbage */
-			break;
-		}
-	}
+	TRACE("retrieved state of container %s", lxc_state2str(state));
+	if (!s[state])
+		return -1;
 
-out_close:
-	lxc_monitor_close(fd);
-	return ret;
+	return 0;
 }

src/lxc/state.h

@@ -24,8 +24,15 @@
 #define __LXC_STATE_H
 
 typedef enum {
-	STOPPED, STARTING, RUNNING, STOPPING,
-	ABORTING, FREEZING, FROZEN, THAWED, MAX_STATE,
+	STOPPED,
+	STARTING,
+	RUNNING,
+	STOPPING,
+	ABORTING,
+	FREEZING,
+	FROZEN,
+	THAWED,
+	MAX_STATE,
 } lxc_state_t;
 
 extern int lxc_rmstate(const char *name);

src/lxc/tools/lxc_execute.c

@@ -27,6 +27,7 @@
 #include <libgen.h>
 #include <string.h>
 #include <unistd.h>
+#include <lxc/lxccontainer.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/param.h>
@@ -40,7 +41,6 @@
 #include "config.h"
 #include "start.h"
 #include "utils.h"
-#include "lxccontainer.h"
 
 lxc_log_define(lxc_execute_ui, lxc);
 
@@ -105,10 +105,10 @@ Options :\n\
 
 int main(int argc, char *argv[])
 {
-	char *rcfile;
-	struct lxc_conf *conf;
+	struct lxc_container *c;
 	struct lxc_log log;
 	int ret;
+	bool bret;
 
 	lxc_list_init(&defines);
 
@@ -129,50 +129,38 @@ int main(int argc, char *argv[])
 		exit(EXIT_FAILURE);
 	lxc_log_options_no_override();
 
-	/* rcfile is specified in the cli option */
-	if (my_args.rcfile)
-		rcfile = (char *)my_args.rcfile;
-	else {
-		int rc;
+	c = lxc_container_new(my_args.name, my_args.lxcpath[0]);
+	if (!c) {
+		ERROR("Failed to create lxc_container");
+		exit(EXIT_FAILURE);
+	}
 
-		rc = asprintf(&rcfile, "%s/%s/config", my_args.lxcpath[0], my_args.name);
-		if (rc == -1) {
-			SYSERROR("failed to allocate memory");
+	if (my_args.rcfile) {
+		c->clear_config(c);
+		if (!c->load_config(c, my_args.rcfile)) {
+			ERROR("Failed to load rcfile");
+			lxc_container_put(c);
 			exit(EXIT_FAILURE);
 		}
-
-		/* container configuration does not exist */
-		if (access(rcfile, F_OK)) {
-			free(rcfile);
-			rcfile = NULL;
+		c->configfile = strdup(my_args.rcfile);
+		if (!c->configfile) {
+			ERROR("Out of memory setting new config filename");
+			lxc_container_put(c);
+			exit(EXIT_FAILURE);
 		}
 	}
 
-	conf = lxc_conf_init();
-	if (!conf) {
-		ERROR("failed to initialize configuration");
-		exit(EXIT_FAILURE);
-	}
-
-	if (rcfile && lxc_config_read(rcfile, conf, NULL)) {
-		ERROR("failed to read configuration file");
-		exit(EXIT_FAILURE);
-	}
-
-	if (lxc_config_define_load(&defines, conf))
-		exit(EXIT_FAILURE);
-
 	if (my_args.uid)
-		conf->init_uid = my_args.uid;
+		c->lxc_conf->init_uid = my_args.uid;
 
 	if (my_args.gid)
-		conf->init_gid = my_args.gid;
-
-	ret = lxc_execute(my_args.name, my_args.argv, my_args.quiet, conf, my_args.lxcpath[0], false);
-
-	lxc_conf_free(conf);
+		c->lxc_conf->init_gid = my_args.gid;
 
-	if (ret < 0)
+	c->daemonize = false;
+	bret = c->start(c, 1, my_args.argv);
+	ret = c->error_num;
+	lxc_container_put(c);
+	if (!bret)
 		exit(EXIT_FAILURE);
 	exit(ret);
 }

src/tests/lxc-test-apparmor-mount

@@ -50,7 +50,7 @@ cleanup() {
 	run_cmd lxc-destroy -f -n $cname || true
 	umount -l $MOUNTSR || true
 	rmdir $dnam || true
-	pkill -u $(id -u $TUSER) -9
+	pkill -u $(id -u $TUSER) -9 || true
 	sed -i '/lxcunpriv/d' /run/lxc/nics /etc/lxc/lxc-usernet
 	sed -i '/^lxcunpriv:/d' /etc/subuid /etc/subgid
 	rm -Rf $HDIR /run/user/$(id -u $TUSER)

src/tests/lxc-test-unpriv

@@ -71,7 +71,7 @@ cleanup() {
 
 	run_cmd lxc-stop -n c2 -k || true
 	run_cmd lxc-stop -n c1 -k || true
-	pkill -u $(id -u $TUSER) -9
+	pkill -u $(id -u $TUSER) -9 || true
 
 	sed -i '/lxcunpriv/d' /run/lxc/nics /etc/lxc/lxc-usernet
 	sed -i '/^lxcunpriv:/d' /etc/subuid /etc/subgid