criu: Add exec-cmd option (v3)

The --exec-cmd option specifies a command that will be execvp()-ed on successful
restore. This way the command specified here will become the parent process of
the restored process tree.

Waiting for the restored processes to finish is responsibility of this command.

All service FDs are closed before we call execvp(). Standad output and error of
the command are redirected to the log file when we are restoring through the RPC
service.

This option will be used when restoring LinuX Containers and it seems helpful
for perf or other use cases when restored processes must be supervised by a
parent.

Two directions were researched in order to integrate CRIU and LXC:

1. We tell to CRIU, that after restoring container is should execve()
   lxc properly explaining to it that there's a new container hanging
   around.

2. We make LXC set himself as child subreaper, then fork() criu and ask
   it to detach (-d) from restore container afterwards. Being a subreaper,
   it should get the container's init into his child list after it.

The main reason for choosing the first option is that the second one can't work
with the RPC service. If we call restore via the service then criu service will
be the top-most task in the hierarchy and will not be able to reparent the
restore trees to any other task in the system. Calling execve from service
worker sub-task (and daemonizing it) should solve this.
Signed-off-by: Deyan Doychev <deyandoichev@gmail.com>
Signed-off-by: Pavel Emelyanov <xemul@parallels.com>

cr-restore.c

@@ -1552,7 +1552,7 @@ static int restore_root_task(struct pstree_item *init)
 
 	write_stats(RESTORE_STATS);
 
-	if (!opts.restore_detach)
+	if (!opts.restore_detach && !opts.exec_cmd)
 		wait(NULL);
 
 	return 0;

cr-service.c

@@ -265,6 +265,12 @@ static int setup_opts_from_req(int sk, CriuOpts *req)
 	if (req->has_force_irmap)
 		opts.force_irmap = req->force_irmap;
 
+	if (req->n_exec_cmd > 0) {
+		opts.exec_cmd = xmalloc((req->n_exec_cmd + 1) * sizeof(char *));
+		memcpy(opts.exec_cmd, req->exec_cmd, req->n_exec_cmd * sizeof(char *));
+		opts.exec_cmd[req->n_exec_cmd] = NULL;
+	}
+
 	if (req->ps) {
 		opts.use_page_server = true;
 		opts.addr = req->ps->address;
@@ -349,6 +355,23 @@ exit:
 		success = false;
 	}
 
+	if (success && opts.exec_cmd) {
+		int logfd;
+
+		logfd = log_get_fd();
+		if (dup2(logfd, STDOUT_FILENO) == -1 || dup2(logfd, STDERR_FILENO) == -1) {
+			pr_perror("Failed to redirect stdout and stderr to the logfile");
+			return 1;
+		}
+
+		close_pid_proc();
+		close(sk);
+
+		execvp(opts.exec_cmd[0], opts.exec_cmd);
+		pr_perror("Failed to exec cmd %s", opts.exec_cmd[0]);
+		success = false;
+	}
+
 	return success ? 0 : 1;
 }
 

crtools.c

@@ -119,6 +119,7 @@ int main(int argc, char *argv[])
 	pid_t pid = 0, tree_id = 0;
 	int ret = -1;
 	bool usage_error = true;
+	bool has_exec_cmd = false;
 	int opt, idx;
 	int log_level = LOG_UNSET;
 	char *imgs_dir = ".";
@@ -162,6 +163,7 @@ int main(int argc, char *argv[])
 		{ "libdir", required_argument, 0, 'L'},
 		{ "cpu-cap", required_argument, 0, 57},
 		{ "force-irmap", no_argument, 0, 58},
+		{ "exec-cmd", no_argument, 0, 59},
 		{ },
 	};
 
@@ -333,6 +335,9 @@ int main(int argc, char *argv[])
 		case 'L':
 			opts.libdir = optarg;
 			break;
+		case 59:
+			has_exec_cmd = true;
+			break;
 		case 'V':
 			pr_msg("Version: %s\n", CRIU_VERSION);
 			if (strcmp(CRIU_GITID, "0"))
@@ -354,6 +359,27 @@ int main(int argc, char *argv[])
 		goto usage;
 	}
 
+	if (has_exec_cmd) {
+		if (argc - optind <= 1) {
+			pr_msg("Error: --exec-cmd requires a command\n");
+			goto usage;
+		}
+
+		if (strcmp(argv[optind], "restore")) {
+			pr_msg("Error: --exec-cmd is available for the restore command only\n");
+			goto usage;
+		}
+
+		if (opts.restore_detach) {
+			pr_msg("Error: --restore-detached and --exec-cmd cannot be used together\n");
+			goto usage;
+		}
+
+		opts.exec_cmd = xmalloc((argc - optind) * sizeof(char *));
+		memcpy(opts.exec_cmd, &argv[optind + 1], (argc - optind - 1) * sizeof(char *));
+		opts.exec_cmd[argc - optind - 1] = NULL;
+	}
+
 	/* We must not open imgs dir, if service is called */
 	if (strcmp(argv[optind], "service")) {
 		ret = open_image_dir(imgs_dir);
@@ -390,7 +416,16 @@ int main(int argc, char *argv[])
 	if (!strcmp(argv[optind], "restore")) {
 		if (tree_id)
 			pr_warn("Using -t with criu restore is obsoleted\n");
-		return cr_restore_tasks() != 0;
+
+		ret = cr_restore_tasks();
+		if (ret == 0 && opts.exec_cmd) {
+			close_pid_proc();
+			execvp(opts.exec_cmd[0], opts.exec_cmd);
+			pr_perror("Failed to exec command %s", opts.exec_cmd[0]);
+			ret = 1;
+		}
+
+		return ret != 0;
 	}
 
 	if (!strcmp(argv[optind], "show"))
@@ -460,6 +495,8 @@ usage:
 "                        (if not specified, value of --images-dir is used)\n"
 "     --cpu-cap CAP      require certain cpu capability. CAP: may be one of:\n"
 "                        'fpu','all'. To disable capability, prefix it with '^'.\n"
+"     --exec-cmd         execute the command specified after '--' on successful\n"
+"                        restore making it the parent of the restored process\n"
 "\n"
 "* Special resources support:\n"
 "  -x|--" USK_EXT_PARAM "      allow external unix connections\n"

include/cr_options.h

@@ -48,6 +48,7 @@ struct cr_options {
 	bool			auto_dedup;
 	unsigned int		cpu_cap;
 	bool			force_irmap;
+	char			**exec_cmd;
 };
 
 extern struct cr_options opts;

lib/criu.c

@@ -115,6 +115,15 @@ void criu_set_cpu_cap(unsigned int cap)
 	opts->cpu_cap		= cap;
 }
 
+void criu_set_exec_cmd(int argc, char *argv[])
+{
+	int i;
+	opts->n_exec_cmd = argc;
+	opts->exec_cmd = malloc((argc) * sizeof(char *));
+	for (i = 0; i < argc; i++)
+		opts->exec_cmd[i] = strdup(argv[i]);
+}
+
 static CriuResp *recv_resp(int socket_fd)
 {
 	unsigned char buf[CR_MAX_MSG_SIZE];

lib/criu.h

@@ -41,6 +41,7 @@ void criu_set_file_locks(bool file_locks);
 void criu_set_log_level(int log_level);
 void criu_set_log_file(char *log_file);
 void criu_set_cpu_cap(unsigned int cap);
+void criu_set_exec_cmd(int argc, char *argv[]);
 
 /* Here is a table of return values and errno's of functions
  * from the list down below.

net.c

@@ -573,7 +573,7 @@ int prepare_net_ns(int pid)
 
 int netns_pre_create(void)
 {
-	ns_fd = open("/proc/self/ns/net", O_RDONLY);
+	ns_fd = open("/proc/self/ns/net", O_RDONLY | O_CLOEXEC);
 	if (ns_fd < 0) {
 		pr_perror("Can't cache net fd");
 		return -1;

protobuf/rpc.proto

@@ -37,6 +37,7 @@ message criu_opts {
 
 	optional uint32			cpu_cap		= 20 [default = 0xffffffff];
 	optional bool			force_irmap	= 21;
+	repeated string			exec_cmd	= 22;
 }
 
 message criu_dump_resp {

util.c

@@ -302,7 +302,7 @@ int install_service_fd(enum sfd_type type, int fd)
 
 	BUG_ON((int)type <= SERVICE_FD_MIN || (int)type >= SERVICE_FD_MAX);
 
-	if (dup2(fd, sfd) != sfd) {
+	if (dup3(fd, sfd, O_CLOEXEC) != sfd) {
 		pr_perror("Dup %d -> %d failed", fd, sfd);
 		return -1;
 	}