parasite: Restore parasite state via rt_sigreturn

This patch reduces a window, when a crtools can kill a dumped process,
because if a parasite in a deamon mode can restore the state of the
process, if crtools detached unexpectedly.

All threads are synchronized on the _exit_ point from sys_rt_sigreturn,
for that crtools traces all syscalls. After that we remove the parasite
blob from dumpee as we did it previously and let the process run further.
Signed-off-by: Andrey Vagin <avagin@openvz.org>
Signed-off-by: Pavel Emelyanov <xemul@parallels.com>

parasite-syscall.c

@@ -808,6 +808,7 @@ int parasite_init_threads_seized(struct parasite_ctl *ctl, struct pstree_item *i
 		ctl->nr_threads++;
 
 		args->id = i;
+		args->sigframe = ctl->threads[i].rsigframe;
 
 		ret = ptrace(PTRACE_GETREGS, tid, NULL, regs_orig);
 		if (ret) {
@@ -821,7 +822,7 @@ int parasite_init_threads_seized(struct parasite_ctl *ctl, struct pstree_item *i
 			goto err;
 		}
 
-		ret = get_task_regs(tid, ctl->threads[i].regs_orig, item->core[i]);
+		ret = get_task_regs(tid, *regs_orig, item->core[i]);
 		if (ret) {
 			pr_err("Can't obtain regs for thread %d\n", tid);
 			goto err;
@@ -839,78 +840,107 @@ err:
 	return -1 ;
 }
 
-int parasite_fini_threads_seized(struct parasite_ctl *ctl)
+static int parasite_fini_seized(struct parasite_ctl *ctl)
 {
-	struct parasite_init_args *args;
-	int ret = 0, i, status;
+	int status, ret = 0, i, nr = 0, nr_dmnz = 0;
 
-	args = parasite_args(ctl, struct parasite_init_args);
-
-	for (i = 1; i < ctl->nr_threads; i++) {
-		pid_t tid = ctl->threads[i].tid;
+	/* Start to trace syscalls for each thread */
+	for (i = 0; i < ctl->nr_threads; i++) {
+		pid_t pid = ctl->threads[i].tid;
 
 		if (!ctl->threads[i].daemonized)
 			break;
 
-		args->id = i;
-		ret = parasite_execute_daemon_by_id(PARASITE_CMD_FINI_THREAD, ctl, i);
-		/*
-		 * Note the thread's fini() can be called even when not
-		 * all threads were init()'ed, say we're rolling back from
-		 * error happened while we were init()'ing some thread, thus
-		 * -ENOENT will be returned but we should continie for the
-		 * rest of threads set.
-		 *
-		 * Strictly speaking we always init() threads in sequence thus
-		 * we could simply break the loop once first -ENOENT returned
-		 * but I prefer to be on a safe side even if some future changes
-		 * would change the code logic.
-		 */
-		if (ret && ret != -ENOENT) {
-			pr_err("Can't fini thread in parasite %d\n", tid);
-			break;
-		} else if (ret == -ENOENT)
-			continue;
+		ptrace(PTRACE_INTERRUPT, pid, NULL, NULL);
 
-		pr_debug("Waiting for %d to trap\n", tid);
-		if (wait4(tid, &status, __WALL, NULL) != tid) {
-			pr_perror("Waited pid mismatch (pid: %d)", tid);
-			break;
+		pr_debug("Waiting for %d to trap\n", pid);
+		if (wait4(pid, &status, __WALL, NULL) != pid) {
+			pr_perror("Waited pid mismatch (pid: %d)", pid);
+			return -1;
 		}
 
-		pr_debug("Daemon %d exited trapping\n", tid);
+		pr_debug("Daemon %d exited trapping\n", pid);
 		if (!WIFSTOPPED(status)) {
-			pr_err("Task is still running (pid: %d)\n", tid);
-			break;
+			pr_err("Task is still running (pid: %d)\n", pid);
+			return -1;
 		}
 
-		if (ptrace(PTRACE_SETREGS, tid, NULL, &ctl->threads[i].regs_orig)) {
-			pr_perror("Can't restore registers (pid: %d)", tid);
+		ret = ptrace(PTRACE_SYSCALL, pid, NULL, NULL);
+		if (ret) {
+			pr_perror("ptrace");
 			return -1;
 		}
+
+		nr_dmnz++;
 	}
 
-	return ret;
-}
+	ret = __parasite_execute_daemon_by_id(PARASITE_CMD_FINI, ctl, 0, false);
+	if (ret)
+		return -1;
 
-static int parasite_fini_seized(struct parasite_ctl *ctl)
-{
-	struct parasite_init_args *args;
-	int status, ret = 0;
+	/* Stop all threads on the enter point in sys_rt_sigreturn */
+	while (1) {
+		user_regs_struct_t regs;
+		pid_t pid;
 
-	args = parasite_args(ctl, struct parasite_init_args);
+		pid = wait4(-1, &status, __WALL, NULL);
+		if (pid < 0) {
+			pr_perror("wait4 failed");
+			return -1;
+		}
 
-	args->id = 0;
-	__parasite_execute_daemon_by_id(PARASITE_CMD_FINI, ctl, 0, false);
+		pr_debug("%d was trapped\n", pid);
+		if (!WIFSTOPPED(status)) {
+			pr_err("%d\n", status);
+			return -1;
+		}
+		ret = ptrace(PTRACE_GETREGS, pid, NULL, &regs);
+		if (ret) {
+			pr_perror("ptrace");
+			return -1;
+		}
 
-	if (wait4(ctl->pid.real, &status, __WALL, NULL) != ctl->pid.real) {
-		pr_perror("Waited pid mismatch (pid: %d)", ctl->pid.real);
-		ret = -1;
+		pr_debug("%d is going to execute the syscall %lx\n", pid, regs.orig_ax);
+		if (regs.orig_ax == __NR_rt_sigreturn) {
+			nr++;
+			pr_debug("%d was stopped\n", pid);
+			if (nr == nr_dmnz)
+				break;
+			continue;
+		}
+
+		ret = ptrace(PTRACE_SYSCALL, pid, NULL, NULL);
+		if (ret) {
+			pr_perror("ptrace");
+			return -1;
+		}
 	}
 
-	if (!WIFSTOPPED(status)) {
-		pr_err("Task is still running (pid: %d)\n", ctl->pid.real);
-		ret = -1;
+	/* Stop all threads on the exit point from sys_rt_sigreturn */
+	for (i = 0; i < ctl->nr_threads; i++) {
+		pid_t pid = ctl->threads[i].tid;
+
+		if (!ctl->threads[i].daemonized)
+			break;
+
+		ctl->threads[i].use_sig_blocked = false;
+
+		ret = ptrace(PTRACE_SYSCALL, pid, NULL, NULL);
+		if (ret) {
+			pr_perror("ptrace");
+			return -1;
+		}
+
+		if (wait4(pid, &status, __WALL, NULL) != pid) {
+			pr_perror("wait4 failed");
+			return -1;
+		}
+
+		pr_debug("Trap %d\n", pid);
+		if (!WIFSTOPPED(status)) {
+			pr_err("%d\n", status);
+			return -1;
+		}
 	}
 
 	return ret;
@@ -920,10 +950,9 @@ int parasite_cure_remote(struct parasite_ctl *ctl)
 {
 	int ret = 0;
 
-	if (ctl->parasite_ip) {
-		ret = parasite_fini_threads_seized(ctl);
-		parasite_fini_seized(ctl);
-	}
+	if (ctl->parasite_ip)
+		if (parasite_fini_seized(ctl))
+			return -1;
 
 	ctl->tsock = -1;
 

pie/parasite.c

@@ -14,7 +14,9 @@
 
 #include <string.h>
 
+#include "asm/types.h"
 #include "asm/parasite.h"
+#include "asm/restorer.h"
 
 static int tsock = -1;
 
@@ -24,9 +26,6 @@ static struct tid_state_s {
 	futex_t		cmd;
 	int		ret;
 
-	bool		use_sig_blocked;
-	k_rtsigset_t	sig_blocked;
-
 	struct rt_sigframe *sigframe;
 } *tid_state;
 
@@ -206,14 +205,11 @@ static int init_thread(struct parasite_init_args *args)
 
 	ksigfillset(&to_block);
 	ret = sys_sigprocmask(SIG_SETMASK, &to_block,
-			      &tid_state[next_tid_state].sig_blocked,
+			      &args->sig_blocked,
 			      sizeof(k_rtsigset_t));
 	if (ret)
 		return -1;
 
-	tid_state[next_tid_state].use_sig_blocked = true;
-	args->sig_blocked = tid_state[next_tid_state].sig_blocked;
-
 	tid_state[next_tid_state].id = next_tid_state;
 	tid_state[next_tid_state].sigframe = args->sigframe;
 
@@ -224,11 +220,25 @@ static int init_thread(struct parasite_init_args *args)
 	return ret;
 }
 
+static void __parasite_daemon_thread_ack(struct tid_state_s *s, int ret)
+{
+	s->ret = ret;
+	futex_set_and_wake(&s->cmd, PARASITE_CMD_IDLE);
+}
+
 static int fini_thread(struct tid_state_s *s)
 {
-	if (s->use_sig_blocked)
-		return sys_sigprocmask(SIG_SETMASK, &s->sig_blocked,
-				       NULL, sizeof(k_rtsigset_t));
+	unsigned long new_sp;
+
+	new_sp = (long)s->sigframe + SIGFRAME_OFFSET;
+	pr_debug("%ld: new_sp=%lx ip %lx\n", sys_gettid(),
+		 new_sp, s->sigframe->uc.uc_mcontext.rip);
+
+	__parasite_daemon_thread_ack(s, 0);
+
+	ARCH_RT_SIGRETURN(new_sp);
+
+	BUG();
 
 	return 0;
 }
@@ -415,13 +425,6 @@ static int parasite_check_vdso_mark(struct parasite_vdso_vma_entry *args)
 	return 0;
 }
 
-static int fini(struct tid_state_s *s)
-{
-	log_set_fd(-1);
-
-	return fini_thread(s);
-}
-
 static int __parasite_daemon_reply_ack(unsigned int id, unsigned int cmd, int err)
 {
 	struct ctl_msg m;
@@ -470,12 +473,6 @@ static int __parasite_daemon_thread_wait_cmd(struct tid_state_s *s)
 	return futex_get(&s->cmd);
 }
 
-static void __parasite_daemon_thread_ack(struct tid_state_s *s, int ret)
-{
-	s->ret = ret;
-	futex_set_and_wake(&s->cmd, PARASITE_CMD_IDLE);
-}
-
 static void noinline __used
 __parasite_daemon_thread(void *args, struct tid_state_s *s)
 {
@@ -497,7 +494,6 @@ __parasite_daemon_thread(void *args, struct tid_state_s *s)
 			ret = dump_thread(args);
 			break;
 		case PARASITE_CMD_FINI_THREAD:
-			__parasite_daemon_thread_ack(s, 0);
 			fini_thread(s);
 			return;
 		default:
@@ -508,6 +504,8 @@ __parasite_daemon_thread(void *args, struct tid_state_s *s)
 		__parasite_daemon_thread_ack(s, ret);
 	}
 
+	pr_err("The thread %d trys to escape!!!", s->id);
+	BUG();
 	return;
 }
 
@@ -524,6 +522,32 @@ static int __parasite_execute_thread(struct ctl_msg *m)
 	return s->ret;
 }
 
+static int fini(struct tid_state_s *s)
+{
+	unsigned long new_sp;
+	int i;
+
+	for (i = 1; i < next_tid_state; i++) {
+		struct ctl_msg m = {.cmd = PARASITE_CMD_FINI_THREAD, .id = i};
+		__parasite_execute_thread(&m);
+	}
+
+	new_sp = (long)s->sigframe + SIGFRAME_OFFSET;
+	pr_debug("%ld: new_sp=%lx ip %lx\n", sys_gettid(),
+		  new_sp, s->sigframe->uc.uc_mcontext.rip);
+
+	sys_close(tsock);
+	log_set_fd(-1);
+
+	sys_munmap(tid_state, TID_STATE_SIZE(nr_tid_state));
+
+	ARCH_RT_SIGRETURN(new_sp);
+
+	BUG();
+
+	return -1;
+}
+
 static void noinline __used
 __parasite_daemon_thread_leader(void *args, struct tid_state_s *s)
 {
@@ -534,7 +558,7 @@ __parasite_daemon_thread_leader(void *args, struct tid_state_s *s)
 
 	/* Reply we're alive */
 	if (__parasite_daemon_reply_ack(0, PARASITE_CMD_DAEMONIZE, 0))
-		return;
+		goto out;
 
 	while (1) {
 		if (__parasite_daemon_wait_msg(&m))
@@ -594,6 +618,9 @@ __parasite_daemon_thread_leader(void *args, struct tid_state_s *s)
 			break;
 	}
 
+out:
+	fini(&tid_state[0]);
+
 	return;
 }
 
@@ -613,9 +640,6 @@ static int noinline parasite_daemon(struct parasite_init_args *args)
 
 	pr_info("Parasite leaving daemon mode for %d\n", s->id);
 
-	if (is_leader)
-		sys_munmap(tid_state, TID_STATE_SIZE(nr_tid_state));
-
 	return 0;
 }