zdtm/lib: add pre-dump-notify test flag

If pre-dump-notify flag is set, zdtm sends a notify to the test after
pre-dump was finished and waits for the test to send back a reply that
test did all it's work and now is ready for a next pre-dump/dump.

How it can be used:

while (!test_wait_pre_dump()) {
	/* Do something after predump */
	test_wait_pre_dump_ack();
}
/* Do something after restore */

Internally we open two pipes for the test one for receiving notify (with
two open ends) and one for replying to it (only write end open). Fds of
pipes are dupped to predefined numbers and zdtm opens these fds through
/proc/<test-pid>/fd/{100,101} and communicates with the test.

v9: switch to two way interface to remove race then operation we try to
run after predump may be yet unfinished at the time of next dump.
Suggested-by: Andrei Vagin <avagin@virtuozzo.com>
Signed-off-by: Pavel Tikhomirov <ptikhomirov@virtuozzo.com>
Signed-off-by: Andrei Vagin <avagin@virtuozzo.com>

test/zdtm.py

@@ -23,6 +23,7 @@ import fcntl
 import errno
 import datetime
 import yaml
+import struct
 import pycriu as crpc
 
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
@@ -411,6 +412,10 @@ class zdtm_test:
 		if not self.__freezer.kernel:
 			env['ZDTM_THREAD_BOMB'] = "5"
 
+		if test_flag(self.__desc, 'pre-dump-notify'):
+			env['ZDTM_NOTIFY_FDIN'] = "100"
+			env['ZDTM_NOTIFY_FDOUT'] = "101"
+
 		if not test_flag(self.__desc, 'suid'):
 			# Numbers should match those in criu
 			env['ZDTM_UID'] = "18943"
@@ -456,6 +461,27 @@ class zdtm_test:
 
 		self.__flavor.fini()
 
+	def pre_dump_notify(self):
+		env = self._env
+
+		if 'ZDTM_NOTIFY_FDIN' not in env:
+			return
+
+		if self.__pid == 0:
+			self.getpid()
+
+		notify_fdout_path = "/proc/%s/fd/%s" % (self.__pid, env['ZDTM_NOTIFY_FDOUT'])
+		notify_fdin_path = "/proc/%s/fd/%s" % (self.__pid, env['ZDTM_NOTIFY_FDIN'])
+
+		print "Send pre-dump notify to %s" % (self.__pid)
+		with open(notify_fdout_path) as fdout:
+			with open(notify_fdin_path, "w") as fdin:
+				fdin.write(struct.pack("i", 0))
+				fdin.flush()
+				print "Wait pre-dump notify reply"
+				ret = struct.unpack('i', fdout.read(4))
+				print "Completed pre-dump notify with %d" % (ret)
+
 	def stop(self):
 		self.__freezer.thaw()
 		self.getpid()  # Read the pid from pidfile back
@@ -1176,6 +1202,7 @@ def cr(cr_api, test, opts):
 			else:
 				cr_api.dump("pre-dump")
 				try_run_hook(test, ["--post-pre-dump"])
+				test.pre_dump_notify()
 			time.sleep(pres[1])
 
 		sbs('pre-dump')

test/zdtm/lib/ns.c

@@ -19,6 +19,8 @@
 #include "zdtmtst.h"
 #include "ns.h"
 
+int criu_status_in = -1, criu_status_in_peer = -1, criu_status_out = -1;
+
 extern int pivot_root(const char *new_root, const char *put_old);
 static int prepare_mntns(void)
 {
@@ -251,6 +253,11 @@ int ns_init(int argc, char **argv)
 		exit(1);
 	}
 
+	if (init_notify()) {
+		fprintf(stderr, "Can't init pre-dump notification: %m");
+		exit(1);
+	}
+
 	reap = getenv("ZDTM_NOREAP") == NULL;
 
 	sigemptyset(&sa.sa_mask);

test/zdtm/lib/ns.h

@@ -12,4 +12,6 @@ extern int ns_init(int argc, char **argv);
 extern void test_waitsig(void);
 extern void parseargs(int, char **);
 
+extern int init_notify(void);
+
 #endif

test/zdtm/lib/test.c

@@ -33,11 +33,14 @@ enum {
 
 static int parent;
 
+extern int criu_status_in, criu_status_in_peer, criu_status_out;
+
 static void sig_hand(int signo)
 {
 	if (parent)
 		futex_set_and_wake(&test_shared_state->stage, TEST_FAIL_STAGE);
 	futex_set_and_wake(&sig_received, signo);
+	close(criu_status_in);
 }
 
 static char *outfile;
@@ -111,6 +114,63 @@ void test_ext_init(int argc, char **argv)
 		exit(1);
 }
 
+#define PIPE_RD 0
+#define PIPE_WR 1
+
+int init_notify(void)
+{
+	char *val;
+	int ret;
+	int p[2];
+
+	val = getenv("ZDTM_NOTIFY_FDIN");
+	if (!val)
+		return 0;
+	criu_status_in = atoi(val);
+
+	val = getenv("ZDTM_NOTIFY_FDOUT");
+	if (!val)
+		return -1;
+	criu_status_out = atoi(val);
+
+	if (pipe(p)) {
+		fprintf(stderr, "Unable to create a pipe: %m\n");
+		return -1;
+	}
+	criu_status_in_peer = p[PIPE_WR];
+
+	ret = dup2(p[PIPE_RD], criu_status_in);
+	if (ret < 0) {
+		fprintf(stderr, "dup2() failed: %m\n");
+		close(p[PIPE_RD]);
+		close(p[PIPE_WR]);
+		return -1;
+	}
+	close(p[PIPE_RD]);
+
+	if (pipe(p)) {
+		fprintf(stderr, "Unable to create a pipe: %m\n");
+		goto err_pipe_in;
+	}
+	close(p[PIPE_RD]);
+
+	ret = dup2(p[PIPE_WR], criu_status_out);
+	if (ret < 0) {
+		fprintf(stderr, "dup2() failed: %m\n");
+		goto err_pipe_out;
+	}
+
+	close(p[PIPE_WR]);
+	return 0;
+err_pipe_out:
+	close(p[PIPE_RD]);
+	close(p[PIPE_WR]);
+err_pipe_in:
+	close(criu_status_in);
+	close(criu_status_in_peer);
+	return -1;
+}
+
 int write_pidfile(int pid)
 {
 	int fd = -1;
@@ -172,6 +232,9 @@ void test_init(int argc, char **argv)
 			redir_stdfds();
 			ns_init(argc, argv);
 		}
+	} else if (init_notify()) {
+		fprintf(stderr, "Can't init pre-dump notification: %m");
+		exit(1);
 	}
 
 	val = getenv("ZDTM_GROUPS");
@@ -297,6 +360,43 @@ void test_waitsig(void)
 	futex_wait_while(&sig_received, 0);
 }
 
+int test_wait_pre_dump(void)
+{
+	int ret;
+
+	if (criu_status_in < 0) {
+		pr_err("Fd criu_status_in is not initialized\n");
+		return -1;
+	}
+
+	if (read(criu_status_in, &ret, sizeof(ret)) != sizeof(ret)) {
+		if (errno != EBADF || !futex_get(&sig_received))
+			pr_perror("Can't wait pre-dump\n");
+		return -1;
+	}
+	pr_err("pre-dump\n");
+
+	return 0;
+}
+
+int test_wait_pre_dump_ack(void)
+{
+	int ret = 0;
+
+	if (criu_status_out < 0) {
+		pr_err("Fd criu_status_out is not initialized\n");
+		return -1;
+	}
+
+	pr_err("pre-dump-ack\n");
+	if (write(criu_status_out, &ret, sizeof(ret)) != sizeof(ret)) {
+		pr_perror("Can't reply to pre-dump notify");
+		return -1;
+	}
+
+	return 0;
+}
+
 pid_t sys_clone_unified(unsigned long flags, void *child_stack, void *parent_tid,
 			void *child_tid, unsigned long newtls)
 {

test/zdtm/lib/zdtmtst.h

@@ -41,6 +41,10 @@ extern void test_msg(const char *format, ...)
 extern int test_go(void);
 /* sleep until SIGTERM is delivered */
 extern void test_waitsig(void);
+/* sleep until zdtm notifies about predump */
+extern int test_wait_pre_dump(void);
+/* notify zdtm that we finished action after predump */
+extern int test_wait_pre_dump_ack(void);
 
 #include <stdint.h>