zdtm: Move towards the new generation of criu testing (v3)

A little bit more stuff added :) With these changes I can run the

  zdtm.py run --all -x cgroup -x maps04 -x different_creds -x rtc

To run cgroups tests need to add .hook calls, for maps04 I don't have
enough RAM and disk in my VM (will fix), for different_creds need to
support crfail test option (dump _must_ fail), for rtc -- plugins.

So changes since v2:

1. Added exclusion (-x option)
2. Bugfix in parallel run
3. Fixed NS root permissions
4. Fixed checks for maps before and after dump
5. Fixed thread_bomb launch
6. Print test output
7. Support .checkskip scripts
8. Support features
9. Fixed test list

Andrey, thoughts?
Signed-off-by: Pavel Emelyanov <xemul@parallels.com>
Acked-by: Andrew Vagin <avagin@openvz.org>

test/zdtm.list

+static/pipe00:
+static/pipe01:
+static/pipe02:
+static/busyloop00:
+static/cwd00:
+static/cwd01:
+static/cwd02:
+static/env00:
+static/maps00:
+static/maps01:
+  flags: suid
+  flavor: h ns
+static/maps02:
+static/maps04:
+static/maps05:
+static/mlock_setuid:
+  flags: suid
+  flavor: h ns
+static/maps_file_prot:
+static/mprotect00:
+static/mtime_mmap:
+static/sleeping00:
+static/write_read00:
+static/write_read01:
+static/write_read02:
+static/write_read10:
+static/wait00:
+static/vdso00:
+static/sched_prio00:
+  flags: suid
+  flavor: h ns
+static/sched_policy00:
+  flags: suid
+  flavor: h ns
+static/file_shared:
+static/file_append:
+static/timers:
+static/posix_timers:
+static/futex:
+static/futex-rl:
+static/xids00:
+static/groups:
+  flags: suid
+static/pthread00:
+static/pthread01:
+static/umask00:
+streaming/pipe_loop00:
+streaming/pipe_shared00:
+transition/file_read:
+static/sockets00:
+  flags: suid
+static/sockets01:
+static/sockets02:
+static/sock_opts00:
+  flags: suid
+static/sock_opts01:
+  flags: suid
+static/sockets_spair:
+static/sockets_dgram:
+static/socket_dgram_data:
+static/socket_queues:
+static/deleted_unix_sock:
+static/sk-unix-unconn:
+static/sk-unix-rel:
+static/pid00:
+  flags: suid
+static/pstree:
+static/caps00:
+  flags: suid
+static/cmdlinenv00:
+  flags: suid
+static/socket_listen:
+static/socket_listen6:
+static/packet_sock:
+  flags: suid
+static/packet_sock_mmap:
+  flags: suid
+static/socket_udp:
+static/sock_filter:
+static/socket6_udp:
+static/socket_udplite:
+static/selfexe00:
+static/link10:
+static/unlink_fstat00:
+static/unlink_fstat01:
+static/unlink_fstat02:
+static/unlink_fstat03:
+  opts: --link-remap
+static/unlink_mmap00:
+static/unlink_mmap01:
+static/unlink_mmap02:
+static/rmdir_open:
+static/eventfs00:
+static/signalfd00:
+static/inotify00:
+  opts: --link-remap
+static/inotify_irmap:
+  flags: suid
+static/fanotify00:
+  flags: suid
+  flavor: h ns
+static/unbound_sock:
+static/fifo-rowo-pair:
+static/fifo-ghost:
+static/fifo:
+static/fifo_wronly:
+static/fifo_ro:
+static/unlink_fifo:
+static/unlink_fifo_wronly:
+static/zombie00:
+static/rlimits00:
+transition/fork:
+transition/fork2:
+transition/thread-bomb:
+static/pty00:
+static/pty01:
+static/pty04:
+static/tty02:
+static/tty03:
+static/console:
+  flags: suid
+  flavor: h ns
+static/vt:
+  flags: suid
+  flavor: h ns
+static/child_opened_proc:
+static/cow01:
+  flags: suid
+  flavor: h ns
+static/pdeath_sig:
+static/fdt_shared:
+static/file_locks00:
+  flags: excl
+  opts: --file-locks
+static/file_locks01:
+  flags: excl
+  opts: --file-locks
+static/file_locks02:
+  flags: excl
+  opts: --file-locks
+static/file_locks03:
+  flags: excl
+  opts: --file-locks
+static/file_locks04:
+  flags: excl
+  opts: --file-locks
+static/file_locks05:
+  flags: excl
+  opts: --file-locks
+static/sigpending:
+static/sigaltstack:
+static/sk-netlink:
+  flags: suid
+static/proc-self:
+static/grow_map:
+static/grow_map02:
+static/grow_map03:
+static/stopped:
+static/chroot:
+  flags: suid
+static/chroot-file:
+  flags: suid
+static/rtc:
+  flags: suid crlib
+  flavor: h
+transition/maps007:
+  flags: suid
+static/dumpable01:
+static/dumpable02:
+  flavor: h ns
+static/deleted_dev:
+  flags: suid
+  flavor: h ns
+static/fpu00:
+  arch: x86_64
+static/fpu01:
+  arch: x86_64
+static/mmx00:
+  arch: x86_64
+static/sse00:
+  arch: x86_64
+static/sse20:
+  arch: x86_64
+static/vdso01:
+  arch: x86_64
+static/vsx:
+  arch: ppc64le
+static/file_fown:
+  flavor: h
+static/socket-ext:
+  flavor: h
+  opts: --ext-unix-sk
+static/socket-tcp:
+  flavor: h
+  opts: --tcp-established
+static/socket-tcp6:
+  flavor: h
+  opts: --tcp-established
+streaming/socket-tcp:
+  flavor: h
+  opts: --tcp-established
+streaming/socket-tcp6:
+  flavor: h
+  opts: --tcp-established
+static/socket-tcpbuf:
+  flavor: h
+  opts: --tcp-established
+static/socket-tcpbuf-local:
+  flavor: h
+  opts: --tcp-established
+static/socket-tcpbuf6:
+  flavor: h
+  opts: --tcp-established
+static/pty03:
+  flavor: h
+static/mountpoints:
+  flags: suid
+  flavor: h
+static/utsname:
+  flavor: h
+static/ipc_namespace:
+  flavor: h
+static/shm:
+  flavor: h
+static/msgque:
+  flavor: h
+static/sem:
+  flavor: h
+transition/ipc:
+  flavor: h
+static/netns-nf:
+  flavor: h
+static/netns:
+  flavor: h
+static/cgroup00:
+  flags: suid
+  flavor: h
+  opts: --manage-cgroups
+static/cgroup01:
+  flags: suid
+  flavor: h
+  opts: --manage-cgroups
+static/cgroup02:
+  flags: suid
+  flavor: h
+  opts: --manage-cgroups --cgroup-root /newroot --cgroup-root name=zdtmtst:/zdtmtstroot
+static/remap_dead_pid:
+  flavor: h
+static/poll:
+  flavor: h
+static/apparmor:
+  flags: suid
+  flavor: h ns
+static/different_creds:
+  flags: suid crfail
+  flavor: h
+static/aio00:
+  feature: aio_remap
+static/timerfd:
+  feature: timerfd
+static/session00:
+  flavor: ns uns
+static/session01:
+  flavor: ns uns
+static/tempfs:
+  flags: suid
+  flavor: ns uns
+static/tempfs_ro:
+  flags: suid
+  flavor: ns
+static/mnt_ro_bind:
+  flags: suid
+  flavor: ns uns
+static/mount_paths:
+  flags: suid
+  flavor: ns uns
+static/bind-mount:
+  flags: suid
+  flavor: ns uns
+static/netns-dev:
+  flags: suid
+  flavor: ns uns
+static/mnt_ext_auto:
+  flavor: ns uns
+  feature: mntid
+  opts: --ext-mount-map auto --enable-external-sharing
+static/mnt_ext_master:
+  flavor: ns uns
+  feature: mntid
+  opts: --ext-mount-map auto --enable-external-masters
+static/mntns_open:
+  flags: suid
+  flavor: ns uns
+  feature: mntid
+static/mntns_link_remap:
+  flags: suid
+  flavor: ns
+  feature: mntid
+  opts: --link-remap
+static/mntns_link_ghost:
+  flags: suid
+  flavor: ns
+  feature: mntid
+static/mntns_shared_bind:
+  flags: suid
+  flavor: ns uns
+  feature: mntid
+static/mntns_shared_bind02:
+  flags: suid
+  flavor: ns uns
+  feature: mntid
+static/mntns_root_bind:
+  flags: suid
+  flavor: ns uns
+  feature: mntid
+static/mntns_deleted:
+  flags: suid
+  flavor: ns uns
+  feature: mntid
+static/tun:
+  flags: suid
+  flavor: ns uns
+  feature: tun
+static/seccomp_strict:
+  flags: suid
+  flavor: h
+  feature: seccomp_suspend
+static/clean_mntns:
+  flags: suid
+  flavor: ns

test/zdtm.py

+#!/bin/env python
+import argparse
+import yaml
+import os
+import subprocess
+import time
+import tempfile
+import shutil
+import re
+import stat
+import signal
+import atexit
+import sys
+import linecache
+
+prev_line = None
+def traceit(f, e, a):
+	if e == "line":
+		lineno = f.f_lineno
+		fil = f.f_globals["__file__"]
+		if fil.endswith("zdtm.py"):
+			global prev_line
+			line = linecache.getline(fil, lineno)
+			if line == prev_line:
+				print "        ..."
+			else:
+				prev_line = line
+				print "+%4d: %s" % (lineno, line.rstrip())
+
+	return traceit
+
+
+# Descriptor for abstract test not in list
+default_test={ }
+
+# Root dir for ns and uns flavors. All tests
+# sit in the same dir
+zdtm_root = None
+
+def clean_zdtm_root():
+	global zdtm_root
+	if zdtm_root:
+		os.rmdir(zdtm_root)
+
+def make_zdtm_root():
+	global zdtm_root
+	if not zdtm_root:
+		zdtm_root = tempfile.mkdtemp("", "criu-root-", "/tmp")
+		atexit.register(clean_zdtm_root)
+	return zdtm_root
+
+# Arch we run on
+arch = os.uname()[4]
+
+#
+# Flavors
+#  h -- host, test is run in the same set of namespaces as criu
+#  ns -- namespaces, test is run in itw own set of namespaces
+#  uns -- user namespace, the same as above plus user namespace
+#
+
+class host_flavor:
+	def __init__(self, opts):
+		self.name = "host"
+		self.ns = False
+		self.root = None
+
+	def init(self, test_bin):
+		pass
+
+	def fini(self):
+		pass
+
+class ns_flavor:
+	def __init__(self, opts):
+		self.name = "ns"
+		self.ns = True
+		self.uns = False
+		self.root = make_zdtm_root()
+
+	def init(self, test_bin):
+		print "Construct root for %s" % test_bin
+		subprocess.check_call(["mount", "--make-private", "--bind", ".", self.root])
+
+		if not os.access(self.root + "/.constructed", os.F_OK):
+			for dir in ["/bin", "/etc", "/lib", "/lib64", "/dev", "/tmp"]:
+				os.mkdir(self.root + dir)
+				os.chmod(self.root + dir, 0777)
+
+			os.mknod(self.root + "/dev/tty", stat.S_IFCHR, os.makedev(5, 0))
+			os.chmod(self.root + "/dev/tty", 0666)
+			os.mknod(self.root + "/.constructed", stat.S_IFREG | 0600)
+
+		ldd = subprocess.Popen(["ldd", test_bin], stdout = subprocess.PIPE)
+		xl = re.compile('^(linux-gate.so|linux-vdso(64)?.so|not a dynamic)')
+
+		# This Mayakovsky-style code gets list of libraries a binary
+		# needs minus vdso and gate .so-s
+		libs = map(lambda x: x[1] == '=>' and x[2] or x[0],		\
+				map(lambda x: x.split(),			\
+					filter(lambda x: not xl.match(x),	\
+						map(lambda x: x.strip(),	\
+							filter(lambda x: x.startswith('\t'), ldd.stdout.readlines())))))
+		ldd.wait()
+
+		for lib in libs:
+			tlib = self.root + lib
+			if not os.access(tlib, os.F_OK):
+				# Copying should be atomic as tests can be
+				# run in parallel
+				dst = tempfile.mktemp(".tso", "", self.root + os.path.dirname(lib))
+				shutil.copy2(lib, dst)
+				os.rename(dst, tlib)
+
+	def fini(self):
+		subprocess.check_call(["mount", "--make-private", self.root])
+		subprocess.check_call(["umount", "-l", self.root])
+
+class userns_flavor(ns_flavor):
+	def __init__(self, opts):
+		ns_flavor.__init__(self, opts)
+		self.name = "userns"
+		self.uns = True
+
+flavors = { 'h': host_flavor, 'ns': ns_flavor, 'uns': userns_flavor }
+
+#
+# Helpers
+#
+
+def tail(path):
+	p = subprocess.Popen(['tail', '-n1', path],
+			stdout = subprocess.PIPE)
+	return p.stdout.readline()
+
+def rpidfile(path):
+	return open(path).readline().strip()
+
+def wait_pid_die(pid, who, tmo = 3):
+	stime = 0.1
+	while stime < tmo:
+		try:
+			os.kill(int(pid), 0)
+		except: # Died
+			break
+
+		print "Wait for %s to die for %f" % (who, stime)
+		time.sleep(stime)
+		stime *= 2
+	else:
+		raise test_fail_exc("%s die" % who)
+
+def test_flag(tdesc, flag):
+	return flag in tdesc.get('flags', '').split()
+
+#
+# Exception thrown when something inside the test goes wrong,
+# e.g. test doesn't start, criu returns with non zero code or
+# test checks fail
+#
+
+class test_fail_exc:
+	def __init__(self, step):
+		self.step = step
+
+#
+# A test from zdtm/ directory.
+#
+
+class zdtm_test:
+	def __init__(self, name, desc, flavor):
+		self.__name = name
+		self.__desc = desc
+		self.__make_action('cleanout')
+		self.__pid = 0
+		self.__flavor = flavor
+
+	@staticmethod
+	def __zdtm_path(name, typ):
+		return os.path.join("zdtm/live/", name + typ)
+
+	def __getpath(self, typ = ''):
+		return self.__zdtm_path(self.__name, typ)
+
+	def __make_action(self, act, env = None, root = None):
+		tpath = self.__getpath('.' + act)
+		s_args = ['make', '--no-print-directory', \
+			 	'-C', os.path.dirname(tpath), \
+				      os.path.basename(tpath)]
+
+		if env:
+			env = dict(os.environ, **env)
+
+		s = subprocess.Popen(s_args, env = env, cwd = root)
+		s.wait()
+
+	def __pidfile(self):
+		if self.__flavor.ns:
+			return self.__getpath('.init.pid')
+		else:
+			return self.__getpath('.pid')
+
+	def __wait_task_die(self):
+		wait_pid_die(int(self.__pid), self.__name)
+
+	def start(self):
+		env = {}
+		self.__flavor.init(self.__getpath())
+
+		print "Start test"
+
+		env['ZDTM_THREAD_BOMB'] = "100"
+		if not test_flag(self.__desc, 'suid'):
+			env['ZDTM_UID'] = "18943"
+			env['ZDTM_GID'] = "58467"
+			env['ZDTM_GROUPS'] = "27495 48244"
+		else:
+			print "Test is SUID"
+
+		if self.__flavor.ns:
+			env['ZDTM_NEWNS'] = "1"
+			env['ZDTM_PIDFILE'] = os.path.realpath(self.__getpath('.init.pid'))
+			env['ZDTM_ROOT'] = self.__flavor.root
+
+			if self.__flavor.uns:
+				env['ZDTM_USERNS'] = "1"
+
+		self.__make_action('pid', env, self.__flavor.root)
+
+		try:
+			os.kill(int(self.getpid()), 0)
+		except:
+			raise test_fail_exc("start")
+
+	def kill(self, sig = signal.SIGKILL):
+		if self.__pid:
+			os.kill(int(self.__pid), sig)
+			self.gone(sig == signal.SIGKILL)
+
+		self.__flavor.fini()
+
+	def stop(self):
+		print "Stop test"
+		self.kill(signal.SIGTERM)
+
+		res = tail(self.__getpath('.out'))
+		if not 'PASS' in res.split():
+			raise test_fail_exc("result check")
+
+	def getpid(self):
+		if self.__pid == 0:
+			self.__pid = rpidfile(self.__pidfile())
+
+		return self.__pid
+
+	def getname(self):
+		return self.__name
+
+	def getcropts(self):
+		opts = self.__desc.get('opts', '').split() + ["--pidfile", os.path.realpath(self.__pidfile())]
+		if self.__flavor.ns:
+			opts += ["--root", self.__flavor.root]
+		return opts
+
+	def gone(self, force = True):
+		self.__wait_task_die()
+		self.__pid = 0
+		if force or self.__flavor.ns:
+			os.unlink(self.__pidfile())
+
+	def print_output(self):
+		print "Test output: " + "=" * 32
+		print open(self.__getpath('.out')).read()
+		print " <<< " + "=" * 32
+
+	@staticmethod
+	def checkskip(name):
+		chs = zdtm_test.__zdtm_path(name, ".checkskip")
+		if os.access(chs, os.X_OK):
+			ch = subprocess.Popen([chs])
+			return ch.wait() == 0 and False or True
+
+		return False
+			
+
+#
+# CRIU when launched using CLI
+#
+
+class criu_cli:
+	def __init__(self, test, opts):
+		self.__test = test
+		self.__dump_path = "dump/" + test.getname() + "/" + test.getpid()
+		self.__iter = 0
+		os.makedirs(self.__dump_path)
+		self.__page_server = (opts['page_server'] and True or False)
+
+	def __ddir(self):
+		return os.path.join(self.__dump_path, "%d" % self.__iter)
+
+	@staticmethod
+	def __criu(action, args):
+		cr = subprocess.Popen(["../criu", action] + args)
+		return cr.wait()
+
+	def __criu_act(self, action, opts, log = None):
+		if not log:
+			log = action + ".log"
+
+		s_args = ["-o", log, "-D", self.__ddir(), "-v4"] + opts
+
+		print "Run CRIU: [" + " ".join(s_args) + "]"
+		ret = self.__criu(action, s_args)
+		if ret != 0:
+			raise test_fail_exc("CRIU %s" % action)
+
+	def __criu_cr(self, action, opts):
+		self.__criu_act(action, opts = opts + self.__test.getcropts())
+
+	def dump(self, action, opts = []):
+		self.__iter += 1
+		os.mkdir(self.__ddir())
+
+		a_opts = ["-t", self.__test.getpid()]
+		if self.__iter > 1:
+			a_opts += ["--prev-images-dir", "../%d" % (self.__iter - 1), "--track-mem"]
+
+		if self.__page_server:
+			print "Adding page server"
+			self.__criu_act("page-server", opts = [ "--port", "12345", \
+					"--daemon", "--pidfile", "ps.pid"])
+			a_opts += ["--page-server", "--address", "127.0.0.1", "--port", "12345"]
+
+		self.__criu_cr(action, opts = a_opts + opts)
+
+		if self.__page_server:
+			wait_pid_die(int(rpidfile(self.__ddir() + "/ps.pid")), "page server")
+
+	def restore(self):
+		self.__criu_cr("restore", opts = ["--restore-detached"])
+
+	@staticmethod
+	def check(feature):
+		return criu_cli.__criu("check", ["-v0", "--feature", feature]) == 0
+
+#
+# Main testing entity -- dump (probably with pre-dumps) and restore
+#
+
+def cr(test, opts):
+	if opts['nocr']:
+		return
+
+	cr_api = criu_cli(test, opts)
+
+	for i in xrange(0, int(opts['iters'] or 1)):
+		for p in xrange(0, int(opts['pre'] or 0)):
+			cr_api.dump("pre-dump")
+
+		if opts['norst']:
+			cr_api.dump("dump", opts = ["--leave-running"])
+		else:
+			cr_api.dump("dump")
+			test.gone()
+			cr_api.restore()
+
+# Additional checks that can be done outside of test process
+
+def get_maps(test):
+	maps = [[0,0]]
+	last = 0
+	for mp in open("/proc/%s/maps" % test.getpid()).readlines():
+		m = map(lambda x: int('0x' + x, 0), mp.split()[0].split('-'))
+		if maps[last][1] == m[0]:
+			maps[last][1] = m[1]
+		else:
+			maps.append(m)
+			last += 1
+	maps.pop(0)
+	return maps
+
+def get_fds(test):
+	return map(lambda x: int(x), os.listdir("/proc/%s/fdinfo" % test.getpid()))
+
+def cmp_lists(m1, m2):
+	return filter(lambda x: x[0] != x[1], zip(m1, m2))
+
+def get_visible_state(test):
+	fds = get_fds(test)
+	maps = get_maps(test)
+	return (fds, maps)
+
+def check_visible_state(test, state):
+	new = get_visible_state(test)
+	if cmp_lists(new[0], state[0]):
+		raise test_fail_exc("fds compare")
+	if cmp_lists(new[1], state[1]):
+		raise test_fail_exc("maps compare")
+
+def do_run_test(tname, tdesc, flavs, opts):
+	print "Run %s in %s" % (tname, flavs)
+
+	for f in flavs:
+		flav = flavors[f](opts)
+		t = zdtm_test(tname, tdesc, flav)
+
+		try:
+			t.start()
+			s = get_visible_state(t)
+			cr(t, opts)
+			check_visible_state(t, s)
+			t.stop()
+		except test_fail_exc as e:
+			t.print_output()
+			t.kill()
+			print "Test %s FAIL at %s" % (tname, e.step)
+			# This exit does two things -- exits from subprocess and
+			# aborts the main script execution on the 1st error met
+			sys.exit(1)
+		else:
+			print "Test %s PASS" % tname
+
+class launcher:
+	def __init__(self, opts):
+		self.__opts = opts
+		self.__max = int(opts['parallel'] or 0)
+		self.__subs = {}
+		self.__fail = False
+
+	def run_test(self, name, desc, flavor):
+		if self.__max == 0:
+			do_run_test(name, desc, flavor, self.__opts)
+			return
+
+		if len(self.__subs) >= self.__max:
+			self.wait()
+			if self.__fail:
+				raise test_fail_exc('')
+
+		nd = ('nocr', 'norst', 'pre', 'iters', 'page_server')
+		arg = repr((name, desc, flavor, { d: self.__opts[d] for d in nd }))
+		log = name.replace('/', '_') + ".log"
+		sub = subprocess.Popen(["zdtm_ct", "zdtm.py"], env = { 'ZDTM_CT_TEST_INFO': arg }, \
+				stdout = open(log, "w"), stderr = subprocess.STDOUT)
+		self.__subs[sub.pid] = { 'sub': sub, 'log': log }
+
+	def __wait_one(self, flags):
+		pid, status = os.waitpid(0, flags)
+		if pid != 0:
+			sub = self.__subs.pop(pid)
+			if status != 0:
+				self.__fail = True
+
+			print open(sub['log']).read()
+			os.unlink(sub['log'])
+			return True
+
+		return False
+
+	def wait(self):
+		self.__wait_one(0)
+		while self.__subs:
+			if not self.__wait_one(os.WNOHANG):
+				break
+
+	def finish(self):
+		while self.__subs:
+			self.__wait_one(0)
+		if self.__fail:
+			sys.exit(1)
+
+def run_tests(opts, tlist):
+	excl = None
+	features = {}
+
+	if opts['all']:
+		torun = tlist
+	elif opts['test']:
+		torun = opts['test']
+	else:
+		print "Specify test with -t <name> or -a"
+		return
+
+	if opts['exclude']:
+		excl = re.compile(".*(" + "|".join(opts['exclude']) + ")")
+		print "Compiled exclusion list"
+
+	l = launcher(opts)
+	try:
+		for t in torun:
+			global arch
+
+			if excl and excl.match(t):
+				print "Skipping %s (exclude)" % t
+				continue
+
+			tdesc = tlist.get(t, default_test) or default_test
+			if tdesc.get('arch', arch) != arch:
+				print "Skipping %s (arch %s)" % (t, tdesc['arch'])
+				continue
+
+			feat = tdesc.get('feature', None)
+			if feat:
+				if not features.has_key(feat):
+					print "Checking feature %s" % feat
+					features[feat] = criu_cli.check(feat)
+
+				if not features[feat]:
+					print "Skipping %s (no %s feature)" % (t, feat)
+					continue
+
+			if zdtm_test.checkskip(t):
+				print "Skipping %s (self)" % t
+				continue
+
+			test_flavs = tdesc.get('flavor', 'h ns uns').split()
+			opts_flavs = (opts['flavor'] or 'h,ns,uns').split(',')
+			run_flavs = set(test_flavs) & set(opts_flavs)
+
+			if run_flavs:
+				l.run_test(t, tdesc, run_flavs)
+	finally:
+		l.finish()
+
+def list_tests(opts, tlist):
+	for t in tlist:
+		print t
+
+#
+# main() starts here
+#
+
+if os.environ.has_key('ZDTM_CT_TEST_INFO'):
+	tinfo = eval(os.environ['ZDTM_CT_TEST_INFO'])
+	do_run_test(tinfo[0], tinfo[1], tinfo[2], tinfo[3])
+	sys.exit(0)
+
+p = argparse.ArgumentParser("ZDTM test suite")
+p.add_argument("--debug", help = "Print what's being executed", action = 'store_true')
+
+sp = p.add_subparsers(help = "Use --help for list of actions")
+
+rp = sp.add_parser("run", help = "Run test(s)")
+rp.set_defaults(action = run_tests)
+rp.add_argument("-a", "--all", action = 'store_true')
+rp.add_argument("-t", "--test", help = "Test name", action = 'append')
+rp.add_argument("-f", "--flavor", help = "Flavor to run")
+rp.add_argument("-x", "--exclude", help = "Exclude tests from --all run", action = 'append')
+
+rp.add_argument("--pre", help = "Do some pre-dumps before dump")
+rp.add_argument("--nocr", help = "Do not CR anything, just check test works", action = 'store_true')
+rp.add_argument("--norst", help = "Don't restore tasks, leave them running after dump", action = 'store_true')
+rp.add_argument("--iters", help = "Do CR cycle several times before check")
+
+rp.add_argument("--page-server", help = "Use page server dump", action = 'store_true')
+rp.add_argument("-p", "--parallel", help = "Run test in parallel")
+
+lp = sp.add_parser("list", help = "List tests")
+lp.set_defaults(action = list_tests)
+
+opts = vars(p.parse_args())
+tlist = yaml.load(open("zdtm.list"))
+
+if opts['debug']:
+	sys.settrace(traceit)
+
+opts['action'](opts, tlist)