crtools: preload libraries (v2)

Libraries (plugins) is going to be used for dumping and restoring
external dependencies (e.g. dbus, systemd journal sockets, charecter
devices, etc)

A plugin can have the cr_plugin_init() and cr_plugin_fini functions for
initialization and deinialization.

criu-plugin.h contains all things, which can be used in plugins.

v2: rename lib to plugin
v3: add a default value for a plugin path.
Signed-off-by: Andrey Vagin <avagin@openvz.org>
Signed-off-by: Pavel Emelyanov <xemul@parallels.com>

Makefile

@@ -82,7 +82,7 @@ cflags-y		+= -iquote $(ARCH_DIR) -iquote $(ARCH_DIR)/include
 cflags-y		+= -fno-strict-aliasing
 export cflags-y
 
-LIBS		:= -lrt -lpthread -lprotobuf-c
+LIBS		:= -lrt -lpthread -lprotobuf-c -ldl
 
 DEFINES		+= -D_FILE_OFFSET_BITS=64
 DEFINES		+= -D_GNU_SOURCE
@@ -167,7 +167,7 @@ PROGRAM-BUILTINS	+= $(ARCH_DIR)/vdso-pie.o
 
 $(PROGRAM): $(SYSCALL-LIB) $(ARCH-LIB) $(PROGRAM-BUILTINS)
 	$(E) "  LINK    " $@
-	$(Q) $(CC) $(CFLAGS) $^ $(LIBS) $(LDFLAGS) -o $@
+	$(Q) $(CC) $(CFLAGS) $^ $(LIBS) $(LDFLAGS) -rdynamic -o $@
 
 zdtm: all
 	$(Q) $(MAKE) -C test/zdtm all

Makefile.crtools

@@ -58,6 +58,7 @@ obj-y	+= sigframe.o
 obj-y	+= arch/$(ARCH)/vdso.o
 obj-y	+= cr-service.o
 obj-y	+= sd-daemon.o
+obj-y	+= plugin.o
 
 ifneq ($(MAKECMDGOALS),clean)
 incdeps := y

cr-dump.c

@@ -70,6 +70,7 @@
 #include "vdso.h"
 #include "vma.h"
 #include "cr-service.h"
+#include "plugin.h"
 
 #include "asm/dump.h"
 
@@ -1670,6 +1671,9 @@ int cr_dump_tasks(pid_t pid)
 	if (init_stats(DUMP_STATS))
 		goto err;
 
+	if (cr_plugin_init())
+		goto err;
+
 	if (kerndat_init())
 		goto err;
 
@@ -1753,6 +1757,8 @@ err:
 
 	close_cr_fdset(&glob_fdset);
 
+	cr_plugin_fini();
+
 	if (!ret) {
 		/*
 		 * It might be a migration case, where we're asked

cr-restore.c

@@ -67,6 +67,7 @@
 #include "vma.h"
 #include "kerndat.h"
 #include "rst-malloc.h"
+#include "plugin.h"
 
 #include "parasite-syscall.h"
 
@@ -1617,33 +1618,41 @@ static int prepare_task_entries()
 
 int cr_restore_tasks(void)
 {
-	if (check_img_inventory() < 0)
+	int ret = -1;
+
+	if (cr_plugin_init())
 		return -1;
 
+	if (check_img_inventory() < 0)
+		goto err;
+
 	if (init_stats(RESTORE_STATS))
-		return -1;
+		goto err;
 
 	if (kerndat_init_rst())
-		return -1;
+		goto err;
 
 	timing_start(TIME_RESTORE);
 
 	if (cpu_init() < 0)
-		return -1;
+		goto err;
 
 	if (vdso_init())
-		return -1;
+		goto err;
 
 	if (prepare_task_entries() < 0)
-		return -1;
+		goto err;
 
 	if (prepare_pstree() < 0)
-		return -1;
+		goto err;
 
 	if (crtools_prepare_shared() < 0)
-		return -1;
+		goto err;
 
-	return restore_root_task(root_item);
+	ret = restore_root_task(root_item);
+err:
+	cr_plugin_fini();
+	return ret;
 }
 
 static long restorer_get_vma_hint(pid_t pid, struct list_head *tgt_vma_list,

crtools.c

@@ -16,6 +16,8 @@
 #include <netinet/in.h>
 #include <arpa/inet.h>
 
+#include <dlfcn.h>
+
 #include "asm/types.h"
 
 #include "compiler.h"
@@ -31,6 +33,7 @@
 #include "tty.h"
 #include "file-lock.h"
 #include "cr-service.h"
+#include "plugin.h"
 
 struct cr_options opts;
 
@@ -95,7 +98,7 @@ int main(int argc, char *argv[])
 		return 1;
 
 	while (1) {
-		static const char short_opts[] = "dsRf:t:p:hcD:o:n:v::xVr:jlW:";
+		static const char short_opts[] = "dsRf:t:p:hcD:o:n:v::xVr:jlW:L:";
 		static struct option long_opts[] = {
 			{ "tree", required_argument, 0, 't' },
 			{ "pid", required_argument, 0, 'p' },
@@ -130,6 +133,7 @@ int main(int argc, char *argv[])
 			{ "ms", no_argument, 0, 54},
 			{ "track-mem", no_argument, 0, 55},
 			{ "auto-dedup", no_argument, 0, 56},
+			{ "libdir", required_argument, 0, 'L'},
 			{ },
 		};
 
@@ -275,6 +279,13 @@ int main(int argc, char *argv[])
 		case 54:
 			opts.check_ms_kernel = true;
 			break;
+		case 'L':
+			opts.libdir = strdup(optarg);
+			if (opts.libdir == NULL) {
+				pr_perror("Can't allocate memory");
+				return -1;
+			}
+			break;
 		case 'V':
 			pr_msg("Version: %s\n", CRIU_VERSION);
 			if (strcmp(CRIU_GITID, "0"))
@@ -420,6 +431,7 @@ usage:
 "  --action-script FILE  add an external action script\n"
 "  -j|--" OPT_SHELL_JOB "        allow to dump and restore shell jobs\n"
 "  -l|--" OPT_FILE_LOCKS "       handle file locks, for safety, only used for container\n"
+"  -L|--libdir           path to a plugin directory (by default " CR_PLUGIN_DEFAULT ")\n"
 "\n"
 "* Logging:\n"
 "  -o|--log-file FILE    log file name\n"

include/cr_options.h

@@ -29,6 +29,7 @@ struct cr_options {
 	char			*pidfile;
 	struct list_head	veth_pairs;
 	struct list_head	scripts;
+	char			*libdir;
 	bool			use_page_server;
 	unsigned short		ps_port;
 	char			*addr;

include/criu-plugin.h

+/*
+    This file defines types and macros for CRIU plugins.
+    Copyright (C) 2013 Parallels, Inc
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+#ifndef __CRIU_PLUGIN_H__
+#define __CRIU_PLUGIN_H__
+
+typedef int (cr_plugin_init_t)(void);
+typedef void (cr_plugin_fini_t)(void);
+
+#endif

include/plugin.h

+#ifndef __CR_PLUGIN_H__
+#define __CR_PLUGIN_H__
+
+#include "criu-plugin.h"
+
+#define CR_PLUGIN_DEFAULT "/var/lib/criu/"
+
+void cr_plugin_fini(void);
+int cr_plugin_init(void);
+#endif

plugin.c

+#include <stdlib.h>
+#include <string.h>
+#include <dirent.h>
+#include <errno.h>
+#include <stdio.h>
+#include <dlfcn.h>
+#include <unistd.h>
+
+#include "cr_options.h"
+#include "plugin.h"
+#include "log.h"
+#include "xmalloc.h"
+
+struct cr_plugin_entry {
+	union {
+		cr_plugin_fini_t *cr_fini;
+	};
+
+	struct cr_plugin_entry *next;
+};
+
+struct cr_plugins {
+	struct cr_plugin_entry *cr_fini;
+};
+
+struct cr_plugins cr_plugins;
+
+static int cr_lib_load(char *path)
+{
+	struct cr_plugin_entry *ce;
+	cr_plugin_init_t *f_init;
+	cr_plugin_fini_t *f_fini;
+	void *h;
+
+	h = dlopen(path, RTLD_LAZY);
+	if (h == NULL) {
+		pr_err("Unable to load %s: %s", path, dlerror());
+		return -1;
+	}
+
+	ce = NULL;
+	f_fini = dlsym(h, "cr_plugin_fini");
+	if (f_fini) {
+		ce = xmalloc(sizeof(struct cr_plugin_entry));
+		if (ce == NULL)
+			return -1;
+		ce->cr_fini = f_fini;
+	}
+
+	f_init = dlsym(h, "cr_plugin_init");
+	if (f_init && f_init()) {
+		xfree(ce);
+		return -1;
+	}
+
+	if (ce) {
+		ce->next = cr_plugins.cr_fini;
+		cr_plugins.cr_fini = ce;
+	}
+
+	return 0;
+}
+
+void cr_plugin_fini(void)
+{
+	struct cr_plugin_entry *ce;
+
+	while (cr_plugins.cr_fini) {
+		ce = cr_plugins.cr_fini;
+		cr_plugins.cr_fini = cr_plugins.cr_fini->next;
+
+		ce->cr_fini();
+		xfree(ce);
+	}
+}
+
+int cr_plugin_init(void)
+{
+	int exit_code = -1;
+	char *path;
+	DIR *d;
+
+	memset(&cr_plugins, 0, sizeof(cr_plugins));
+
+	if (opts.libdir == NULL) {
+		path = getenv("CRIU_LIBS_DIR");
+		if (path) {
+			opts.libdir = strdup(path);
+		} else {
+			if (access(CR_PLUGIN_DEFAULT, F_OK))
+				return 0;
+			opts.libdir = strdup(CR_PLUGIN_DEFAULT);
+		}
+		if (opts.libdir == NULL) {
+			pr_perror("Can't allocate memory");
+			return -1;
+		}
+	}
+
+	d = opendir(opts.libdir);
+	if (d == NULL) {
+		pr_perror("Unable to open directory %s", opts.libdir);
+		return -1;
+	}
+
+	while (1) {
+		char path[PATH_MAX];
+		struct dirent *de;
+		int len;
+
+		errno = 0;
+		de = readdir(d);
+		if (de == NULL) {
+			if (errno == 0)
+				break;
+			pr_perror("Unable to read the libraries directory");
+			goto err;
+		}
+
+		len = strlen(de->d_name);
+
+		if (len < 3 || strncmp(de->d_name + len - 3, ".so", 3))
+			continue;
+
+		snprintf(path, sizeof(path), "%s/%s", opts.libdir, de->d_name);
+
+		if (cr_lib_load(path))
+			goto err;
+	}
+
+	exit_code = 0;
+err:
+	closedir(d);
+
+	if (exit_code)
+		cr_plugin_fini();
+
+	return exit_code;
+}