vdso: Remap runtime vdso copy to safe place

Runtime vdso need to be kept in some safe place when all
self-vmas are unmapped. So we reserve space for it in restorer
blob area and then remap it into. It's quite important to do
a remap here rather than data copy because otherwise pfn
of vdso disappear and in future we won't be able to detect
vdso are on dumping stage.
Signed-off-by: Cyrill Gorcunov <gorcunov@openvz.org>
Signed-off-by: Pavel Emelyanov <xemul@parallels.com>

arch/arm/include/asm/vdso.h

@@ -7,5 +7,6 @@ struct vdso_symtable;
 
 extern int vdso_redirect_calls(void *base_to, void *base_from, struct vdso_symtable *to, struct vdso_symtable *from);
 extern int vdso_fill_symtable(char *mem, size_t size,struct vdso_symtable *t);
+extern int vdso_remap(char *who, unsigned long from, unsigned long to, size_t size);
 
 #endif /* __CR_ASM_VDSO_H__ */

arch/arm/vdso-pie.c

@@ -19,3 +19,8 @@ int vdso_fill_symtable(char *mem, size_t size, struct vdso_symtable *t)
 {
 	return 0;
 }
+
+int vdso_remap(char *who, unsigned long from, unsigned long to, size_t size)
+{
+	return 0;
+}

arch/x86/include/asm/vdso.h

@@ -7,5 +7,6 @@ struct vdso_symtable;
 
 extern int vdso_redirect_calls(void *base_to, void *base_from, struct vdso_symtable *to, struct vdso_symtable *from);
 extern int vdso_fill_symtable(char *mem, size_t size,struct vdso_symtable *t);
+extern int vdso_remap(char *who, unsigned long from, unsigned long to, size_t size);
 
 #endif /* __CR_ASM_VDSO_H__ */

arch/x86/vdso-pie.c

@@ -224,3 +224,19 @@ int vdso_fill_symtable(char *mem, size_t size, struct vdso_symtable *t)
 err:
 	return -1;
 }
+
+int vdso_remap(char *who, unsigned long from, unsigned long to, size_t size)
+{
+	unsigned long addr;
+
+	pr_debug("Remap %s %lx -> %lx\n", who, from, to);
+
+	addr = sys_mremap(from, size, size, MREMAP_MAYMOVE | MREMAP_FIXED, to);
+	if (addr != to) {
+		pr_err("Unable to remap %lx -> %lx %lx\n",
+		       from, to, addr);
+		return -1;
+	}
+
+	return 0;
+}

cr-restore.c

@@ -1796,6 +1796,10 @@ static int sigreturn_restore(pid_t pid, CoreEntry *core)
 	int *siginfo_priv_nr;
 	unsigned long siginfo_size = 0;
 
+	unsigned long vdso_rt_vma_size = 0;
+	unsigned long vdso_rt_size = 0;
+	unsigned long vdso_rt_delta = 0;
+
 	struct vm_area_list self_vmas;
 	int i;
 
@@ -1856,6 +1860,15 @@ static int sigreturn_restore(pid_t pid, CoreEntry *core)
 				rst_tcp_socks_size +
 				siginfo_size;
 
+	/*
+	 * Figure out how much memory runtime vdso will need.
+	 */
+	vdso_rt_vma_size = vdso_vma_size(&vdso_sym_rt);
+	if (vdso_rt_vma_size) {
+		vdso_rt_delta = ALIGN(restore_bootstrap_len, PAGE_SIZE) - restore_bootstrap_len;
+		vdso_rt_size = vdso_rt_vma_size + vdso_rt_delta;
+	}
+
 	/*
 	 * Restorer is a blob (code + args) that will get mapped in some
 	 * place, that should _not_ intersect with both -- current mappings
@@ -1868,15 +1881,16 @@ static int sigreturn_restore(pid_t pid, CoreEntry *core)
 	 */
 
 	exec_mem_hint = restorer_get_vma_hint(pid, &rst_vmas.h, &self_vmas.h,
-					      restore_bootstrap_len);
+					      restore_bootstrap_len +
+					      vdso_rt_size);
 	if (exec_mem_hint == -1) {
 		pr_err("No suitable area for task_restore bootstrap (%ldK)\n",
-		       restore_bootstrap_len);
+		       restore_bootstrap_len + vdso_rt_size);
 		goto err;
 	}
 
 	pr_info("Found bootstrap VMA hint at: 0x%lx (needs ~%ldK)\n", exec_mem_hint,
-			KBYTES(restore_bootstrap_len));
+			KBYTES(restore_bootstrap_len + vdso_rt_size));
 
 	ret = remap_restorer_blob((void *)exec_mem_hint);
 	if (ret < 0)
@@ -2051,6 +2065,16 @@ static int sigreturn_restore(pid_t pid, CoreEntry *core)
 	memcpy(&task_args->t->blk_sigset, &core->tc->blk_sigset, sizeof(k_rtsigset_t));
 	task_args->t->has_blk_sigset	= true;
 
+	/*
+	 * Restorer needs own copy of vdso parameters. Runtime
+	 * vdso must be kept non intersecting with anything else,
+	 * since we need it being accessible even when own
+	 * self-vmas are unmaped.
+	 */
+	mem += (unsigned long)rst_tcp_socks_size;
+	task_args->vdso_rt_parked_at = (unsigned long)mem + vdso_rt_delta;
+	task_args->vdso_sym_rt = vdso_sym_rt;
+
 	/*
 	 * Adjust stack.
 	 */

include/restorer.h

@@ -13,6 +13,8 @@
 #include "util.h"
 #include "crtools.h"
 
+#include "vdso.h"
+
 #include "protobuf/mm.pb-c.h"
 #include "protobuf/vma.pb-c.h"
 #include "protobuf/creds.pb-c.h"
@@ -132,6 +134,9 @@ struct task_restore_core_args {
 
 	struct rst_tcp_sock		*rst_tcp_socks;
 	int				rst_tcp_socks_size;
+
+	struct vdso_symtable		vdso_sym_rt;		/* runtime vdso symbols */
+	unsigned long			vdso_rt_parked_at;	/* safe place to keep vdso */
 } __aligned(sizeof(long));
 
 #define SHMEMS_SIZE	4096

pie/restorer.c

@@ -521,6 +521,24 @@ long __export_restore_task(struct task_restore_core_args *args)
 
 		pr_debug("Examine %"PRIx64"-%"PRIx64"\n", vma_entry->start, vma_entry->end);
 
+		/*
+		 * Park runtime vdso at safe place, thus we can access it
+		 * during restore of targets vma, it's quite important to
+		 * remap it instead of copying to save page frame number
+		 * associated with vdso, we will use it if there is subsequent
+		 * checkpoint done on previously restored program.
+		 */
+		if (vma_entry_is(vma_entry, VMA_AREA_VDSO)) {
+			BUG_ON(vma_entry->start != args->vdso_sym_rt.vma_start);
+			BUG_ON(vma_entry_len(vma_entry) != vdso_vma_size(&args->vdso_sym_rt));
+
+			if (vdso_remap("rt-vdso", vma_entry->start,
+				       args->vdso_rt_parked_at,
+				       vdso_vma_size(&args->vdso_sym_rt)))
+				goto core_restore_end;
+			continue;
+		}
+
 		if (addr < args->premmapped_addr) {
 			if (vma_entry->end >= args->premmapped_addr)
 				len = args->premmapped_addr - addr;