vdso: Keep {vvar, vdso} sizes in symtable instead of end address

The plan is to keep boot-persistent vdso properties in symtable,
to omit parsing it in each invocation of criu.
As sizes of vdso/vvar are being stable on the same kernel,
move them into symtable, substituting end addresses.
Begin/end addresses are randomized by ASLR so there is no point
in storing them in kdat.
Reviewed-by: Cyrill Gorcunov <gorcunov@openvz.org>
Signed-off-by: Dmitry Safonov <dsafonov@virtuozzo.com>
Signed-off-by: Andrei Vagin <avagin@virtuozzo.com>

criu/cr-restore.c

@@ -3089,9 +3089,9 @@ static int sigreturn_restore(pid_t pid, struct task_restore_args *task_args, uns
 	/*
 	 * Figure out how much memory runtime vdso and vvar will need.
 	 */
-	vdso_rt_size = vdso_vma_size(&vdso_symtab_rt);
-	if (vdso_rt_size && vvar_vma_size(&vdso_symtab_rt))
-		vdso_rt_size += ALIGN(vvar_vma_size(&vdso_symtab_rt), PAGE_SIZE);
+	vdso_rt_size = vdso_symtab_rt.vdso_size;
+	if (vdso_rt_size && vdso_symtab_rt.vvar_size)
+		vdso_rt_size += ALIGN(vdso_symtab_rt.vvar_size, PAGE_SIZE);
 	task_args->bootstrap_len += vdso_rt_size;
 #endif
 

criu/include/asm-generic/vdso.h

@@ -4,9 +4,12 @@
 #define VDSO_PROT               (PROT_READ | PROT_EXEC)
 #define VVAR_PROT               (PROT_READ)
 
-#define VDSO_BAD_ADDR           (-1ul)
-#define VVAR_BAD_ADDR           VDSO_BAD_ADDR
-#define VDSO_BAD_PFN            (-1ull)
-#define VVAR_BAD_PFN            VDSO_BAD_PFN
+/* Just in case of LPAE system PFN is u64. */
+#define VDSO_BAD_PFN		(-1ull)
+#define VVAR_BAD_PFN		(-1ull)
+#define VDSO_BAD_ADDR		(-1ul)
+#define VVAR_BAD_ADDR		(-1ul)
+#define VDSO_BAD_SIZE		(-1ul)
+#define VVAR_BAD_SIZE		(-1ul)
 
 #endif /* __CR_ASM_GENERIC_VDSO_H__ */

criu/include/util-vdso.h

@@ -28,10 +28,10 @@ struct vdso_symbol {
 };
 
 struct vdso_symtable {
-	unsigned long		vma_start;
-	unsigned long		vma_end;
+	unsigned long		vdso_start;
+	unsigned long		vdso_size;
 	unsigned long		vvar_start;
-	unsigned long		vvar_end;
+	unsigned long		vvar_size;
 	struct vdso_symbol	symbols[VDSO_SYMBOL_MAX];
 };
 
@@ -39,10 +39,10 @@ struct vdso_symtable {
 
 #define VDSO_SYMTABLE_INIT						\
 	{								\
-		.vma_start	= VDSO_BAD_ADDR,			\
-		.vma_end	= VDSO_BAD_ADDR,			\
+		.vdso_start	= VDSO_BAD_ADDR,			\
+		.vdso_size	= VDSO_BAD_SIZE,			\
 		.vvar_start	= VVAR_BAD_ADDR,			\
-		.vvar_end	= VVAR_BAD_ADDR,			\
+		.vvar_size	= VVAR_BAD_SIZE,			\
 		.symbols		= {				\
 			[0 ... VDSO_SYMBOL_MAX - 1] =			\
 				(struct vdso_symbol)VDSO_SYMBOL_INIT,	\
@@ -77,17 +77,6 @@ struct vdso_symtable {
 
 #endif /* CONFIG_VDSO_32 */
 
-/* Size of VMA associated with vdso */
-static inline unsigned long vdso_vma_size(struct vdso_symtable *t)
-{
-	return t->vma_end - t->vma_start;
-}
-
-static inline unsigned long vvar_vma_size(struct vdso_symtable *t)
-{
-	return t->vvar_end - t->vvar_start;
-}
-
 #if defined(CONFIG_VDSO_32)
 # define vdso_fill_symtable vdso_fill_symtable_compat
 #endif

criu/include/vdso.h

@@ -19,7 +19,7 @@ extern int parasite_fixup_vdso(struct parasite_ctl *ctl, pid_t pid,
 			       struct vm_area_list *vma_area_list);
 
 #ifdef CONFIG_COMPAT
-void compat_vdso_helper(struct vdso_symtable *native, int pipe_fd,
+extern void compat_vdso_helper(struct vdso_symtable *native, int pipe_fd,
 		int err_fd, void *vdso_buf, size_t buf_size);
 #endif
 

criu/pie/parasite-vdso.c

@@ -46,25 +46,25 @@ int vdso_do_park(struct vdso_symtable *sym_rt, unsigned long park_at, unsigned l
 {
 	int ret;
 
-	BUG_ON((vdso_vma_size(sym_rt) + vvar_vma_size(sym_rt)) < park_size);
+	BUG_ON((sym_rt->vdso_size + sym_rt->vvar_size) < park_size);
 
 	if (sym_rt->vvar_start != VDSO_BAD_ADDR) {
-		if (sym_rt->vma_start < sym_rt->vvar_start) {
-			ret  = vdso_remap("rt-vdso", sym_rt->vma_start,
-					  park_at, vdso_vma_size(sym_rt));
-			park_at += vdso_vma_size(sym_rt);
+		if (sym_rt->vdso_start < sym_rt->vvar_start) {
+			ret  = vdso_remap("rt-vdso", sym_rt->vdso_start,
+					  park_at, sym_rt->vdso_size);
+			park_at += sym_rt->vdso_size;
 			ret |= vdso_remap("rt-vvar", sym_rt->vvar_start,
-					  park_at, vvar_vma_size(sym_rt));
+					  park_at, sym_rt->vvar_size);
 		} else {
 			ret  = vdso_remap("rt-vvar", sym_rt->vvar_start,
-					  park_at, vvar_vma_size(sym_rt));
-			park_at += vvar_vma_size(sym_rt);
-			ret |= vdso_remap("rt-vdso", sym_rt->vma_start,
-					  park_at, vdso_vma_size(sym_rt));
+					  park_at, sym_rt->vvar_size);
+			park_at += sym_rt->vvar_size;
+			ret |= vdso_remap("rt-vdso", sym_rt->vdso_start,
+					  park_at, sym_rt->vdso_size);
 		}
 	} else
-		ret = vdso_remap("rt-vdso", sym_rt->vma_start,
-				 park_at, vdso_vma_size(sym_rt));
+		ret = vdso_remap("rt-vdso", sym_rt->vdso_start,
+				 park_at, sym_rt->vdso_size);
 	return ret;
 }
 
@@ -157,7 +157,7 @@ int vdso_proxify(struct vdso_symtable *sym_rt, unsigned long vdso_rt_parked_at,
 	 *    b) Symbols offsets must match
 	 *    c) Have same number of vDSO zones
 	 */
-	if (vma_entry_len(vma_vdso) == vdso_vma_size(sym_rt)) {
+	if (vma_entry_len(vma_vdso) == sym_rt->vdso_size) {
 		size_t i;
 
 		for (i = 0; i < ARRAY_SIZE(s.symbols); i++) {
@@ -167,9 +167,9 @@ int vdso_proxify(struct vdso_symtable *sym_rt, unsigned long vdso_rt_parked_at,
 
 		if (i == ARRAY_SIZE(s.symbols)) {
 			if (vma_vvar && sym_rt->vvar_start != VVAR_BAD_ADDR) {
-				remap_rt = (vvar_vma_size(sym_rt) == vma_entry_len(vma_vvar));
+				remap_rt = (sym_rt->vvar_size == vma_entry_len(vma_vvar));
 				if (remap_rt) {
-					long delta_rt = sym_rt->vvar_start - sym_rt->vma_start;
+					long delta_rt = sym_rt->vvar_start - sym_rt->vdso_start;
 					long delta_this = vma_vvar->start - vma_vdso->start;
 
 					remap_rt = (delta_rt ^ delta_this) < 0 ? false : true;
@@ -212,16 +212,16 @@ int vdso_proxify(struct vdso_symtable *sym_rt, unsigned long vdso_rt_parked_at,
 			}
 
 			if (vma_vdso->start < vma_vvar->start) {
-				ret  = vdso_remap("rt-vdso", vdso_rt_parked_at, vma_vdso->start, vdso_vma_size(sym_rt));
-				vdso_rt_parked_at += vdso_vma_size(sym_rt);
-				ret |= vdso_remap("rt-vvar", vdso_rt_parked_at, vma_vvar->start, vvar_vma_size(sym_rt));
+				ret  = vdso_remap("rt-vdso", vdso_rt_parked_at, vma_vdso->start, sym_rt->vdso_size);
+				vdso_rt_parked_at += sym_rt->vdso_size;
+				ret |= vdso_remap("rt-vvar", vdso_rt_parked_at, vma_vvar->start, sym_rt->vvar_size);
 			} else {
-				ret  = vdso_remap("rt-vvar", vdso_rt_parked_at, vma_vvar->start, vvar_vma_size(sym_rt));
-				vdso_rt_parked_at += vvar_vma_size(sym_rt);
-				ret |= vdso_remap("rt-vdso", vdso_rt_parked_at, vma_vdso->start, vdso_vma_size(sym_rt));
+				ret  = vdso_remap("rt-vvar", vdso_rt_parked_at, vma_vvar->start, sym_rt->vvar_size);
+				vdso_rt_parked_at += sym_rt->vvar_size;
+				ret |= vdso_remap("rt-vdso", vdso_rt_parked_at, vma_vdso->start, sym_rt->vdso_size);
 			}
 		} else
-			ret = vdso_remap("rt-vdso", vdso_rt_parked_at, vma_vdso->start, vdso_vma_size(sym_rt));
+			ret = vdso_remap("rt-vdso", vdso_rt_parked_at, vma_vdso->start, sym_rt->vdso_size);
 
 		return ret;
 	}
@@ -237,8 +237,8 @@ int vdso_proxify(struct vdso_symtable *sym_rt, unsigned long vdso_rt_parked_at,
 	 * Don't forget to shift if vvar is before vdso.
 	 */
 	if (sym_rt->vvar_start != VDSO_BAD_ADDR &&
-	    sym_rt->vvar_start < sym_rt->vma_start)
-		vdso_rt_parked_at += vvar_vma_size(sym_rt);
+	    sym_rt->vvar_start < sym_rt->vdso_start)
+		vdso_rt_parked_at += sym_rt->vvar_size;
 
 	if (vdso_redirect_calls(vdso_rt_parked_at,
 				vma_vdso->start,
@@ -252,8 +252,8 @@ int vdso_proxify(struct vdso_symtable *sym_rt, unsigned long vdso_rt_parked_at,
 	 * routine we could detect this vdso and do not dump it, since
 	 * it's auto-generated every new session if proxy required.
 	 */
-	sys_mprotect((void *)vdso_rt_parked_at,  vdso_vma_size(sym_rt), PROT_WRITE);
+	sys_mprotect((void *)vdso_rt_parked_at,  sym_rt->vdso_size, PROT_WRITE);
 	vdso_put_mark((void *)vdso_rt_parked_at, vma_vdso->start, vma_vvar ? vma_vvar->start : VVAR_BAD_ADDR);
-	sys_mprotect((void *)vdso_rt_parked_at,  vdso_vma_size(sym_rt), VDSO_PROT);
+	sys_mprotect((void *)vdso_rt_parked_at,  sym_rt->vdso_size, VDSO_PROT);
 	return 0;
 }

criu/vdso-compat.c

@@ -36,24 +36,21 @@ static void exit_on(int ret, int err_fd, char *reason)
 void compat_vdso_helper(struct vdso_symtable *native, int pipe_fd,
 		int err_fd, void *vdso_buf, size_t buf_size)
 {
-	size_t vma_size;
 	void *vdso_addr;
 	long vdso_size;
 	long ret;
 
-	if (native->vma_start != VDSO_BAD_ADDR) {
-		vma_size = native->vma_end - native->vma_start;
-		ret = syscall(__NR_munmap, native->vma_start, vma_size);
+	if (native->vdso_start != VDSO_BAD_ADDR) {
+		ret = syscall(__NR_munmap, native->vdso_start, native->vdso_size);
 		exit_on(ret, err_fd, "Error: Failed to unmap native vdso\n");
 	}
 
 	if (native->vvar_start != VVAR_BAD_ADDR) {
-		vma_size = native->vvar_end - native->vvar_start;
-		ret = syscall(__NR_munmap, native->vvar_start, vma_size);
+		ret = syscall(__NR_munmap, native->vvar_start, native->vvar_size);
 		exit_on(ret, err_fd, "Error: Failed to unmap native vvar\n");
 	}
 
-	ret = syscall(__NR_arch_prctl, ARCH_MAP_VDSO_32, native->vma_start);
+	ret = syscall(__NR_arch_prctl, ARCH_MAP_VDSO_32, native->vdso_start);
 	if (ret < 0)
 		exit_on(ret, err_fd, "Error: ARCH_MAP_VDSO failed\n");
 

criu/vdso.c

@@ -28,9 +28,10 @@
 #endif
 #define LOG_PREFIX "vdso: "
 
-struct vdso_symtable vdso_sym_rt = VDSO_SYMTABLE_INIT;
 u64 vdso_pfn = VDSO_BAD_PFN;
-struct vdso_symtable vdso_compat_rt = VDSO_SYMTABLE_INIT;
+struct vdso_symtable vdso_sym_rt	= VDSO_SYMTABLE_INIT;
+struct vdso_symtable vdso_compat_rt	= VDSO_SYMTABLE_INIT;
+
 /*
  * The VMAs list might have proxy vdso/vvar areas left
  * from previous dump/restore cycle so we need to detect
@@ -266,19 +267,19 @@ static int vdso_parse_maps(pid_t pid, struct vdso_symtable *s)
 		}
 
 		if (has_vdso) {
-			if (s->vma_start != VDSO_BAD_ADDR) {
+			if (s->vdso_start != VDSO_BAD_ADDR) {
 				pr_err("Got second vDSO entry\n");
 				goto err;
 			}
-			s->vma_start = start;
-			s->vma_end = end;
+			s->vdso_start = start;
+			s->vdso_size = end - start;
 		} else {
 			if (s->vvar_start != VVAR_BAD_ADDR) {
 				pr_err("Got second VVAR entry\n");
 				goto err;
 			}
 			s->vvar_start = start;
-			s->vvar_end = end;
+			s->vvar_size = end - start;
 		}
 	}
 
@@ -290,6 +291,8 @@ err:
 
 static int validate_vdso_addr(struct vdso_symtable *s)
 {
+	unsigned long vdso_end = s->vdso_start + s->vdso_size;
+	unsigned long vvar_end = s->vvar_start + s->vvar_size;
 	/*
 	 * Validate its structure -- for new vDSO format the
 	 * structure must be like
@@ -303,10 +306,10 @@ static int validate_vdso_addr(struct vdso_symtable *s)
 	 * 7fffc3504000-7fffc3506000 r-xp 00000000 00:00 0 [vdso]
 	 *
 	 */
-	if (s->vma_start != VDSO_BAD_ADDR) {
+	if (s->vdso_start != VDSO_BAD_ADDR) {
 		if (s->vvar_start != VVAR_BAD_ADDR) {
-			if (s->vma_end != s->vvar_start &&
-			    s->vvar_end != s->vma_start) {
+			if (vdso_end != s->vvar_start &&
+			    vvar_end != s->vdso_start) {
 				pr_err("Unexpected rt vDSO area bounds\n");
 				return -1;
 			}
@@ -325,15 +328,15 @@ static int vdso_fill_self_symtable(struct vdso_symtable *s)
 	if (vdso_parse_maps(PROC_SELF, s))
 		return -1;
 
-	if (vdso_fill_symtable(s->vma_start, s->vma_end - s->vma_start, s))
+	if (vdso_fill_symtable(s->vdso_start, s->vdso_size, s))
 		return -1;
 
 	if (validate_vdso_addr(s))
 		return -1;
 
 	pr_debug("rt [vdso] %lx-%lx [vvar] %lx-%lx\n",
-		 s->vma_start, s->vma_end,
-		 s->vvar_start, s->vvar_end);
+		 s->vdso_start, s->vdso_start + s->vdso_size,
+		 s->vvar_start, s->vvar_start + s->vvar_size);
 
 	return 0;
 }
@@ -391,8 +394,8 @@ static int vdso_mmap_compat(struct vdso_symtable *native,
 		pr_perror("Failed to kill(SIGCONT) for compat vdso helper\n");
 		goto out_kill;
 	}
-	if (write(fds[1], &compat->vma_start, sizeof(void *)) !=
-			sizeof(compat->vma_start)) {
+	if (write(fds[1], &compat->vdso_start, sizeof(void *)) !=
+			sizeof(compat->vdso_start)) {
 		pr_perror("Failed write to pipe\n");
 		goto out_kill;
 	}
@@ -437,14 +440,14 @@ static int vdso_fill_compat_symtable(struct vdso_symtable *native,
 	}
 
 	if (vdso_fill_symtable_compat((uintptr_t)vdso_mmap,
-				compat->vma_end - compat->vma_start, compat)) {
+				compat->vdso_size, compat)) {
 		pr_err("Failed to parse mmaped compatible vdso blob\n");
 		goto out_unmap;
 	}
 
 	pr_debug("compat [vdso] %lx-%lx [vvar] %lx-%lx\n",
-		 compat->vma_start, compat->vma_end,
-		 compat->vvar_start, compat->vvar_end);
+		 compat->vdso_start, compat->vdso_start + compat->vdso_size,
+		 compat->vvar_start, compat->vvar_start + compat->vvar_size);
 	ret = 0;
 
 out_unmap:
@@ -475,7 +478,7 @@ int vdso_init(void)
 
 	if (kdat.pmap != PM_FULL)
 		pr_info("VDSO detection turned off\n");
-	else if (vaddr_to_pfn(vdso_sym_rt.vma_start, &vdso_pfn))
+	else if (vaddr_to_pfn(vdso_sym_rt.vdso_start, &vdso_pfn))
 		return -1;
 
 	return 0;