vdso.c: share the PIE part of the vDSO proxy machinery between all architectures

This patch splits the file arch/x86/vdso-pie.c into machine-dependent
and machine-independent parts by moving the routines vdso_fill_symtable(),
vdso_proxify(), and vdso_remap() to the file pie/vdso.c.

The ARM version of the routines is moved to the source pie/vdso-stub.c
to provide the vDSO proxy stub implementation for architectures
that don't provide the vDSO.
Signed-off-by: Alexander Kartashov <alekskartashov@parallels.com>
Looks-good-to: Cyrill Gorcunov <gorcunov@openvz.org>
Reviewed-by: Christopher Covington <cov@codeaurora.org>
Signed-off-by: Pavel Emelyanov <xemul@parallels.com>

Makefile

@@ -180,6 +180,10 @@ PROGRAM-BUILTINS	+= built-in.o
 $(ARCH_DIR)/vdso-pie.o: pie
 	$(Q) $(MAKE) $(build)=pie $(ARCH_DIR)/vdso-pie.o
 PROGRAM-BUILTINS	+= $(ARCH_DIR)/vdso-pie.o
+pie/$(VDSO_O): pie
+	$(Q) $(MAKE) $(build)=pie pie/$(VDSO_O)
+PROGRAM-BUILTINS	+= pie/$(VDSO_O)
+
 
 $(PROGRAM): $(SYSCALL-LIB) $(ARCH-LIB) $(PROGRAM-BUILTINS)
 	$(E) "  LINK    " $@

arch/arm/include/asm/vdso.h

@@ -12,9 +12,6 @@ struct vm_area_list;
 #define VDSO_SYMBOL_MAX 1
 
 extern int vdso_redirect_calls(void *base_to, void *base_from, struct vdso_symtable *to, struct vdso_symtable *from);
-extern int vdso_fill_symtable(char *mem, size_t size,struct vdso_symtable *t);
-extern int vdso_remap(char *who, unsigned long from, unsigned long to, size_t size);
-extern int vdso_proxify(char *who, struct vdso_symtable *sym_rt, VmaEntry *vma_entry, unsigned long vdso_rt_parked_at);
 extern int parasite_fixup_vdso(struct parasite_ctl *ctl, pid_t pid,
 			       struct vm_area_list *vma_area_list);
 

arch/arm/vdso-pie.c

@@ -15,18 +15,3 @@ int vdso_redirect_calls(void *base_to, void *base_from,
 {
 	return 0;
 }
-
-int vdso_fill_symtable(char *mem, size_t size, struct vdso_symtable *t)
-{
-	return 0;
-}
-
-int vdso_remap(char *who, unsigned long from, unsigned long to, size_t size)
-{
-	return 0;
-}
-
-int vdso_proxify(char *who, struct vdso_symtable *sym_rt, VmaEntry *vma, unsigned long vdso_rt_parked_at)
-{
-	return 0;
-}

arch/x86/include/asm/vdso.h

@@ -45,9 +45,6 @@ struct parasite_ctl;
 struct vm_area_list;
 
 extern int vdso_redirect_calls(void *base_to, void *base_from, struct vdso_symtable *to, struct vdso_symtable *from);
-extern int vdso_fill_symtable(char *mem, size_t size,struct vdso_symtable *t);
-extern int vdso_remap(char *who, unsigned long from, unsigned long to, size_t size);
-extern int vdso_proxify(char *who, struct vdso_symtable *sym_rt, VmaEntry *vma_entry, unsigned long vdso_rt_parked_at);
 extern int parasite_fixup_vdso(struct parasite_ctl *ctl, pid_t pid,
 			       struct vm_area_list *vma_area_list);
 

arch/x86/vdso-pie.c

@@ -56,240 +56,3 @@ int vdso_redirect_calls(void *base_to, void *base_from,
 
 	return 0;
 }
-
-static unsigned int get_symbol_index(char *symbol, char *symbols[], size_t size)
-{
-	unsigned int i;
-
-	for (i = 0; symbol && i < size; i++) {
-		if (!builtin_strcmp(symbol, symbols[i]))
-			return i;
-	}
-
-	return VDSO_SYMBOL_MAX;
-}
-
-/* Check if pointer is out-of-bound */
-static bool __ptr_oob(void *ptr, void *start, size_t size)
-{
-	void *end = (void *)((unsigned long)start + size);
-	return ptr > end || ptr < start;
-}
-
-int vdso_fill_symtable(char *mem, size_t size, struct vdso_symtable *t)
-{
-	Elf64_Ehdr *ehdr = (void *)mem;
-	Elf64_Shdr *shdr, *shdr_strtab;
-	Elf64_Shdr *shdr_dynsym;
-	Elf64_Shdr *shdr_dynstr;
-	Elf64_Phdr *phdr;
-	Elf64_Shdr *text;
-	Elf64_Sym *sym;
-
-	char *section_names, *dynsymbol_names;
-
-	unsigned long base = VDSO_BAD_ADDR;
-	unsigned int i, j, k;
-
-	DECLARE_VDSO(vdso_ident, vdso_symbols);
-
-	BUILD_BUG_ON(sizeof(vdso_ident) != sizeof(ehdr->e_ident));
-
-	pr_debug("Parsing at %lx %lx\n",
-		 (long)mem, (long)mem + (long)size);
-
-	/*
-	 * Make sure it's a file we support.
-	 */
-	if (builtin_memcmp(ehdr->e_ident, vdso_ident, sizeof(vdso_ident))) {
-		pr_debug("Elf header magic mismatch\n");
-		goto err;
-	}
-
-	/*
-	 * Figure out base virtual address.
-	 */
-	phdr = (void *)&mem[ehdr->e_phoff];
-	for (i = 0; i < ehdr->e_phnum; i++, phdr++) {
-		if (__ptr_oob(phdr, mem, size))
-			goto err;
-		if (phdr->p_type == PT_LOAD) {
-			base = phdr->p_vaddr;
-			break;
-		}
-	}
-	if (base != VDSO_BAD_ADDR) {
-		pr_debug("Base address %lx\n", base);
-	} else {
-		pr_debug("No base address found\n");
-		goto err;
-	}
-
-	/*
-	 * Where the section names lays.
-	 */
-	if (ehdr->e_shstrndx == SHN_UNDEF) {
-		pr_err("Section names are not found\n");
-		goto err;
-	}
-
-	shdr = (void *)&mem[ehdr->e_shoff];
-	shdr_strtab = &shdr[ehdr->e_shstrndx];
-	if (__ptr_oob(shdr_strtab, mem, size))
-		goto err;
-
-	section_names = (void *)&mem[shdr_strtab->sh_offset];
-	shdr_dynsym = shdr_dynstr = text = NULL;
-
-	for (i = 0; i < ehdr->e_shnum; i++, shdr++) {
-		if (__ptr_oob(shdr, mem, size))
-			goto err;
-		if (__ptr_oob(&section_names[shdr->sh_name], mem, size))
-			goto err;
-
-#if 0
-		pr_debug("section: %2d -> %s\n",
-			 i, &section_names[shdr->sh_name]);
-#endif
-
-		if (shdr->sh_type == SHT_DYNSYM &&
-		    builtin_strcmp(&section_names[shdr->sh_name],
-				   ".dynsym") == 0) {
-			shdr_dynsym = shdr;
-		} else if (shdr->sh_type == SHT_STRTAB &&
-			   builtin_strcmp(&section_names[shdr->sh_name],
-					  ".dynstr") == 0) {
-			shdr_dynstr = shdr;
-		} else if (shdr->sh_type == SHT_PROGBITS &&
-			   builtin_strcmp(&section_names[shdr->sh_name],
-					  ".text") == 0) {
-			text = shdr;
-		}
-	}
-
-	if (!shdr_dynsym || !shdr_dynstr || !text) {
-		pr_debug("No required sections found\n");
-		goto err;
-	}
-
-	dynsymbol_names = (void *)&mem[shdr_dynstr->sh_offset];
-	if (__ptr_oob(dynsymbol_names, mem, size)	||
-	    __ptr_oob(shdr_dynsym, mem, size)		||
-	    __ptr_oob(text, mem, size))
-		goto err;
-
-	/*
-	 * Walk over global symbols and choose ones we need.
-	 */
-	j = shdr_dynsym->sh_size / sizeof(*sym);
-	sym = (void *)&mem[shdr_dynsym->sh_offset];
-
-	for (i = 0; i < j; i++, sym++) {
-		if (__ptr_oob(sym, mem, size))
-			goto err;
-
-		if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL ||
-		    ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
-			continue;
-
-		if (__ptr_oob(&dynsymbol_names[sym->st_name], mem, size))
-			goto err;
-
-		k = get_symbol_index(&dynsymbol_names[sym->st_name],
-				     vdso_symbols,
-				     ARRAY_SIZE(vdso_symbols));
-		if (k != VDSO_SYMBOL_MAX) {
-			builtin_memcpy(t->symbols[k].name, vdso_symbols[k],
-				       sizeof(t->symbols[k].name));
-			t->symbols[k].offset = (unsigned long)sym->st_value - base;
-#if 0
-			pr_debug("symbol: %#-16lx %2d %s\n",
-				 t->symbols[k].offset, sym->st_shndx, t->symbols[k].name);
-#endif
-		}
-	}
-	return 0;
-err:
-	return -1;
-}
-
-int vdso_remap(char *who, unsigned long from, unsigned long to, size_t size)
-{
-	unsigned long addr;
-
-	pr_debug("Remap %s %lx -> %lx\n", who, from, to);
-
-	addr = sys_mremap(from, size, size, MREMAP_MAYMOVE | MREMAP_FIXED, to);
-	if (addr != to) {
-		pr_err("Unable to remap %lx -> %lx %lx\n",
-		       from, to, addr);
-		return -1;
-	}
-
-	return 0;
-}
-
-int vdso_proxify(char *who, struct vdso_symtable *sym_rt, VmaEntry *vma, unsigned long vdso_rt_parked_at)
-{
-	struct vdso_symtable s = VDSO_SYMTABLE_INIT;
-	size_t size = vma_entry_len(vma);
-	bool remap_rt = true;
-
-	/*
-	 * Find symbols in dumpee vdso.
-	 */
-	if (vdso_fill_symtable((void *)vma->start, size, &s))
-		return -1;
-
-	if (size == vdso_vma_size(sym_rt)) {
-		int i;
-
-		for (i = 0; i < ARRAY_SIZE(s.symbols); i++) {
-			if (s.symbols[i].offset != sym_rt->symbols[i].offset) {
-				remap_rt = false;
-				break;
-			}
-		}
-	} else
-		remap_rt = false;
-
-	/*
-	 * Easy case -- the vdso from image has same offsets and size
-	 * as runtime, so we simply remap runtime vdso to dumpee position
-	 * without generating any proxy.
-	 */
-	if (remap_rt) {
-		pr_info("Runtime vdso matches dumpee, remap inplace\n");
-
-		if (sys_munmap((void *)vma->start, size)) {
-			pr_err("Failed to unmap %s\n", who);
-			return -1;
-		}
-
-		return vdso_remap(who, vdso_rt_parked_at, vma->start, size);
-	}
-
-	/*
-	 * Now complex case -- we need to proxify calls. We redirect
-	 * calls from dumpee vdso to runtime vdso, making dumpee
-	 * to operate as proxy vdso.
-	 */
-	pr_info("Runtime vdso mismatches dumpee, generate proxy\n");
-
-	if (vdso_redirect_calls((void *)vdso_rt_parked_at,
-				(void *)vma->start,
-				sym_rt, &s)) {
-		pr_err("Failed to proxify dumpee contents\n");
-		return -1;
-	}
-
-	/*
-	 * Put a special mark into runtime vdso, thus at next checkpoint
-	 * routine we could detect this vdso and do not dump it, since
-	 * it's auto-generated every new session if proxy required.
-	 */
-	sys_mprotect((void *)vdso_rt_parked_at,  vdso_vma_size(sym_rt), PROT_WRITE);
-	vdso_put_mark((void *)vdso_rt_parked_at, vma->start);
-	sys_mprotect((void *)vdso_rt_parked_at,  vdso_vma_size(sym_rt), VDSO_PROT);
-	return 0;
-}

include/vdso.h

@@ -93,5 +93,8 @@ static inline void vdso_put_mark(void *where, unsigned long proxy_addr)
 extern struct vdso_symtable vdso_sym_rt;
 extern u64 vdso_pfn;
 extern int vdso_init(void);
+extern int vdso_remap(char *who, unsigned long from, unsigned long to, size_t size);
+extern int vdso_fill_symtable(char *mem, size_t size, struct vdso_symtable *t);
+extern int vdso_proxify(char *who, struct vdso_symtable *sym_rt, VmaEntry *vma_entry, unsigned long vdso_rt_parked_at);
 
 #endif /* __CR_VDSO_H__ */

pie/Makefile

@@ -4,6 +4,9 @@ targets			+= restorer
 obj-y			+= log-simple.o
 obj-y			+= util.o
 obj-y			+= util-fd.o
+obj-y			+= $(VDSO_O)
+
+obj-e			+= $(ARCH_DIR)/vdso-pie.o
 
 parasite-obj-y		+= parasite.o
 parasite-asm-e		+= $(ARCH_DIR)/parasite-head.o
@@ -11,7 +14,6 @@ parasite-libs-e		+= $(SYSCALL-LIB)
 
 restorer-obj-y		+= restorer.o
 restorer-obj-e		+= $(ARCH_DIR)/restorer.o
-restorer-obj-e		+= $(ARCH_DIR)/vdso-pie.o
 restorer-libs-e		+= $(SYSCALL-LIB)
 
 #

pie/vdso-stub.c

+#include <elf.h>
+
+#include <sys/mman.h>
+
+#include "compiler.h"
+#include "vdso.h"
+#include "syscall.h"
+#include "log.h"
+
+#include "asm/string.h"
+
+
+#ifdef LOG_PREFIX
+# undef LOG_PREFIX
+#endif
+#define LOG_PREFIX "vdso: "
+
+int vdso_remap(char *who, unsigned long from, unsigned long to, size_t size)
+{
+	return 0;
+}
+
+int vdso_fill_symtable(char *mem, size_t size, struct vdso_symtable *t)
+{
+	return 0;
+}
+
+int vdso_proxify(char *who, struct vdso_symtable *sym_rt, VmaEntry *vma, unsigned long vdso_rt_parked_at)
+{
+	return 0;
+}

pie/vdso.c

+#include <elf.h>
+
+#include <sys/mman.h>
+
+#include "compiler.h"
+#include "vdso.h"
+#include "syscall.h"
+#include "log.h"
+#include "vma.h"
+
+#include "asm/string.h"
+
+
+#ifdef LOG_PREFIX
+# undef LOG_PREFIX
+#endif
+#define LOG_PREFIX "vdso: "
+
+
+static unsigned int get_symbol_index(char *symbol, char *symbols[], size_t size)
+{
+	unsigned int i;
+
+	for (i = 0; symbol && i < size; i++) {
+		if (!builtin_strcmp(symbol, symbols[i]))
+			return i;
+	}
+
+	return VDSO_SYMBOL_MAX;
+}
+
+/* Check if pointer is out-of-bound */
+static bool __ptr_oob(void *ptr, void *start, size_t size)
+{
+	void *end = (void *)((unsigned long)start + size);
+	return ptr > end || ptr < start;
+}
+
+int vdso_fill_symtable(char *mem, size_t size, struct vdso_symtable *t)
+{
+	Elf64_Ehdr *ehdr = (void *)mem;
+	Elf64_Shdr *shdr, *shdr_strtab;
+	Elf64_Shdr *shdr_dynsym;
+	Elf64_Shdr *shdr_dynstr;
+	Elf64_Phdr *phdr;
+	Elf64_Shdr *text;
+	Elf64_Sym *sym;
+
+	char *section_names, *dynsymbol_names;
+
+	unsigned long base = VDSO_BAD_ADDR;
+	unsigned int i, j, k;
+
+	DECLARE_VDSO(vdso_ident, vdso_symbols);
+
+	BUILD_BUG_ON(sizeof(vdso_ident) != sizeof(ehdr->e_ident));
+
+	pr_debug("Parsing at %lx %lx\n",
+		 (long)mem, (long)mem + (long)size);
+
+	/*
+	 * Make sure it's a file we support.
+	 */
+	if (builtin_memcmp(ehdr->e_ident, vdso_ident, sizeof(vdso_ident))) {
+		pr_debug("Elf header magic mismatch\n");
+		goto err;
+	}
+
+	/*
+	 * Figure out base virtual address.
+	 */
+	phdr = (void *)&mem[ehdr->e_phoff];
+	for (i = 0; i < ehdr->e_phnum; i++, phdr++) {
+		if (__ptr_oob(phdr, mem, size))
+			goto err;
+		if (phdr->p_type == PT_LOAD) {
+			base = phdr->p_vaddr;
+			break;
+		}
+	}
+	if (base != VDSO_BAD_ADDR) {
+		pr_debug("Base address %lx\n", base);
+	} else {
+		pr_debug("No base address found\n");
+		goto err;
+	}
+
+	/*
+	 * Where the section names lays.
+	 */
+	if (ehdr->e_shstrndx == SHN_UNDEF) {
+		pr_err("Section names are not found\n");
+		goto err;
+	}
+
+	shdr = (void *)&mem[ehdr->e_shoff];
+	shdr_strtab = &shdr[ehdr->e_shstrndx];
+	if (__ptr_oob(shdr_strtab, mem, size))
+		goto err;
+
+	section_names = (void *)&mem[shdr_strtab->sh_offset];
+	shdr_dynsym = shdr_dynstr = text = NULL;
+
+	for (i = 0; i < ehdr->e_shnum; i++, shdr++) {
+		if (__ptr_oob(shdr, mem, size))
+			goto err;
+		if (__ptr_oob(&section_names[shdr->sh_name], mem, size))
+			goto err;
+
+#if 0
+		pr_debug("section: %2d -> %s\n",
+			 i, &section_names[shdr->sh_name]);
+#endif
+
+		if (shdr->sh_type == SHT_DYNSYM &&
+		    builtin_strcmp(&section_names[shdr->sh_name],
+				   ".dynsym") == 0) {
+			shdr_dynsym = shdr;
+		} else if (shdr->sh_type == SHT_STRTAB &&
+			   builtin_strcmp(&section_names[shdr->sh_name],
+					  ".dynstr") == 0) {
+			shdr_dynstr = shdr;
+		} else if (shdr->sh_type == SHT_PROGBITS &&
+			   builtin_strcmp(&section_names[shdr->sh_name],
+					  ".text") == 0) {
+			text = shdr;
+		}
+	}
+
+	if (!shdr_dynsym || !shdr_dynstr || !text) {
+		pr_debug("No required sections found\n");
+		goto err;
+	}
+
+	dynsymbol_names = (void *)&mem[shdr_dynstr->sh_offset];
+	if (__ptr_oob(dynsymbol_names, mem, size)	||
+	    __ptr_oob(shdr_dynsym, mem, size)		||
+	    __ptr_oob(text, mem, size))
+		goto err;
+
+	/*
+	 * Walk over global symbols and choose ones we need.
+	 */
+	j = shdr_dynsym->sh_size / sizeof(*sym);
+	sym = (void *)&mem[shdr_dynsym->sh_offset];
+
+	for (i = 0; i < j; i++, sym++) {
+		if (__ptr_oob(sym, mem, size))
+			goto err;
+
+		if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL ||
+		    ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
+			continue;
+
+		if (__ptr_oob(&dynsymbol_names[sym->st_name], mem, size))
+			goto err;
+
+		k = get_symbol_index(&dynsymbol_names[sym->st_name],
+				     vdso_symbols,
+				     ARRAY_SIZE(vdso_symbols));
+		if (k != VDSO_SYMBOL_MAX) {
+			builtin_memcpy(t->symbols[k].name, vdso_symbols[k],
+				       sizeof(t->symbols[k].name));
+			t->symbols[k].offset = (unsigned long)sym->st_value - base;
+#if 0
+			pr_debug("symbol: %#-16lx %2d %s\n",
+				 t->symbols[k].offset, sym->st_shndx, t->symbols[k].name);
+#endif
+		}
+	}
+	return 0;
+err:
+	return -1;
+}
+
+int vdso_remap(char *who, unsigned long from, unsigned long to, size_t size)
+{
+	unsigned long addr;
+
+	pr_debug("Remap %s %lx -> %lx\n", who, from, to);
+
+	addr = sys_mremap(from, size, size, MREMAP_MAYMOVE | MREMAP_FIXED, to);
+	if (addr != to) {
+		pr_err("Unable to remap %lx -> %lx %lx\n",
+		       from, to, addr);
+		return -1;
+	}
+
+	return 0;
+}
+
+int vdso_proxify(char *who, struct vdso_symtable *sym_rt, VmaEntry *vma, unsigned long vdso_rt_parked_at)
+{
+	struct vdso_symtable s = VDSO_SYMTABLE_INIT;
+	size_t size = vma_entry_len(vma);
+	bool remap_rt = true;
+
+	/*
+	 * Find symbols in dumpee vdso.
+	 */
+	if (vdso_fill_symtable((void *)vma->start, size, &s))
+		return -1;
+
+	if (size == vdso_vma_size(sym_rt)) {
+		int i;
+
+		for (i = 0; i < ARRAY_SIZE(s.symbols); i++) {
+			if (s.symbols[i].offset != sym_rt->symbols[i].offset) {
+				remap_rt = false;
+				break;
+			}
+		}
+	} else
+		remap_rt = false;
+
+	/*
+	 * Easy case -- the vdso from image has same offsets and size
+	 * as runtime, so we simply remap runtime vdso to dumpee position
+	 * without generating any proxy.
+	 */
+	if (remap_rt) {
+		pr_info("Runtime vdso matches dumpee, remap inplace\n");
+
+		if (sys_munmap((void *)vma->start, size)) {
+			pr_err("Failed to unmap %s\n", who);
+			return -1;
+		}
+
+		return vdso_remap(who, vdso_rt_parked_at, vma->start, size);
+	}
+
+	/*
+	 * Now complex case -- we need to proxify calls. We redirect
+	 * calls from dumpee vdso to runtime vdso, making dumpee
+	 * to operate as proxy vdso.
+	 */
+	pr_info("Runtime vdso mismatches dumpee, generate proxy\n");
+
+	if (vdso_redirect_calls((void *)vdso_rt_parked_at,
+				(void *)vma->start,
+				sym_rt, &s)) {
+		pr_err("Failed to proxify dumpee contents\n");
+		return -1;
+	}
+
+	/*
+	 * Put a special mark into runtime vdso, thus at next checkpoint
+	 * routine we could detect this vdso and do not dump it, since
+	 * it's auto-generated every new session if proxy required.
+	 */
+	sys_mprotect((void *)vdso_rt_parked_at,  vdso_vma_size(sym_rt), PROT_WRITE);
+	vdso_put_mark((void *)vdso_rt_parked_at, vma->start);
+	sys_mprotect((void *)vdso_rt_parked_at,  vdso_vma_size(sym_rt), VDSO_PROT);
+	return 0;
+}