restore: A number of fixups and debug printing

An idea is to align restorer size to be sure it
wont be stripped in a middle of code.
Signed-off-by: Cyrill Gorcunov <gorcunov@gmail.com>

Makefile

@@ -23,7 +23,7 @@ MAKE		:= make
 CFLAGS		+= -I./include
 CFLAGS		+= -O0 -ggdb3
 
-LIBS		+= -lrt
+LIBS		+= -lrt -lpthread
 
 # Additional ARCH settings for x86
 ARCH ?= $(shell echo $(uname_M) | sed -e s/i.86/i386/ -e s/sun4u/sparc64/ \
@@ -109,7 +109,7 @@ $(OBJS): $(HEAD-BLOB) $(DEPS) $(HEAD-BLOB-GEN)
 	$(E) "  CC      " $@
 	$(Q) $(CC) -c $(CFLAGS) $< -o $@
 
-$(PROGRAM): $(OBJS)
+$(PROGRAM): $(OBJS) restorer.o
 	$(E) "  LINK    " $@
 	$(Q) $(CC) $(CFLAGS) $(OBJS) $(LIBS) -o $@
 
@@ -124,6 +124,13 @@ test:
 	$(Q) $(MAKE) -C test all
 .PHONY: test
 
+rebuild:
+	$(E) "  FORCE-REBUILD"
+	$(Q) $(RM) -f ./*.o
+	$(Q) $(RM) -f ./*.d
+	$(Q) $(MAKE)
+.PHONY: rebuild
+
 clean:
 	$(E) "  CLEAN"
 	$(Q) $(RM) -f ./*.o

cr-restore.c

@@ -1262,7 +1262,7 @@ static void restorer_test(pid_t pid)
 	code_len	= restorer_fcall(RESTORER_CMD__GET_SELF_LEN) - (long)restorer;
 	args_offset	= restorer_fcall(RESTORER_CMD__GET_ARG_OFFSET) - (long)restorer;
 	code_len	= round_up(code_len, 16);
-	vma_len		= round_up(code_len + RESTORER_STACK_SIZE, PAGE_SIZE);
+	vma_len		= round_up(code_len + RESTORER_STACK_SIZE + RESTORER_STACK_FRAME, PAGE_SIZE);
 
 	/*
 	 * Here we need some heuristics -- the VMA which restorer will
@@ -1280,28 +1280,30 @@ static void restorer_test(pid_t pid)
 		return;
 	}
 
+
 	/*
 	 * Prepare a stack for the restorer. It's a bit
 	 * tricky -- since compiler generates function
 	 * prologue we need to manually tune up stack
 	 * value.
 	 */
-	exec_start = exec_mem + RESTORER_STACK_SIZE;
-	memzero(exec_mem, RESTORER_STACK_SIZE);
+	memzero(exec_mem, RESTORER_STACK_SIZE + RESTORER_STACK_FRAME);
+	exec_start = exec_mem + RESTORER_STACK_SIZE + RESTORER_STACK_FRAME;
 
 	/* Restorer content at the new location */
 	memcpy(exec_start, &restorer, code_len);
 	restorer_fcall = exec_start;
 
 	/*
-	 * Stack pointer in a middle of allocated stack zone.
+	 * Adjust stack with red-zone area.
 	 */
-	new_sp = (long)exec_mem + RESTORER_STACK_MIDDLE;
+	new_sp = (long)exec_mem + RESTORER_STACK_SIZE - RESTORER_STACK_REDZONE;
 
 	/*
 	 * Pass arguments and run a command.
 	 */
 	args			= (struct restore_core_args *)(exec_start + args_offset);
+	args->rt_sigframe	= (void *)((long)exec_mem + RESTORER_STACK_SIZE + RESTORER_STACK_REDZONE);
 	args->self_entry	= exec_mem;
 	args->self_size		= vma_len;
 
@@ -1310,6 +1312,14 @@ static void restorer_test(pid_t pid)
 	snprintf(path, sizeof(path), "core-%d.img", pid);
 	strcpy(args->core_path, path);
 
+	pr_info("vma_len: %li code_len: %li exec_mem: %p exec_start: %p new_sp: %p args: %p\n",
+		vma_len, code_len, exec_mem, exec_start, new_sp, args);
+
+	pr_info("args: %p args->rt_sigframe: %p args->self_entry: %p \nargs->self_size: %p "
+		"args->self_vmas_path: %p args->core_path: %p\n",
+		args, args->rt_sigframe, args->self_entry, args->self_size,
+		args->self_vmas_path, args->core_path);
+
 	/*
 	 * An indirect call to restorer, note it never resturns
 	 * and restoreing core is extremely destructive.

include/restorer.h

@@ -12,10 +12,11 @@
 #endif
 
 #define RESTORER_ARGS_SIZE	512
-#define RESTORER_STACK_MIDDLE	(16 << 10)
-#define RESTORER_STACK_SIZE	(RESTORER_STACK_MIDDLE * 2)
+#define RESTORER_STACK_REDZONE	(128)
+#define RESTORER_STACK_FRAME	(16 << 10)
+#define RESTORER_STACK_SIZE	(32 << 10)
 
-long restorer(long cmd);
+extern long restorer(long cmd);
 
 typedef long (*restorer_fcall_t) (long cmd);
 
@@ -27,18 +28,61 @@ typedef long (*restorer_fcall_t) (long cmd);
 
 #define ABI_RED_ZONE 128
 
-#define align_sigframe(sp)			\
-	do {					\
-		sp = round_down(sp, 16) - 8;	\
-	} while (0)
+#define align_sigframe(sp)		round_down(sp, 16) - 8
 
 struct restore_core_args {
 	void	*self_entry;		/* restorer placed at */
+	void	*rt_sigframe;		/* sigframe placed at */
 	long	self_size;		/* size for restorer granted */
 	char	core_path[64];		/* path to a core file */
 	char	self_vmas_path[64];	/* path to a self-vmas file */
 };
 
+struct pt_regs {
+	unsigned long	r15;
+	unsigned long	r14;
+	unsigned long	r13;
+	unsigned long	r12;
+	unsigned long	bp;
+	unsigned long	bx;
+
+	unsigned long	r11;
+	unsigned long	r10;
+	unsigned long	r9;
+	unsigned long	r8;
+	unsigned long	ax;
+	unsigned long	cx;
+	unsigned long	dx;
+	unsigned long	si;
+	unsigned long	di;
+	unsigned long	orig_ax;
+
+	unsigned long	ip;
+	unsigned long	cs;
+	unsigned long	flags;
+	unsigned long	sp;
+	unsigned long	ss;
+};
+
+struct partial_pt_regs {
+	unsigned long	r11;
+	unsigned long	r10;
+	unsigned long	r9;
+	unsigned long	r8;
+	unsigned long	ax;
+	unsigned long	cx;
+	unsigned long	dx;
+	unsigned long	si;
+	unsigned long	di;
+	unsigned long	orig_ax;
+
+	unsigned long	ip;
+	unsigned long	cs;
+	unsigned long	flags;
+	unsigned long	sp;
+	unsigned long	ss;
+};
+
 struct rt_sigcontext {
 	unsigned long			r8;
 	unsigned long			r9;

restorer.c

@@ -42,6 +42,8 @@
 #define inline_memzero(d,l)	__builtin_memset(d,0,l)
 #define inline_memzero_p(d)	__builtin_memset(d,0,sizeof(*(d)))
 
+#define sigframe_addr(p)	((long)p)
+
 static void always_inline write_char(char c)
 {
 	sys_write(1, &c, 1);
@@ -67,7 +69,6 @@ static void always_inline write_string_n(char *str)
 
 static void always_inline write_hex_n(unsigned long num)
 {
-	bool tailing = false;
 	unsigned char *s = (unsigned char *)#
 	unsigned char c;
 	int i;
@@ -152,6 +153,8 @@ self_len_end:
 		struct user_regs_entry *gpregs;
 		struct rt_sigframe *rt_sigframe;
 
+		unsigned long new_sp, *stack;
+
 		lea_args_off(args);
 
 		write_string_n(args->core_path);
@@ -296,13 +299,9 @@ self_len_end:
 		 * by the kernel with stack overflow error.
 		 */
 
-		/*
-		 * Reuse arguments space for own needs,
-		 * no more arguments needed so lets try
-		 * to be thrifty with memory usage.
-		*/
-		lea_args_off(rt_sigframe);
-		inline_memzero_p(rt_sigframe);
+		rt_sigframe = args->rt_sigframe;
+		write_hex_n((long)rt_sigframe);
+		write_hex_n((long)&rt_sigframe->uc);
 
 #define CPREG1(d)	rt_sigframe->uc.uc_mcontext.d = core_entry.u.arch.gpregs.d
 #define CPREG2(d,s)	rt_sigframe->uc.uc_mcontext.d = core_entry.u.arch.gpregs.s
@@ -331,7 +330,10 @@ self_len_end:
 
 		/* FIXME: What with cr2 and friends which are rest there? */
 
-		write_hex_n(__LINE__);
+		new_sp = core_entry.u.arch.gpregs.sp - 8;
+		write_hex_n(new_sp);
+		stack = (void *)new_sp;
+		*stack = (long)rt_sigframe;
 
 		/*
 		 * Prepare the stack and call for sigreturn,
@@ -344,7 +346,7 @@ self_len_end:
 			"movl $"__stringify(__NR_rt_sigreturn)", %%eax	\t\n"
 			"syscall					\t\n"
 			:
-			: "r"((long)rt_sigframe + sizeof(*rt_sigframe))
+			: "r"(new_sp)
 			: "rax","rsp","memory");
 
 core_restore_end:
@@ -364,11 +366,11 @@ core_restore_end:
 
 self_len_start:
 	asm volatile(
-		".align 16				\t\n"
+		".align 64				\t\n"
 		"self:					\t\n"
 		"leaq self(%%rip), %%rax		\t\n"
-		"addq $16, %%rax			\t\n"
-		"andq $~15, %%rax			\t\n"
+		"addq $64, %%rax			\t\n"
+		"andq $~63, %%rax			\t\n"
 		"movq %%rax, %0				\t\n"
 		: "=r"(ret)
 		: