page-read: Callback on io completion

This one is called by PR once IO is complete (right now
for sync cases only, more work is required here) and
lets us unify local and remote PF code in uffd.
Signed-off-by: Pavel Emelyanov <xemul@virtuozzo.com>
Acked-by: Mike Rapoport <rppt@linux.vnet.ibm.com>

criu/include/pagemap.h

@@ -53,6 +53,7 @@ struct page_read {
 	int (*sync)(struct page_read *pr);
 	int (*seek_pagemap)(struct page_read *pr, unsigned long vaddr);
 	void (*reset)(struct page_read *pr);
+	int (*io_complete)(struct page_read *, unsigned long vaddr, int nr);
 	int (*maybe_read_page)(struct page_read *pr, unsigned long vaddr,
 			       int nr, void *buf, unsigned flags);
 

criu/pagemap.c

@@ -409,8 +409,11 @@ static int maybe_read_page_local(struct page_read *pr, unsigned long vaddr,
 	 */
 	if ((flags & (PR_ASYNC|PR_ASAP)) == PR_ASYNC)
 		ret = pagemap_enqueue_iovec(pr, buf, len, &pr->async);
-	else
+	else {
 		ret = read_local_page(pr, vaddr, len, buf);
+		if (ret == 0 && pr->io_complete)
+			ret = pr->io_complete(pr, vaddr, nr);
+	}
 
 	pr->pi_off += len;
 
@@ -435,6 +438,9 @@ static int maybe_read_page_remote(struct page_read *pr, unsigned long vaddr,
 	if (ret == 0)
 		ret = receive_remote_pages(nr * PAGE_SIZE, buf);
 
+	if (ret == 0 && pr->io_complete)
+		ret = pr->io_complete(pr, vaddr, nr);
+
 	return ret;
 }
 
@@ -546,6 +552,8 @@ more:
 		if (opts.auto_dedup && punch_hole(pr, start, ret, false))
 			return -1;
 
+		BUG_ON(pr->io_complete); /* FIXME -- implement once needed */
+
 		list_del(&piov->l);
 		xfree(iovs);
 		xfree(piov);
@@ -775,6 +783,7 @@ int open_page_read_at(int dfd, int pid, struct page_read *pr, int pr_flags)
 	pr->sync = process_async_reads;
 	pr->seek_pagemap = seek_pagemap;
 	pr->reset = reset_pagemap;
+	pr->io_complete = NULL; /* set up by the client if needed */
 	pr->id = ids++;
 	pr->pid = pid;
 

criu/uffd.c

@@ -476,6 +476,8 @@ free_mm:
 	return ret;
 }
 
+static int uffd_io_complete(struct page_read *pr, unsigned long vaddr, int nr);
+
 static int ud_open(int client, struct lazy_pages_info **_lpi)
 {
 	struct lazy_pages_info *lpi;
@@ -523,6 +525,8 @@ static int ud_open(int client, struct lazy_pages_info **_lpi)
 		goto out;
 	}
 
+	lpi->pr.io_complete = uffd_io_complete;
+
 	/*
 	 * Find the memory pages belonging to the restored process
 	 * so that it is trackable when all pages have been transferred.
@@ -587,6 +591,14 @@ static int complete_page_fault(struct lazy_pages_info *lpi, unsigned long vaddr,
 	return update_lazy_iovecs(lpi, vaddr, nr * PAGE_SIZE);
 }
 
+static int uffd_io_complete(struct page_read *pr, unsigned long vaddr, int nr)
+{
+	struct lazy_pages_info *lpi;
+
+	lpi = container_of(pr, struct lazy_pages_info, pr);
+	return complete_page_fault(lpi, vaddr, nr);
+}
+
 static int uffd_zero(struct lazy_pages_info *lpi, __u64 address, int nr_pages)
 {
 	struct uffdio_zeropage uffdio_zeropage;
@@ -649,7 +661,7 @@ static int uffd_handle_pages(struct lazy_pages_info *lpi, __u64 address, int nr)
 		return ret;
 	}
 
-	return complete_page_fault(lpi, address, nr);
+	return 0;
 }
 
 static int handle_remaining_pages(struct lazy_pages_info *lpi)
@@ -694,10 +706,7 @@ static int page_fault_common(struct lazy_pages_info *lpi, __u64 address, int nr,
 
 static int page_fault_local(struct lazy_pages_info *lpi, __u64 address, int nr)
 {
-	if (page_fault_common(lpi, address, nr, PR_ASYNC | PR_ASAP))
-		return -1;
-
-	return complete_page_fault(lpi, address, nr);
+	return page_fault_common(lpi, address, nr, PR_ASYNC | PR_ASAP);
 }
 
 static int page_fault_remote(struct lazy_pages_info *lpi, __u64 address, int nr)