pycriu: add python package

pycriu is a python package, that, for now, consists
of rpc module and images package. rpc module contains
data structures for interacting with CRIU RPC.
images package contains methods for loading\dumping
criu images.

See a big comment below in pycriu/images/images.py file.
Signed-off-by: Ruslan Kuprieiev <kupruser@gmail.com>
Signed-off-by: Pavel Emelyanov <xemul@parallels.com>

pycriu/.gitignore

+*_pb2.py
+*.pyc
+rpc.py

pycriu/Makefile

+all: images rpc.py
+
+.PHONY: all images clean
+
+images:
+	$(Q) $(MAKE) -C images all
+
+# rpc_pb2.py doesn't depend on any other file, so
+# it is safe to rename it, dropping ugly _pb2 suffix.
+rpc.py:
+	$(Q) protoc -I=$(SRC_DIR)/protobuf/ --python_out=./ $(SRC_DIR)/protobuf/$(@:.py=.proto)
+	$(Q) mv $(@:.py=_pb2.py) $@
+
+clean:
+	$(Q) $(MAKE) -C images clean
+	$(Q) $(RM) rpc.py *.pyc

pycriu/__init__.py

+import rpc
+import images

pycriu/images/.gitignore

+*.pyc
+*_pb2.py
+magic.py
+pb.py

pycriu/images/Makefile

+all: pb.py protobuf magic.py
+
+.PHONY: all protobuf clean pb.py
+
+proto := $(filter-out $(SRC_DIR)/protobuf/rpc.proto, $(wildcard $(SRC_DIR)/protobuf/*.proto))
+proto-py-modules := $(foreach m,$(proto),$(subst -,_,$(notdir $(m:.proto=_pb2))))
+
+# We don't need rpc_pb2.py here, as it is not related to the
+# images.
+# Unfortunately, we can't drop ugly _pb2 suffixes here, because
+# some _pb2 files depend on others _pb2 files.
+protobuf:
+	$(Q) protoc -I=$(SRC_DIR)/protobuf --python_out=./ $(proto)
+
+magic.py: $(SRC_DIR)/scripts/magic-gen.py $(SRC_DIR)/include/magic.h
+	$(E) "  GEN  " $@
+	$(Q) python $^ $@
+
+pb.py: protobuf
+	$(Q) echo "# Autogenerated. Do not edit!" > $@
+	$(Q) for m in $(proto-py-modules); do \
+		echo "from $$m import *" >> $@ ;\
+	done
+
+clean:
+	$(Q) $(RM) ./*_pb2.py ./*.pyc magic.py pb.py

pycriu/images/__init__.py

+from magic import *
+from images import *
+from pb import *

pycriu/images/images.py

+#!/bin/env python
+
+# This file contains methods to deal with criu images.
+#
+# According to http://criu.org/Images, criu images can be described
+# with such IOW:
+# IMAGE_FILE ::= MAGIC { ENTRY }
+# ENTRY      ::= SIZE PAYLOAD [ EXTRA ]
+# PAYLOAD    ::= "message encoded in ProtocolBuffer format"
+# EXTRA      ::= "arbitrary blob, depends on the PAYLOAD contents"
+#
+# MAGIC      ::= "32 bit integer"
+# SIZE       ::= "32 bit integer, equals the PAYLOAD length"
+#
+# In order to convert images to human-readable format, we use dict(json).
+# Using json not only allows us to easily read\write images, but also
+# to use a great variety of tools out there to manipulate them.
+# It also allows us to clearly describe criu images structure.
+#
+# Using dict(json) format, criu images can be described like:
+#
+# {
+#	'magic' : 'FOO',
+#	'entries' : [
+#		entry,
+#		...
+#	]
+# }
+#
+# Entry, in its turn, could be described as:
+#
+# {
+#	'payload' : pb_msg,
+#	'extra' : extra_msg
+# }
+#
+import io
+import google
+import struct
+import os
+import sys
+import json
+import pb2dict
+
+import magic
+from pb import *
+
+# Generic class to handle loading/dumping criu images entries from/to bin
+# format to/from dict(json).
+class entry_handler:
+	"""
+	Generic class to handle loading/dumping criu images
+	entries from/to bin format to/from dict(json).
+	"""
+	def __init__(self, payload, extra_handler=None):
+		"""
+		Sets payload class and extra handler class.
+		"""
+		self.payload		= payload
+		self.extra_handler	= extra_handler
+
+	def load(self, f):
+		"""
+		Convert criu image entries from binary format to dict(json).
+		Takes a file-like object and returnes a list with entries in
+		dict(json) format.
+		"""
+		entries = []
+
+		while True:
+			entry = {}
+
+			# Read payload
+			pb = self.payload()
+			buf = f.read(4)
+			if buf == '':
+				break
+			size, = struct.unpack('i', buf)
+			pb.ParseFromString(f.read(size))
+			entry['payload'] = pb2dict.pb2dict(pb)
+
+			# Read extra
+			if self.extra_handler:
+				entry['extra'] = self.extra_handler.load(f, pb)
+
+			entries.append(entry)
+
+		return entries
+
+	def loads(self, s):
+		"""
+		Same as load(), but takes a string as an argument.
+		"""
+		f = io.BytesIO(s)
+		return self.load(f)
+
+	def dump(self, entries, f):
+		"""
+		Convert criu image entries from dict(json) format to binary.
+		Takes a list of entries and a file-like object to write entries
+		in binary format to.
+		"""
+		for entry in entries:
+			# Write payload
+			pb = self.payload()
+			pb2dict.dict2pb(entry['payload'], pb)
+			pb_str = pb.SerializeToString()
+			size = len(pb_str)
+			f.write(struct.pack('i', size))
+			f.write(pb_str)
+
+			# Write extra
+			if self.extra_handler:
+				self.extra_handler.dump(entry['extra'], f, pb)
+
+	def dumps(self, entries):
+		"""
+		Same as dump(), but doesn't take file-like object and just
+		returns a string.
+		"""
+		f = io.BytesIO('')
+		self.dump(entries, f)
+		return f.read()
+
+# Some custom extra handlers
+
+class pagemap_extra_handler:
+	def load(self, f, pload):
+		array = []
+
+		while True:
+			pb = pagemap_entry()
+			buf = f.read(4)
+			if buf == '':
+				break
+			size, = struct.unpack('i', buf)
+			pb.ParseFromString(f.read(size))
+			array.append(pb2dict.pb2dict(pb))
+
+		return array
+
+	def dump(self, extra, f, pload):
+		for item in extra:
+			pb = pagemap_entry()
+			pb2dict.dict2pb(item, pb)
+			pb_str = pb.SerializeToString()
+			size = len(pb_str)
+			f.write(struct.pack('i', size))
+			f.write(pb_str)
+
+# In following handlers we use base64 encoding
+# to store binary data. Even though, the nature
+# of base64 is that it increases the total size,
+# it doesn't really matter, because our images
+# do not store big amounts of binary data. They
+# are negligible comparing to pages size.
+class pipes_data_extra_handler:
+	def load(self, f, pload):
+		size = pload.bytes
+		data = f.read(size)
+		return data.encode('base64')
+
+	def dump(self, extra, f, pload):
+		data = extra.decode('base64')
+		f.write(data)
+
+class sk_queues_extra_handler:
+	def load(self, f, pb):
+		size = pload.length
+		data = f.read(size)
+		return data.encode('base64')
+
+	def dump(self, extra, f, pb):
+		data = extra.decode('base64')
+		f.write(data)
+
+class ghost_file_extra_handler:
+	def load(self, f, pb):
+		data = f.read()
+		return data.encode('base64')
+
+	def dump(self, extra, f, pb):
+		data = extra.decode('base64')
+		f.write(data)
+
+handlers = {
+	'INVENTORY'		: entry_handler(inventory_entry),
+	'CORE'			: entry_handler(core_entry),
+	'IDS'			: entry_handler(task_kobj_ids_entry),
+	'CREDS'			: entry_handler(creds_entry),
+	'UTSNS'			: entry_handler(utsns_entry),
+	'IPC_VAR'		: entry_handler(ipc_var_entry),
+	'FS'			: entry_handler(fs_entry),
+	'GHOST_FILE'		: entry_handler(ghost_file_entry, ghost_file_extra_handler()),
+	'MM'			: entry_handler(mm_entry),
+	'CGROUP'		: entry_handler(cgroup_entry),
+	'TCP_STREAM'		: entry_handler(tcp_stream_entry),
+	'STATS'			: entry_handler(stats_entry),
+	'PAGEMAP'		: entry_handler(pagemap_head, pagemap_extra_handler()),
+	'PSTREE'		: entry_handler(pstree_entry),
+	'REG_FILES'		: entry_handler(reg_file_entry),
+	'NS_FILES'		: entry_handler(ns_file_entry),
+	'EVENTFD_FILE'		: entry_handler(eventfd_file_entry),
+	'EVENTPOLL_FILE'	: entry_handler(eventpoll_file_entry),
+	'EVENTPOLL_TFD'		: entry_handler(eventpoll_tfd_entry),
+	'SIGNALFD'		: entry_handler(signalfd_entry),
+	'TIMERFD'		: entry_handler(timerfd_entry),
+	'INOTIFY_FILE'		: entry_handler(inotify_file_entry),
+	'INOTIFY_WD'		: entry_handler(inotify_wd_entry),
+	'FANOTIFY_FILE'		: entry_handler(fanotify_file_entry),
+	'FANOTIFY_MARK'		: entry_handler(fanotify_mark_entry),
+	'VMAS'			: entry_handler(vma_entry),
+	'PIPES'			: entry_handler(pipe_entry),
+	'FIFO'			: entry_handler(fifo_entry),
+	'SIGACT'		: entry_handler(sa_entry),
+	'NETLINK_SK'		: entry_handler(netlink_sk_entry),
+	'REMAP_FPATH'		: entry_handler(remap_file_path_entry),
+	'MNTS'			: entry_handler(mnt_entry),
+	'TTY_FILES'		: entry_handler(tty_file_entry),
+	'TTY_INFO'		: entry_handler(tty_info_entry),
+	'RLIMIT'		: entry_handler(rlimit_entry),
+	'TUNFILE'		: entry_handler(tunfile_entry),
+	'EXT_FILES'		: entry_handler(ext_file_entry),
+	'IRMAP_CACHE'		: entry_handler(irmap_cache_entry),
+	'FILE_LOCKS'		: entry_handler(file_lock_entry),
+	'FDINFO'		: entry_handler(fdinfo_entry),
+	'UNIXSK'		: entry_handler(unix_sk_entry),
+	'INETSK'		: entry_handler(inet_sk_entry),
+	'PACKETSK'		: entry_handler(packet_sock_entry),
+	'ITIMERS'		: entry_handler(itimer_entry),
+	'POSIX_TIMERS'		: entry_handler(posix_timer_entry),
+	'NETDEV'		: entry_handler(net_device_entry),
+	'PIPES_DATA'		: entry_handler(pipe_data_entry, pipes_data_extra_handler()),
+	'FIFO_DATA'		: entry_handler(pipe_data_entry, pipes_data_extra_handler()),
+	'SK_QUEUES'		: entry_handler(sk_packet_entry, sk_queues_extra_handler()),
+	'IPCNS_SHM'		: entry_handler(ipc_shm_entry),
+	'IPCNS_SEM'		: entry_handler(ipc_sem_entry),
+	'IPCNS_MSG'		: entry_handler(ipc_msg_entry)
+	}
+
+def load(f):
+	"""
+	Convert criu image from binary format to dict(json).
+	Takes a file-like object to read criu image from.
+	Returns criu image in dict(json) format.
+	"""
+	image = {}
+
+	img_magic, = struct.unpack('i', f.read(4))
+
+	try:
+		m = magic.by_val[img_magic]
+	except:
+		raise Exception("Unknown magic "+str(img_magic))
+
+	try:
+		handler = handlers[m]
+	except:
+		raise Exception("No handler found for image with such magic "+m)
+
+	image['magic']		= m
+	image['entries']	= handler.load(f)
+
+	return image
+
+def loads(s):
+	"""
+	Same as load(), but takes a string.
+	"""
+	f = io.BytesIO(s)
+	return load(f)
+
+def dump(img, f):
+	"""
+	Convert criu image from dict(json) format to binary.
+	Takes an image in dict(json) format and file-like
+	object to write to.
+	"""
+	m = img['magic']
+	magic_val = magic.by_name[img['magic']]
+
+	f.write(struct.pack('i', magic_val))
+
+	try:
+		handler = handlers[m]
+	except:
+		raise Exception("No handler found for image with such magic")
+
+	handler.dump(img['entries'], f)
+
+def dumps(img):
+	"""
+	Same as dump(), but takes only an image and returns
+	a string.
+	"""
+	f = io.BytesIO('')
+	dump(img, f)
+	return f.getvalue()

pycriu/images/pb2dict.py

+import google
+import io
+
+# pb2dict and dict2pb are using protobuf text format to
+# convert protobuf msgs to/from dictionary.
+
+def pb2dict(pb):
+	"""
+	Convert protobuf msg to dictionary.
+	Takes a protobuf message and returns a dict.
+	"""
+	pb_text = io.BytesIO('')
+	google.protobuf.text_format.PrintMessage(pb, pb_text)
+	pb_text.seek(0)
+	return _text2dict(pb_text)
+
+def _text2dict(pb_text):
+	"""
+	Convert protobuf text format msg to dict
+	Takes a protobuf message in text format and
+	returns a dict.
+	"""
+	d = {}
+	while True:
+		s = pb_text.readline()
+		s.strip()
+		if s == '' or '}' in s:
+			break
+
+		name, value = s.split()
+		if value == '{':
+			value = _text2dict(pb_text)
+		elif name.endswith(':'):
+			name = name[:-1]
+		else:
+			raise Exception("Unknown format" + s)
+
+		if d.get(name):
+			if not isinstance(d[name], list):
+				d[name] = [d[name]]
+			d[name].append(value)
+		else:
+			d[name] = value
+
+	return d
+
+def dict2pb(d, pb):
+	"""
+	Convert dictionary to protobuf msg.
+	Takes dict and protobuf message to be merged into.
+	"""
+	pb_text = io.BytesIO('')
+	_dict2text(d, pb_text, 0)
+	pb_text.seek(0)
+	s = pb_text.read()
+	google.protobuf.text_format.Merge(s, pb)
+
+def _write_struct(name, text, indent, inside):
+	"""
+	Convert "inside" dict to protobuf text format
+	wrap it inside block named "name" and write
+	it to "text".
+	"""
+	text.write(indent*" " + name.encode() + " {\n")
+	_dict2text(inside, text, indent+2)
+	text.write(indent*" " + "}\n")
+
+def _write_field(name, value, text, indent):
+	"""
+	Write "name: value" to "text".
+	"""
+	text.write(indent*" " + name.encode() + ": " + value.encode() + "\n")
+
+def _dict2text(d, pb_text, indent):
+	"""
+	Convert dict to protobuf text format.
+	Takes dict, protobuf message in text format and a number
+	of spaces to be put before each field.
+	"""
+	for name, value in d.iteritems():
+		if isinstance(value, unicode):
+			_write_field(name, value, pb_text, indent)
+		elif isinstance(value, list):
+			for x in value:
+				if isinstance(x, dict):
+					_write_struct(name, pb_text, indent, x)
+				else:
+					_write_field(name, x, pb_text, indent)
+		else:
+			_write_struct(name, pb_text, indent, value)