Allow to do serializing in machine-independent format

pire/Makefile.am

@@ -43,12 +43,15 @@ libpire_la_SOURCES = \
 	stub/stl.h \
 	stub/lexical_cast.h \
 	stub/saveload.h \
+	stub/saveload.cpp \
 	stub/singleton.h \
 	stub/utf8.cpp \
 	stub/utf8.h \
 	stub/noncopyable.h \
 	stub/codepage_h.h \
 	stub/doccodes_h.h \
+	stub/msgpuck.h \
+	stub/msgpuck.c \
 	stub/unidata_h.h \
 	stub/unidata_cpp.h
 
@@ -105,6 +108,7 @@ pire_scanners_HEADERS = \
 pire_stubdir = $(includedir)/pire/stub
 pire_stub_HEADERS = \
 	stub/stl.h \
+	stub/msgpuck.h \
 	stub/defaults.h \
 	stub/memstreams.h \
 	stub/singleton.h \

pire/scanners/common.h

@@ -26,6 +26,7 @@
 #include <stdlib.h>
 #include "../align.h"
 #include "../stub/defaults.h"
+#include "../stub/msgpuck.h"
 #include "../defs.h"
 #include "../platform.h"
 
@@ -42,6 +43,8 @@ namespace Pire {
 		static const ui32 MAGIC = 0x45524950;   // "PIRE" on litte-endian
 		static const ui32 RE_VERSION = 7;       // Should be incremented each time when the format of serialized scanner changes
 		static const ui32 RE_VERSION_WITH_MACTIONS = 6;  // LoadedScanner with m_actions, which is ignored
+		static const ui32 PACKED_VERSION = 1;
+		static const ui32 PACKED_HEADERS = 5;
 
 		explicit Header(ui32 type, size_t hdrsize)
 			: Magic(MAGIC)
@@ -61,6 +64,36 @@ namespace Pire {
 			if ((type != 0 && type != Type) || (hdrsize != 0 && HdrSize != hdrsize))
 				throw Error("Serialized regexp incompatible with your system");
 		}
+
+		void Pack(yostream* s)
+		{
+			char buf[PACKED_HEADERS * 8 * 4];
+			char *ptr = buf;
+
+			ptr = mp_encode_array(ptr, PACKED_HEADERS);
+			ptr = mp_encode_uint(ptr, Magic);
+			ptr = mp_encode_uint(ptr, PACKED_VERSION);
+			ptr = mp_encode_uint(ptr, MaxCharUnaligned);
+			ptr = mp_encode_uint(ptr, MaxWordSize);
+			ptr = mp_encode_uint(ptr, Type);
+			s->write(buf, ptr - buf);
+		}
+
+		void Unpack(yistream* s) const
+		{
+			if (MsgpuckReadArray(s) != PACKED_HEADERS)
+				throw Error("Headers number is invalid");
+			if (MsgpuckReadUint(s) != Magic)
+				throw Error("Unknown magic");
+			if (MsgpuckReadUint(s) != PACKED_VERSION)
+				throw Error("You are trying to use an incompatible version of a serialized regexp");
+			if (MsgpuckReadUint(s) != MaxCharUnaligned)
+				throw Error("MaxCharUnaligned mismatch");
+			if (MsgpuckReadUint(s) != MaxWordSize)
+				throw Error("MaxWordSize mismatch");
+			if (MsgpuckReadUint(s) != Type)
+				throw Error("Incompatible regexp type");
+		}
 	};
 
 	namespace Impl {

pire/scanners/multi.h

@@ -302,6 +302,9 @@ class Scanner {
 	void Save(yostream*) const;
 	void Load(yistream*);
 
+	void Pack(yostream*) const;
+	void Unpack(yistream*);
+
 	ScannerRowHeader& Header(State s) { return *(ScannerRowHeader*) s; }
 	const ScannerRowHeader& Header(State s) const { return *(const ScannerRowHeader*) s; }
 
@@ -592,6 +595,185 @@ struct ScannerSaver {
 		scanner.Swap(sc);
 	}
 
+	template<class Relocation, class Shortcutting>
+	static void PackScanner(const Scanner<Relocation, Shortcutting>& scanner, yostream* s)
+	{
+		Pire::Header hdr(1, 0);
+		char buf[256];
+		char *ptr;
+		size_t i;
+
+		ptr = mp_encode_array(buf, 2);
+		s->write(buf, ptr - buf);
+
+		hdr.Pack(s);
+
+		ptr = mp_encode_array(buf, 2);
+		ptr = mp_encode_bool(ptr, scanner.Empty());
+		if (scanner.Empty()) {
+			ptr = mp_encode_nil(ptr);
+			s->write(buf, ptr - buf);
+			return;
+		}
+		ptr = mp_encode_array(ptr, 9);
+		ptr = mp_encode_uint(ptr, scanner.m.statesCount);
+		ptr = mp_encode_uint(ptr, scanner.m.lettersCount);
+		ptr = mp_encode_uint(ptr, scanner.m.regexpsCount);
+		ptr = mp_encode_uint(ptr, scanner.StateIndex(scanner.m.initial));
+		ptr = mp_encode_uint(ptr, scanner.m.finalTableSize);
+		/* Pack is relocation safe, so we don't need to encode scanner.m.relocationSignature */
+		ptr = mp_encode_uint(ptr, scanner.m.shortcuttingSignature);
+
+		ptr = mp_encode_array(ptr, MaxCharUnaligned);
+		s->write(buf, ptr - buf);
+		for (i = 0; i != MaxCharUnaligned; i++) {
+			if (scanner.m_letters[i] != 0)
+				ptr = mp_encode_uint(buf, scanner.m_letters[i] - scanner.HEADER_SIZE);
+			else
+				ptr = mp_encode_nil(buf);
+			s->write(buf, ptr - buf);
+		}
+		ptr = mp_encode_array(buf, scanner.m.finalTableSize);
+		s->write(buf, ptr - buf);
+		for (i = 0; i != scanner.m.finalTableSize; i++) {
+			size_t out = scanner.m_final[i];
+
+			if (out == static_cast<size_t>(-1))
+				ptr = mp_encode_nil(buf);
+			else
+				ptr = mp_encode_uint(buf, out);
+			s->write(buf, ptr - buf);
+		}
+		ptr = mp_encode_array(buf, scanner.m.statesCount);
+		s->write(buf, ptr - buf);
+		for (i = 0; i != scanner.m.statesCount; i++) {
+			size_t st = scanner.IndexToState(i);
+			const typename Scanner<Relocation, Shortcutting>::ScannerRowHeader hdr = scanner.Header(st);
+			const typename Scanner<Relocation, Shortcutting>::Transition* os
+				= reinterpret_cast<const typename Scanner<Relocation, Shortcutting>::Transition*>(st);
+			ptr = mp_encode_array(buf, 4);
+			ptr = mp_encode_uint(ptr, scanner.m_finalIndex[i]);
+			/* save Header(st) content: flags & ExitMasks */
+			ptr = mp_encode_uint(ptr, hdr.Common.Flags);
+			ptr = mp_encode_array(ptr, Shortcutting::ExitMaskCount);
+			s->write(buf, ptr - buf);
+			for (size_t ind = 0; ind != Shortcutting::ExitMaskCount; ind++) {
+				size_t mask = Shortcutting::GetMaskRaw(hdr, ind);
+
+				/* write 'special mask' flag and then the mask itself */
+				ptr = mp_encode_array(buf, 2);
+				ptr = mp_encode_bool(ptr, mask != 0 && (mask & 0xff00) == 0);
+				ptr = mp_encode_uint(ptr, mask & 0xff);
+				s->write(buf, ptr - buf);
+			}
+			/* save transitions */
+			ptr = mp_encode_array(buf, scanner.LettersCount());
+			s->write(buf, ptr - buf);
+			for (size_t let = 0; let != scanner.LettersCount(); let++) {
+				size_t destIndex = scanner.StateIndex(Relocation::Go(st, os[let + scanner.HEADER_SIZE]));
+				ptr = mp_encode_uint(buf, destIndex);
+				s->write(buf, ptr - buf);
+			}
+		}
+	}
+
+	template<class Relocation, class Shortcutting>
+	static void UnpackScanner(Scanner<Relocation, Shortcutting>& scanner, yistream* s)
+	{
+		typedef Scanner<Relocation, Shortcutting> ScannerType;
+		Pire::Header hdr(1, 0);
+		Scanner<Relocation, Shortcutting> sc;
+		size_t i;
+
+		if (MsgpuckReadArray(s) != 2)
+			throw Error("Invalid format");
+		hdr.Unpack(s);
+
+		if (MsgpuckReadArray(s) != 2)
+			throw Error("Invalid format");
+		if (MsgpuckReadBool(s)) {
+			sc.Alias(ScannerType::Null());
+			MsgpuckReadNil(s);
+			scanner.Swap(sc);
+			return;
+		}
+		if (MsgpuckReadArray(s) != 9)
+			throw Error("Invalid format");
+		sc.m.statesCount = MsgpuckReadUint(s);
+		sc.m.lettersCount = MsgpuckReadUint(s);
+		sc.m.regexpsCount = MsgpuckReadUint(s);
+		sc.m.initial = MsgpuckReadUint(s);
+		sc.m.finalTableSize = MsgpuckReadUint(s);
+		sc.m.shortcuttingSignature = MsgpuckReadUint(s);
+		if (Shortcutting::Signature != sc.m.shortcuttingSignature)
+			throw Error("This scanner has different shortcutting type");
+		sc.m_buffer = new char[sc.BufSize() + sizeof(size_t)];
+		memset(sc.m_buffer, 0, sc.BufSize() + sizeof(size_t));
+		sc.Markup(AlignUp(sc.m_buffer, sizeof(size_t)));
+		sc.m_finalEnd = sc.m_final; // Actually not used
+		sc.m.initial = sc.IndexToState(sc.m.initial);
+
+		if (MsgpuckReadArray(s) != MaxCharUnaligned)
+			throw Error("Invalid format");
+		for (i = 0; i != MaxCharUnaligned; i++) {
+			if (MsgpuckTypeof(s) != MP_NIL)
+				sc.m_letters[i] = MsgpuckReadUint(s) + sc.HEADER_SIZE;
+			else
+				MsgpuckReadNil(s);
+		}
+		/* This should be filled for all the remaining letters */
+		for (; i != MaxChar; i++)
+			sc.m_letters[i] = sc.HEADER_SIZE;
+		if (MsgpuckReadArray(s) != sc.m.finalTableSize)
+			throw Error("Invalid format");
+		for (i = 0; i != sc.m.finalTableSize; i++) {
+			if (MsgpuckTypeof(s) != MP_NIL)
+				sc.m_final[i] = MsgpuckReadUint(s);
+			else {
+				sc.m_final[i] = static_cast<size_t>(-1);
+				MsgpuckReadNil(s);
+			}
+		}
+		if (MsgpuckReadArray(s) != sc.m.statesCount)
+			throw Error("Invalid format");
+		for (i = 0; i != sc.m.statesCount; i++) {
+			size_t st = sc.IndexToState(i);
+
+			sc.Header(st) = typename ScannerType::ScannerRowHeader();
+			typename ScannerType::ScannerRowHeader &hdr = sc.Header(st);
+			typename ScannerType::Transition* ns = reinterpret_cast<typename ScannerType::Transition*>(st);
+
+			if (MsgpuckReadArray(s) != 4)
+				throw Error("Invalid format");
+			sc.m_finalIndex[i] = MsgpuckReadUint(s);
+			hdr.Common.Flags = MsgpuckReadUint(s);
+
+			if (MsgpuckReadArray(s) != Shortcutting::ExitMaskCount)
+				throw Error("Invalid format");
+			for (size_t ind = 0; ind != Shortcutting::ExitMaskCount; ind++) {
+				bool special;
+				size_t val;
+
+				if (MsgpuckReadArray(s) != 2)
+					throw Error("Invalid format");
+				special = MsgpuckReadBool(s);
+				val = MsgpuckReadUint(s);
+				if (special)
+					Shortcutting::SetMaskRaw(hdr, ind, val);
+				else
+					Shortcutting::SetMask(hdr, ind, (char)val);
+			}
+			if (MsgpuckReadArray(s) != sc.LettersCount())
+				throw Error("Invalid format");
+			for (size_t let = 0; let != sc.LettersCount(); let++) {
+				size_t destIndex = MsgpuckReadUint(s);
+				typename ScannerType::Transition tr = Relocation::Diff(st, sc.IndexToState(destIndex));
+				ns[let + sc.HEADER_SIZE] = tr;
+			}
+		}
+		scanner.Swap(sc);
+	}
+
 	// TODO: implement more effective serialization
 	// of nonrelocatable scanner if necessary
 
@@ -623,6 +805,18 @@ void Scanner<Relocation, Shortcutting>::Load(yistream* s)
 	ScannerSaver::LoadScanner(*this, s);
 }
 
+template<class Relocation, class Shortcutting>
+void Scanner<Relocation, Shortcutting>::Pack(yostream* s) const
+{
+	ScannerSaver::PackScanner(*this, s);
+}
+
+template<class Relocation, class Shortcutting>
+void Scanner<Relocation, Shortcutting>::Unpack(yistream* s)
+{
+	ScannerSaver::UnpackScanner(*this, s);
+}
+
 template<class Relocation, class Shortcutting>
 const Scanner<Relocation, Shortcutting>* Scanner<Relocation, Shortcutting>::m_null = &Null();
 
@@ -772,6 +966,18 @@ class ExitMasks {
 		header.SetMask(0, NO_SHORTCUT_MASK);
 	}
 
+	template <class Header>
+	static size_t GetMaskRaw(Header& header, size_t ind)
+	{
+		return header.Mask(ind);
+	}
+
+	template <class Header>
+	static void SetMaskRaw(Header& header, size_t ind, size_t val)
+	{
+		return header.SetMask(ind, val);
+	}
+
 	template <class Header>
 	static void SetMask(Header& header, size_t ind, char c)
 	{
@@ -840,6 +1046,12 @@ struct NoShortcuts {
 	template <class Header>
 	static void SetNoShortcut(Header&) {}
 
+	template <class Header>
+	static size_t GetMaskRaw(Header&, size_t) {return 0;}
+
+	template <class Header>
+	static void SetMaskRaw(Header& header, size_t ind, size_t val) {}
+
 	template <class Header>
 	static void SetMask(Header&, size_t, char) {}
 

pire/stub/defaults.h

@@ -18,6 +18,12 @@
 #ifndef PIRE_STUB_DEFAULTS_H_INCLUDED
 #define PIRE_STUB_DEFAULTS_H_INCLUDED
 
+#if defined(__cplusplus) && !defined(__STDC_CONSTANT_MACROS)
+#define __STDC_CONSTANT_MACROS 1 /* make С++ to be happy */
+#endif
+#if defined(__cplusplus) && !defined(__STDC_LIMIT_MACROS)
+#define __STDC_LIMIT_MACROS 1    /* make С++ to be happy */
+#endif
 #include <stdint.h>
 #include <stddef.h>
 

pire/stub/msgpuck.c

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2013-2016 MsgPuck Authors
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ *    copyright notice, this list of conditions and the
+ *    following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ *    copyright notice, this list of conditions and the following
+ *    disclaimer in the documentation and/or other materials
+ *    provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#define MP_SOURCE 1
+#include "msgpuck.h"