1 /*******************************************************************************
2  * Serialization/deserialization code
3  *
4  * Author: Matthew Soucy, msoucy@csh.rit.edu
5  * Date: Oct 5, 2013
6  * Version: 0.0.2
7  */
8 module dproto.serialize;
9 
10 import dproto.exception;
11 
12 import std.algorithm;
13 import std.array;
14 import std.bitmanip;
15 import std.conv;
16 import std.exception;
17 import std.range;
18 import std.system : Endian;
19 
20 /*******************************************************************************
21  * Returns whether the given string is a protocol buffer primitive
22  *
23  * Params:
24  *  	type = The type to check for
25  * Returns: True if the type is a protocol buffer primitive
26  */
27 bool IsBuiltinType(string type) @safe pure nothrow {
28 	return ["int32" , "sint32", "int64", "sint64", "uint32", "uint64", "bool",
29 			"fixed64", "sfixed64", "double", "bytes", "string",
30 			"fixed32", "sfixed32", "float"].canFind(type);
31 }
32 
33 unittest {
34 	assert(IsBuiltinType("sfixed32") == true);
35 	assert(IsBuiltinType("double") == true);
36 	assert(IsBuiltinType("string") == true);
37 	assert(IsBuiltinType("int128") == false);
38 	assert(IsBuiltinType("quad") == false);
39 }
40 
41 /*******************************************************************************
42  * Maps the given type string to the data type it represents
43  */
44 template BuffType(string T) {
45 	// Msg type 0
46 	static if(T == "int32"  || T == "sint32") alias BuffType = int;
47 	else static if(T == "int64" || T == "sint64") alias BuffType = long;
48 	else static if(T == "uint32") alias BuffType = uint;
49 	else static if(T == "uint64") alias BuffType = ulong;
50 	else static if(T == "bool") alias BuffType = bool;
51 	// Msg type 1
52 	else static if(T == "fixed64") alias BuffType = ulong;
53 	else static if(T == "sfixed64") alias BuffType = long;
54 	else static if(T == "double") alias BuffType = double;
55 	// Msg type 2
56 	else static if(T == "bytes") alias BuffType = ubyte[];
57 	else static if(T == "string") alias BuffType = string;
58 	// Msg type 3,4 deprecated. Will not support.
59 	// Msg type 5
60 	else static if(T == "fixed32") alias BuffType = uint;
61 	else static if(T == "sfixed32") alias BuffType = int;
62 	else static if(T == "float") alias BuffType = float;
63 }
64 
65 unittest {
66 	assert(is(BuffType!"sfixed32" == int) == true);
67 	assert(is(BuffType!"double" == double) == true);
68 	assert(is(BuffType!"string" == string) == true);
69 	assert(is(BuffType!"bytes" == ubyte[]) == true);
70 	assert(is(BuffType!"sfixed64" == int) == false);
71 }
72 
73 /*******************************************************************************
74  * Removes bytes from the range as if it were read in
75  *
76  * Params:
77  *  	header = The data header
78  *  	data   = The data to read from
79  */
80 void defaultDecode(ulong header, ubyte[] data)
81 {
82 	switch(header.wireType) {
83 		case 0:
84 			data.readProto!"int32"();
85 			break;
86 		case 1:
87 			data.readProto!"fixed64"();
88 			break;
89 		case 2:
90 			data.readProto!"bytes"();
91 			break;
92 		case 5:
93 			data.readProto!"fixed32"();
94 			break;
95 		default:
96 			break;
97 	}
98 }
99 
100 /*******************************************************************************
101  * Maps the given type string to the wire type number
102  */
103 template MsgType(string T) {
104 	static if(T == "int32"  || T == "sint32" || T == "int64" || T == "sint64" ||
105 			T == "uint32" || T == "uint64" || T == "bool") {
106 		enum MsgType = 0;
107 	} else static if(T == "fixed64" || T == "sfixed64" || T == "double") {
108 		enum MsgType = 1;
109 	} else static if(T == "bytes" || T == "string") {
110 		enum MsgType = 2;
111 	} else static if(T == "fixed32" || T == "sfixed32" || T == "float") {
112 		enum MsgType = 5;
113 	} else {
114 		enum MsgType = 2;
115 	}
116 }
117 
118 /*******************************************************************************
119  * Encodes a number in its zigzag encoding
120  *
121  * Params:
122  *  	src = The raw integer to encode
123  * Returns: The zigzag-encoded value
124  */
125 ulong toZigZag(long src) @safe @property pure nothrow {
126 	return (src << 1) ^ (src >> 63);
127 }
128 
129 unittest {
130 	assert(0.toZigZag() == 0);
131 	assert((-1).toZigZag() == 1);
132 	assert(1.toZigZag() == 2);
133 	assert((-2).toZigZag() == 3);
134 	assert(2147483647.toZigZag() == 4294967294);
135 	assert((-2147483648).toZigZag() == 4294967295);
136 }
137 
138 /*******************************************************************************
139  * Decodes a number from its zigzag encoding
140  *
141  * Params:
142  *  	src = The zigzag-encoded value to decode
143  * Returns: The raw integer
144  */
145 long fromZigZag(ulong src) @safe @property pure nothrow {
146 	return (cast(long)(src >> 1)) ^ (cast(long)(-(src & 1)));
147 }
148 
149 unittest {
150 	assert(0.fromZigZag() == 0);
151 	assert(1.fromZigZag() == -1);
152 	assert(2.fromZigZag() == 1);
153 	assert(3.fromZigZag() == -2);
154 	assert(4294967294.fromZigZag() == 2147483647);
155 	assert(4294967295.fromZigZag() == -2147483648);
156 }
157 
158 /*******************************************************************************
159  * Get the wire type from the encoding value
160  *
161  * Params:
162  *  	data = The data header
163  * Returns: The wire type value
164  */
165 ubyte wireType(ulong data) @safe @property pure nothrow {
166 	return data&7;
167 }
168 
169 unittest {
170 	assert((0x08).wireType() == 0); // Test for varints
171 	assert((0x09).wireType() == 1); // Test 64-bit
172 	assert((0x12).wireType() == 2); // Test length-delimited
173 }
174 
175 /*******************************************************************************
176  * Get the message number from the encoding value
177  *
178  * Params:
179  *  	data = The data header
180  * Returns: The message number
181  */
182 ulong msgNum(ulong data) @safe @property pure nothrow {
183 	return data>>3;
184 }
185 
186 unittest {
187 	assert((0x08).msgNum() == 1);
188 	assert((0x11).msgNum() == 2);
189 	assert((0x1a).msgNum() == 3);
190 	assert((0x22).msgNum() == 4);
191 }
192 
193 /*******************************************************************************
194  * Read a VarInt-encoded value from a data stream
195  *
196  * Removes the bytes that represent the data from the stream
197  *
198  * Params:
199  *  	src = The data stream
200  * Returns: The decoded value
201  */
202 long readVarint(ref ubyte[] src) {
203 	size_t i = src.countUntil!( a=>!(a&0x80) )();
204 	auto ret = src[0..i+1].fromVarint();
205 	src = src[i+1..$];
206 	return ret;
207 }
208 
209 /*******************************************************************************
210  * Encode a value into a VarInt-encoded series of bytes
211  *
212  * Params:
213  *  	src = The value to encode
214  * Returns: The created VarInt
215  */
216 ubyte[] toVarint(long src) @property pure nothrow {
217 	ubyte[] ret;
218 	while(src&(~0x7FUL)) {
219 		ret ~= 0x80 | src&0x7F;
220 		src >>= 7;
221 	}
222 	ret ~= src&0x7F;
223 	return ret;
224 }
225 
226 unittest {
227 	assert(150.toVarint == [0x96, 0x01]);
228 	assert(3.toVarint == [0x03]);
229 	assert(270.toVarint == [0x8E, 0x02]);
230 	assert(86942.toVarint == [0x9E, 0xA7, 0x05]);
231 }
232 
233 /*******************************************************************************
234  * Decode a VarInt-encoded series of bytes into a value
235  *
236  * Params:
237  *  	src = The data stream
238  * Returns: The decoded value
239  */
240 long fromVarint(ubyte[] src) @property {
241 	return 0L.reduce!((a,b) => (a<<7)|(b&0x7F))(src.retro());
242 }
243 
244 unittest {
245 	assert([0x96, 0x01].fromVarint() == 150);
246 	assert([0x03].fromVarint() == 3);
247 	assert([0x8E, 0x02].fromVarint() == 270);
248 	assert([0x9E, 0xA7, 0x05].fromVarint() == 86942);
249 }
250 
251 /// The type to encode an enum as
252 enum ENUM_SERIALIZATION = "int32";
253 /// The message type to encode a packed message as
254 enum PACKED_MSG_TYPE = 2;
255 
256 /*******************************************************************************
257  * Decode a series of bytes into a value
258  *
259  * Params:
260  *  	src = The data stream
261  * Returns: The decoded value
262  */
263 BuffType!T readProto(string T)(ref ubyte[] src)
264 	if(T == "int32" || T == "int64" || T == "uint32" || T == "uint64" || T == "bool")
265 {
266 	return src.readVarint().to!(BuffType!T)();
267 }
268 
269 /// Ditto
270 BuffType!T readProto(string T)(ref ubyte[] src)
271 	if(T == "sint32" || T == "sint64")
272 {
273 	return src.readVarint().fromZigZag().to!(BuffType!T)();
274 }
275 
276 /// Ditto
277 BuffType!T readProto(string T)(ref ubyte[] src)
278 	if(T == "double" || T == "fixed64" || T == "sfixed64" ||
279 		T == "float" || T == "fixed32" || T == "sfixed32")
280 {
281 	enforce(src.length >= BuffType!T.sizeof, new DProtoException("Not enough data in buffer"));
282 	return src.read!(BuffType!T, Endian.littleEndian)();
283 }
284 
285 /// Ditto
286 BuffType!T readProto(string T)(ref ubyte[] src)
287 	if(T == "string" || T == "bytes")
288 {
289 	auto length = src.readProto!"uint32"();
290 	enforce(src.length >= length, new DProtoException("Not enough data in buffer"));
291 	auto s = cast(BuffType!T)(src.take(length));
292 	src=src[length..$];
293 	return s;
294 }
295 
296 /*******************************************************************************
297  * Encode a value into a series of bytes
298  *
299  * Params:
300  *  	src = The raw data
301  * Returns: The encoded value
302  */
303 ubyte[] writeProto(string T)(BuffType!T src)
304 	if(T == "int32" || T == "int64" || T == "uint32" || T == "uint64" || T == "bool")
305 {
306 	return src.toVarint().dup;
307 }
308 
309 /// Ditto
310 ubyte[] writeProto(string T)(BuffType!T src)
311 	if(T == "sint32" || T == "sint64")
312 {
313 	return src.toZigZag().toVarint().dup;
314 }
315 
316 /// Ditto
317 ubyte[] writeProto(string T)(BuffType!T src)
318 	if(T == "double" || T == "fixed64" || T == "sfixed64" ||
319 		T == "float" || T == "fixed32" || T == "sfixed32")
320 {
321 	return src.nativeToLittleEndian!(BuffType!T)().dup;
322 }
323 
324 /// Ditto
325 ubyte[] writeProto(string T)(BuffType!T src)
326 	if(T == "string" || T == "bytes")
327 {
328 	return src.length.toVarint() ~ cast(ubyte[])(src);
329 }