1 /******************************************************************************* 2 * Serialization/deserialization code 3 * 4 * Author: Matthew Soucy, msoucy@csh.rit.edu 5 * Date: Oct 5, 2013 6 * Version: 0.0.2 7 */ 8 module dproto.serialize; 9 10 import dproto.exception; 11 12 import std.algorithm; 13 import std.array; 14 import std.bitmanip; 15 import std.conv; 16 import std.exception; 17 import std.range; 18 import std.system : Endian; 19 20 /******************************************************************************* 21 * Returns whether the given string is a protocol buffer primitive 22 * 23 * Params: 24 * type = The type to check for 25 * Returns: True if the type is a protocol buffer primitive 26 */ 27 bool IsBuiltinType(string type) @safe pure nothrow { 28 return ["int32" , "sint32", "int64", "sint64", "uint32", "uint64", "bool", 29 "fixed64", "sfixed64", "double", "bytes", "string", 30 "fixed32", "sfixed32", "float"].canFind(type); 31 } 32 33 unittest { 34 assert(IsBuiltinType("sfixed32") == true); 35 assert(IsBuiltinType("double") == true); 36 assert(IsBuiltinType("string") == true); 37 assert(IsBuiltinType("int128") == false); 38 assert(IsBuiltinType("quad") == false); 39 } 40 41 /******************************************************************************* 42 * Maps the given type string to the data type it represents 43 */ 44 template BuffType(string T) { 45 // Msg type 0 46 static if(T == "int32" || T == "sint32") alias BuffType = int; 47 else static if(T == "int64" || T == "sint64") alias BuffType = long; 48 else static if(T == "uint32") alias BuffType = uint; 49 else static if(T == "uint64") alias BuffType = ulong; 50 else static if(T == "bool") alias BuffType = bool; 51 // Msg type 1 52 else static if(T == "fixed64") alias BuffType = ulong; 53 else static if(T == "sfixed64") alias BuffType = long; 54 else static if(T == "double") alias BuffType = double; 55 // Msg type 2 56 else static if(T == "bytes") alias BuffType = ubyte[]; 57 else static if(T == "string") alias BuffType = string; 58 // Msg type 3,4 deprecated. Will not support. 59 // Msg type 5 60 else static if(T == "fixed32") alias BuffType = uint; 61 else static if(T == "sfixed32") alias BuffType = int; 62 else static if(T == "float") alias BuffType = float; 63 } 64 65 unittest { 66 assert(is(BuffType!"sfixed32" == int) == true); 67 assert(is(BuffType!"double" == double) == true); 68 assert(is(BuffType!"string" == string) == true); 69 assert(is(BuffType!"bytes" == ubyte[]) == true); 70 assert(is(BuffType!"sfixed64" == int) == false); 71 } 72 73 /******************************************************************************* 74 * Removes bytes from the range as if it were read in 75 * 76 * Params: 77 * header = The data header 78 * data = The data to read from 79 */ 80 void defaultDecode(ulong header, ubyte[] data) 81 { 82 switch(header.wireType) { 83 case 0: 84 data.readProto!"int32"(); 85 break; 86 case 1: 87 data.readProto!"fixed64"(); 88 break; 89 case 2: 90 data.readProto!"bytes"(); 91 break; 92 case 5: 93 data.readProto!"fixed32"(); 94 break; 95 default: 96 break; 97 } 98 } 99 100 /******************************************************************************* 101 * Maps the given type string to the wire type number 102 */ 103 template MsgType(string T) { 104 static if(T == "int32" || T == "sint32" || T == "int64" || T == "sint64" || 105 T == "uint32" || T == "uint64" || T == "bool") { 106 enum MsgType = 0; 107 } else static if(T == "fixed64" || T == "sfixed64" || T == "double") { 108 enum MsgType = 1; 109 } else static if(T == "bytes" || T == "string") { 110 enum MsgType = 2; 111 } else static if(T == "fixed32" || T == "sfixed32" || T == "float") { 112 enum MsgType = 5; 113 } else { 114 enum MsgType = 2; 115 } 116 } 117 118 /******************************************************************************* 119 * Encodes a number in its zigzag encoding 120 * 121 * Params: 122 * src = The raw integer to encode 123 * Returns: The zigzag-encoded value 124 */ 125 ulong toZigZag(long src) @safe @property pure nothrow { 126 return (src << 1) ^ (src >> 63); 127 } 128 129 unittest { 130 assert(0.toZigZag() == 0); 131 assert((-1).toZigZag() == 1); 132 assert(1.toZigZag() == 2); 133 assert((-2).toZigZag() == 3); 134 assert(2147483647.toZigZag() == 4294967294); 135 assert((-2147483648).toZigZag() == 4294967295); 136 } 137 138 /******************************************************************************* 139 * Decodes a number from its zigzag encoding 140 * 141 * Params: 142 * src = The zigzag-encoded value to decode 143 * Returns: The raw integer 144 */ 145 long fromZigZag(ulong src) @safe @property pure nothrow { 146 return (cast(long)(src >> 1)) ^ (cast(long)(-(src & 1))); 147 } 148 149 unittest { 150 assert(0.fromZigZag() == 0); 151 assert(1.fromZigZag() == -1); 152 assert(2.fromZigZag() == 1); 153 assert(3.fromZigZag() == -2); 154 assert(4294967294.fromZigZag() == 2147483647); 155 assert(4294967295.fromZigZag() == -2147483648); 156 } 157 158 /******************************************************************************* 159 * Get the wire type from the encoding value 160 * 161 * Params: 162 * data = The data header 163 * Returns: The wire type value 164 */ 165 ubyte wireType(ulong data) @safe @property pure nothrow { 166 return data&7; 167 } 168 169 unittest { 170 assert((0x08).wireType() == 0); // Test for varints 171 assert((0x09).wireType() == 1); // Test 64-bit 172 assert((0x12).wireType() == 2); // Test length-delimited 173 } 174 175 /******************************************************************************* 176 * Get the message number from the encoding value 177 * 178 * Params: 179 * data = The data header 180 * Returns: The message number 181 */ 182 ulong msgNum(ulong data) @safe @property pure nothrow { 183 return data>>3; 184 } 185 186 unittest { 187 assert((0x08).msgNum() == 1); 188 assert((0x11).msgNum() == 2); 189 assert((0x1a).msgNum() == 3); 190 assert((0x22).msgNum() == 4); 191 } 192 193 /******************************************************************************* 194 * Read a VarInt-encoded value from a data stream 195 * 196 * Removes the bytes that represent the data from the stream 197 * 198 * Params: 199 * src = The data stream 200 * Returns: The decoded value 201 */ 202 long readVarint(ref ubyte[] src) { 203 size_t i = src.countUntil!( a=>!(a&0x80) )(); 204 auto ret = src[0..i+1].fromVarint(); 205 src = src[i+1..$]; 206 return ret; 207 } 208 209 /******************************************************************************* 210 * Encode a value into a VarInt-encoded series of bytes 211 * 212 * Params: 213 * src = The value to encode 214 * Returns: The created VarInt 215 */ 216 ubyte[] toVarint(long src) @property pure nothrow { 217 ubyte[] ret; 218 while(src&(~0x7FUL)) { 219 ret ~= 0x80 | src&0x7F; 220 src >>= 7; 221 } 222 ret ~= src&0x7F; 223 return ret; 224 } 225 226 unittest { 227 assert(150.toVarint == [0x96, 0x01]); 228 assert(3.toVarint == [0x03]); 229 assert(270.toVarint == [0x8E, 0x02]); 230 assert(86942.toVarint == [0x9E, 0xA7, 0x05]); 231 } 232 233 /******************************************************************************* 234 * Decode a VarInt-encoded series of bytes into a value 235 * 236 * Params: 237 * src = The data stream 238 * Returns: The decoded value 239 */ 240 long fromVarint(ubyte[] src) @property { 241 return 0L.reduce!((a,b) => (a<<7)|(b&0x7F))(src.retro()); 242 } 243 244 unittest { 245 assert([0x96, 0x01].fromVarint() == 150); 246 assert([0x03].fromVarint() == 3); 247 assert([0x8E, 0x02].fromVarint() == 270); 248 assert([0x9E, 0xA7, 0x05].fromVarint() == 86942); 249 } 250 251 /// The type to encode an enum as 252 enum ENUM_SERIALIZATION = "int32"; 253 /// The message type to encode a packed message as 254 enum PACKED_MSG_TYPE = 2; 255 256 /******************************************************************************* 257 * Decode a series of bytes into a value 258 * 259 * Params: 260 * src = The data stream 261 * Returns: The decoded value 262 */ 263 BuffType!T readProto(string T)(ref ubyte[] src) 264 if(T == "int32" || T == "int64" || T == "uint32" || T == "uint64" || T == "bool") 265 { 266 return src.readVarint().to!(BuffType!T)(); 267 } 268 269 /// Ditto 270 BuffType!T readProto(string T)(ref ubyte[] src) 271 if(T == "sint32" || T == "sint64") 272 { 273 return src.readVarint().fromZigZag().to!(BuffType!T)(); 274 } 275 276 /// Ditto 277 BuffType!T readProto(string T)(ref ubyte[] src) 278 if(T == "double" || T == "fixed64" || T == "sfixed64" || 279 T == "float" || T == "fixed32" || T == "sfixed32") 280 { 281 enforce(src.length >= BuffType!T.sizeof, new DProtoException("Not enough data in buffer")); 282 return src.read!(BuffType!T, Endian.littleEndian)(); 283 } 284 285 /// Ditto 286 BuffType!T readProto(string T)(ref ubyte[] src) 287 if(T == "string" || T == "bytes") 288 { 289 auto length = src.readProto!"uint32"(); 290 enforce(src.length >= length, new DProtoException("Not enough data in buffer")); 291 auto s = cast(BuffType!T)(src.take(length)); 292 src=src[length..$]; 293 return s; 294 } 295 296 /******************************************************************************* 297 * Encode a value into a series of bytes 298 * 299 * Params: 300 * src = The raw data 301 * Returns: The encoded value 302 */ 303 ubyte[] writeProto(string T)(BuffType!T src) 304 if(T == "int32" || T == "int64" || T == "uint32" || T == "uint64" || T == "bool") 305 { 306 return src.toVarint().dup; 307 } 308 309 /// Ditto 310 ubyte[] writeProto(string T)(BuffType!T src) 311 if(T == "sint32" || T == "sint64") 312 { 313 return src.toZigZag().toVarint().dup; 314 } 315 316 /// Ditto 317 ubyte[] writeProto(string T)(BuffType!T src) 318 if(T == "double" || T == "fixed64" || T == "sfixed64" || 319 T == "float" || T == "fixed32" || T == "sfixed32") 320 { 321 return src.nativeToLittleEndian!(BuffType!T)().dup; 322 } 323 324 /// Ditto 325 ubyte[] writeProto(string T)(BuffType!T src) 326 if(T == "string" || T == "bytes") 327 { 328 return src.length.toVarint() ~ cast(ubyte[])(src); 329 }