1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31 """Contains routines for printing protocol messages in JSON format.
32
33 Simple usage example:
34
35 # Create a proto object and serialize it to a json format string.
36 message = my_proto_pb2.MyMessage(foo='bar')
37 json_string = json_format.MessageToJson(message)
38
39 # Parse a json format string to proto object.
40 message = json_format.Parse(json_string, my_proto_pb2.MyMessage())
41 """
42
43 __author__ = 'jieluo@google.com (Jie Luo)'
44
45
46 try:
47 from collections import OrderedDict
48 except ImportError:
49 from ordereddict import OrderedDict
50
51
52 import base64
53 import json
54 import math
55
56 from operator import methodcaller
57
58 import re
59 import sys
60
61 import six
62
63 from google.protobuf import descriptor
64 from google.protobuf import symbol_database
65
66
67 _TIMESTAMPFOMAT = '%Y-%m-%dT%H:%M:%S'
68 _INT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT32,
69 descriptor.FieldDescriptor.CPPTYPE_UINT32,
70 descriptor.FieldDescriptor.CPPTYPE_INT64,
71 descriptor.FieldDescriptor.CPPTYPE_UINT64])
72 _INT64_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT64,
73 descriptor.FieldDescriptor.CPPTYPE_UINT64])
74 _FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT,
75 descriptor.FieldDescriptor.CPPTYPE_DOUBLE])
76 _INFINITY = 'Infinity'
77 _NEG_INFINITY = '-Infinity'
78 _NAN = 'NaN'
79
80 _UNPAIRED_SURROGATE_PATTERN = re.compile(six.u(
81 r'[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]'
82 ))
83
84 _VALID_EXTENSION_NAME = re.compile(r'\[[a-zA-Z0-9\._]*\]$')
85
86
88 """Top-level module error for json_format."""
89
90
92 """Thrown if serialization to JSON fails."""
93
94
96 """Thrown in case of parsing error."""
97
98
99 -def MessageToJson(
100 message,
101 including_default_value_fields=False,
102 preserving_proto_field_name=False,
103 indent=2,
104 sort_keys=False,
105 use_integers_for_enums=False,
106 descriptor_pool=None):
107 """Converts protobuf message to JSON format.
108
109 Args:
110 message: The protocol buffers message instance to serialize.
111 including_default_value_fields: If True, singular primitive fields,
112 repeated fields, and map fields will always be serialized. If
113 False, only serialize non-empty fields. Singular message fields
114 and oneof fields are not affected by this option.
115 preserving_proto_field_name: If True, use the original proto field
116 names as defined in the .proto file. If False, convert the field
117 names to lowerCamelCase.
118 indent: The JSON object will be pretty-printed with this indent level.
119 An indent level of 0 or negative will only insert newlines.
120 sort_keys: If True, then the output will be sorted by field names.
121 use_integers_for_enums: If true, print integers instead of enum names.
122 descriptor_pool: A Descriptor Pool for resolving types. If None use the
123 default.
124
125 Returns:
126 A string containing the JSON formatted protocol buffer message.
127 """
128 printer = _Printer(
129 including_default_value_fields,
130 preserving_proto_field_name,
131 use_integers_for_enums,
132 descriptor_pool)
133 return printer.ToJsonString(message, indent, sort_keys)
134
135
136 -def MessageToDict(
137 message,
138 including_default_value_fields=False,
139 preserving_proto_field_name=False,
140 use_integers_for_enums=False,
141 descriptor_pool=None):
142 """Converts protobuf message to a dictionary.
143
144 When the dictionary is encoded to JSON, it conforms to proto3 JSON spec.
145
146 Args:
147 message: The protocol buffers message instance to serialize.
148 including_default_value_fields: If True, singular primitive fields,
149 repeated fields, and map fields will always be serialized. If
150 False, only serialize non-empty fields. Singular message fields
151 and oneof fields are not affected by this option.
152 preserving_proto_field_name: If True, use the original proto field
153 names as defined in the .proto file. If False, convert the field
154 names to lowerCamelCase.
155 use_integers_for_enums: If true, print integers instead of enum names.
156 descriptor_pool: A Descriptor Pool for resolving types. If None use the
157 default.
158
159 Returns:
160 A dict representation of the protocol buffer message.
161 """
162 printer = _Printer(
163 including_default_value_fields,
164 preserving_proto_field_name,
165 use_integers_for_enums,
166 descriptor_pool)
167
168 return printer._MessageToJsonObject(message)
169
170
171 -def _IsMapEntry(field):
172 return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and
173 field.message_type.has_options and
174 field.message_type.GetOptions().map_entry)
175
176
178 """JSON format printer for protocol message."""
179
180 - def __init__(
181 self,
182 including_default_value_fields=False,
183 preserving_proto_field_name=False,
184 use_integers_for_enums=False,
185 descriptor_pool=None):
186 self.including_default_value_fields = including_default_value_fields
187 self.preserving_proto_field_name = preserving_proto_field_name
188 self.use_integers_for_enums = use_integers_for_enums
189 self.descriptor_pool = descriptor_pool
190
194
205
207 """Converts normal message according to Proto3 JSON Specification."""
208 fields = message.ListFields()
209
210 try:
211 for field, value in fields:
212 if self.preserving_proto_field_name:
213 name = field.name
214 else:
215 name = field.json_name
216 if _IsMapEntry(field):
217
218 v_field = field.message_type.fields_by_name['value']
219 js_map = {}
220 for key in value:
221 if isinstance(key, bool):
222 if key:
223 recorded_key = 'true'
224 else:
225 recorded_key = 'false'
226 else:
227 recorded_key = key
228 js_map[recorded_key] = self._FieldToJsonObject(
229 v_field, value[key])
230 js[name] = js_map
231 elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
232
233 js[name] = [self._FieldToJsonObject(field, k)
234 for k in value]
235 elif field.is_extension:
236 full_qualifier = field.full_name[:-len(field.name)]
237 name = '[%s%s]' % (full_qualifier, name)
238 js[name] = self._FieldToJsonObject(field, value)
239 else:
240 js[name] = self._FieldToJsonObject(field, value)
241
242
243 if self.including_default_value_fields:
244 message_descriptor = message.DESCRIPTOR
245 for field in message_descriptor.fields:
246
247 if ((field.label != descriptor.FieldDescriptor.LABEL_REPEATED and
248 field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE) or
249 field.containing_oneof):
250 continue
251 if self.preserving_proto_field_name:
252 name = field.name
253 else:
254 name = field.json_name
255 if name in js:
256
257 continue
258 if _IsMapEntry(field):
259 js[name] = {}
260 elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
261 js[name] = []
262 else:
263 js[name] = self._FieldToJsonObject(field, field.default_value)
264
265 except ValueError as e:
266 raise SerializeToJsonError(
267 'Failed to serialize {0} field: {1}.'.format(field.name, e))
268
269 return js
270
305
326
328 """Converts message according to Proto3 JSON Specification."""
329
330
331 return message.ToJsonString()
332
334 """Converts Value message according to Proto3 JSON Specification."""
335 which = message.WhichOneof('kind')
336
337
338 if which is None or which == 'null_value':
339 return None
340 if which == 'list_value':
341 return self._ListValueMessageToJsonObject(message.list_value)
342 if which == 'struct_value':
343 value = message.struct_value
344 else:
345 value = getattr(message, which)
346 oneof_descriptor = message.DESCRIPTOR.fields_by_name[which]
347 return self._FieldToJsonObject(oneof_descriptor, value)
348
353
355 """Converts Struct message according to Proto3 JSON Specification."""
356 fields = message.fields
357 ret = {}
358 for key in fields:
359 ret[key] = self._ValueMessageToJsonObject(fields[key])
360 return ret
361
365
366
368 return message_descriptor.file.name == 'google/protobuf/wrappers.proto'
369
370
372 result = {}
373 for name, value in js:
374 if name in result:
375 raise ParseError('Failed to load JSON: duplicate key {0}.'.format(name))
376 result[name] = value
377 return result
378
379
381 """Creates a message from a type URL."""
382 db = symbol_database.Default()
383 pool = db.pool if descriptor_pool is None else descriptor_pool
384 type_name = type_url.split('/')[-1]
385 try:
386 message_descriptor = pool.FindMessageTypeByName(type_name)
387 except KeyError:
388 raise TypeError(
389 'Can not find message descriptor by type_url: {0}.'.format(type_url))
390 message_class = db.GetPrototype(message_descriptor)
391 return message_class()
392
393
394 -def Parse(text, message, ignore_unknown_fields=False, descriptor_pool=None):
395 """Parses a JSON representation of a protocol message into a message.
396
397 Args:
398 text: Message JSON representation.
399 message: A protocol buffer message to merge into.
400 ignore_unknown_fields: If True, do not raise errors for unknown fields.
401 descriptor_pool: A Descriptor Pool for resolving types. If None use the
402 default.
403
404 Returns:
405 The same message passed as argument.
406
407 Raises::
408 ParseError: On JSON parsing problems.
409 """
410 if not isinstance(text, six.text_type): text = text.decode('utf-8')
411 try:
412 js = json.loads(text, object_pairs_hook=_DuplicateChecker)
413 except ValueError as e:
414 raise ParseError('Failed to load JSON: {0}.'.format(str(e)))
415 return ParseDict(js, message, ignore_unknown_fields, descriptor_pool)
416
417
418 -def ParseDict(js_dict,
419 message,
420 ignore_unknown_fields=False,
421 descriptor_pool=None):
422 """Parses a JSON dictionary representation into a message.
423
424 Args:
425 js_dict: Dict representation of a JSON message.
426 message: A protocol buffer message to merge into.
427 ignore_unknown_fields: If True, do not raise errors for unknown fields.
428 descriptor_pool: A Descriptor Pool for resolving types. If None use the
429 default.
430
431 Returns:
432 The same message passed as argument.
433 """
434 parser = _Parser(ignore_unknown_fields, descriptor_pool)
435 parser.ConvertMessage(js_dict, message)
436 return message
437
438
439 _INT_OR_FLOAT = six.integer_types + (float,)
440
441
443 """JSON format parser for protocol message."""
444
445 - def __init__(self, ignore_unknown_fields, descriptor_pool):
448
467
469 """Convert field value pairs into regular message.
470
471 Args:
472 js: A JSON object to convert the field value pairs.
473 message: A regular protocol message to record the data.
474
475 Raises:
476 ParseError: In case of problems converting.
477 """
478 names = []
479 message_descriptor = message.DESCRIPTOR
480 fields_by_json_name = dict((f.json_name, f)
481 for f in message_descriptor.fields)
482 for name in js:
483 try:
484 field = fields_by_json_name.get(name, None)
485 if not field:
486 field = message_descriptor.fields_by_name.get(name, None)
487 if not field and _VALID_EXTENSION_NAME.match(name):
488 if not message_descriptor.is_extendable:
489 raise ParseError('Message type {0} does not have extensions'.format(
490 message_descriptor.full_name))
491 identifier = name[1:-1]
492
493 field = message.Extensions._FindExtensionByName(identifier)
494
495 if not field:
496
497
498 identifier = '.'.join(identifier.split('.')[:-1])
499
500 field = message.Extensions._FindExtensionByName(identifier)
501
502 if not field:
503 if self.ignore_unknown_fields:
504 continue
505 raise ParseError(
506 ('Message type "{0}" has no field named "{1}".\n'
507 ' Available Fields(except extensions): {2}').format(
508 message_descriptor.full_name, name,
509 [f.json_name for f in message_descriptor.fields]))
510 if name in names:
511 raise ParseError('Message type "{0}" should not have multiple '
512 '"{1}" fields.'.format(
513 message.DESCRIPTOR.full_name, name))
514 names.append(name)
515
516 if field.containing_oneof is not None:
517 oneof_name = field.containing_oneof.name
518 if oneof_name in names:
519 raise ParseError('Message type "{0}" should not have multiple '
520 '"{1}" oneof fields.'.format(
521 message.DESCRIPTOR.full_name, oneof_name))
522 names.append(oneof_name)
523
524 value = js[name]
525 if value is None:
526 if (field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE
527 and field.message_type.full_name == 'google.protobuf.Value'):
528 sub_message = getattr(message, field.name)
529 sub_message.null_value = 0
530 else:
531 message.ClearField(field.name)
532 continue
533
534
535 if _IsMapEntry(field):
536 message.ClearField(field.name)
537 self._ConvertMapFieldValue(value, message, field)
538 elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
539 message.ClearField(field.name)
540 if not isinstance(value, list):
541 raise ParseError('repeated field {0} must be in [] which is '
542 '{1}.'.format(name, value))
543 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
544
545 for item in value:
546 sub_message = getattr(message, field.name).add()
547
548 if (item is None and
549 sub_message.DESCRIPTOR.full_name != 'google.protobuf.Value'):
550 raise ParseError('null is not allowed to be used as an element'
551 ' in a repeated field.')
552 self.ConvertMessage(item, sub_message)
553 else:
554
555 for item in value:
556 if item is None:
557 raise ParseError('null is not allowed to be used as an element'
558 ' in a repeated field.')
559 getattr(message, field.name).append(
560 _ConvertScalarFieldValue(item, field))
561 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
562 if field.is_extension:
563 sub_message = message.Extensions[field]
564 else:
565 sub_message = getattr(message, field.name)
566 sub_message.SetInParent()
567 self.ConvertMessage(value, sub_message)
568 else:
569 if field.is_extension:
570 message.Extensions[field] = _ConvertScalarFieldValue(value, field)
571 else:
572 setattr(message, field.name, _ConvertScalarFieldValue(value, field))
573 except ParseError as e:
574 if field and field.containing_oneof is None:
575 raise ParseError('Failed to parse {0} field: {1}.'.format(name, e))
576 else:
577 raise ParseError(str(e))
578 except ValueError as e:
579 raise ParseError('Failed to parse {0} field: {1}.'.format(name, e))
580 except TypeError as e:
581 raise ParseError('Failed to parse {0} field: {1}.'.format(name, e))
582
607
609 """Convert a JSON representation into message with FromJsonString."""
610
611
612 try:
613 message.FromJsonString(value)
614 except ValueError as e:
615 raise ParseError(e)
616
618 """Convert a JSON representation into Value message."""
619 if isinstance(value, dict):
620 self._ConvertStructMessage(value, message.struct_value)
621 elif isinstance(value, list):
622 self. _ConvertListValueMessage(value, message.list_value)
623 elif value is None:
624 message.null_value = 0
625 elif isinstance(value, bool):
626 message.bool_value = value
627 elif isinstance(value, six.string_types):
628 message.string_value = value
629 elif isinstance(value, _INT_OR_FLOAT):
630 message.number_value = value
631 else:
632 raise ParseError('Unexpected type for Value message.')
633
635 """Convert a JSON representation into ListValue message."""
636 if not isinstance(value, list):
637 raise ParseError(
638 'ListValue must be in [] which is {0}.'.format(value))
639 message.ClearField('values')
640 for item in value:
641 self._ConvertValueMessage(item, message.values.add())
642
644 """Convert a JSON representation into Struct message."""
645 if not isinstance(value, dict):
646 raise ParseError(
647 'Struct must be in a dict which is {0}.'.format(value))
648
649
650 message.Clear()
651 for key in value:
652 self._ConvertValueMessage(value[key], message.fields[key])
653 return
654
659
661 """Convert map field value for a message map field.
662
663 Args:
664 value: A JSON object to convert the map field value.
665 message: A protocol message to record the converted data.
666 field: The descriptor of the map field to be converted.
667
668 Raises:
669 ParseError: In case of convert problems.
670 """
671 if not isinstance(value, dict):
672 raise ParseError(
673 'Map field {0} must be in a dict which is {1}.'.format(
674 field.name, value))
675 key_field = field.message_type.fields_by_name['key']
676 value_field = field.message_type.fields_by_name['value']
677 for key in value:
678 key_value = _ConvertScalarFieldValue(key, key_field, True)
679 if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
680 self.ConvertMessage(value[key], getattr(
681 message, field.name)[key_value])
682 else:
683 getattr(message, field.name)[key_value] = _ConvertScalarFieldValue(
684 value[key], value_field)
685
686
688 """Convert a single scalar field value.
689
690 Args:
691 value: A scalar value to convert the scalar field value.
692 field: The descriptor of the field to convert.
693 require_str: If True, the field value must be a str.
694
695 Returns:
696 The converted scalar field value
697
698 Raises:
699 ParseError: In case of convert problems.
700 """
701 if field.cpp_type in _INT_TYPES:
702 return _ConvertInteger(value)
703 elif field.cpp_type in _FLOAT_TYPES:
704 return _ConvertFloat(value)
705 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
706 return _ConvertBool(value, require_str)
707 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
708 if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
709 return base64.b64decode(value)
710 else:
711
712
713 if _UNPAIRED_SURROGATE_PATTERN.search(value):
714 raise ParseError('Unpaired surrogate')
715 return value
716 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
717
718 enum_value = field.enum_type.values_by_name.get(value, None)
719 if enum_value is None:
720 try:
721 number = int(value)
722 enum_value = field.enum_type.values_by_number.get(number, None)
723 except ValueError:
724 raise ParseError('Invalid enum value {0} for enum type {1}.'.format(
725 value, field.enum_type.full_name))
726 if enum_value is None:
727 if field.file.syntax == 'proto3':
728
729 return number
730 raise ParseError('Invalid enum value {0} for enum type {1}.'.format(
731 value, field.enum_type.full_name))
732 return enum_value.number
733
734
736 """Convert an integer.
737
738 Args:
739 value: A scalar value to convert.
740
741 Returns:
742 The integer value.
743
744 Raises:
745 ParseError: If an integer couldn't be consumed.
746 """
747 if isinstance(value, float) and not value.is_integer():
748 raise ParseError('Couldn\'t parse integer: {0}.'.format(value))
749
750 if isinstance(value, six.text_type) and value.find(' ') != -1:
751 raise ParseError('Couldn\'t parse integer: "{0}".'.format(value))
752
753 return int(value)
754
755
757 """Convert an floating point number."""
758 if value == 'nan':
759 raise ParseError('Couldn\'t parse float "nan", use "NaN" instead.')
760 try:
761
762 return float(value)
763 except ValueError:
764
765 if value == _NEG_INFINITY:
766 return float('-inf')
767 elif value == _INFINITY:
768 return float('inf')
769 elif value == _NAN:
770 return float('nan')
771 else:
772 raise ParseError('Couldn\'t parse float: {0}.'.format(value))
773
774
776 """Convert a boolean value.
777
778 Args:
779 value: A scalar value to convert.
780 require_str: If True, value must be a str.
781
782 Returns:
783 The bool parsed.
784
785 Raises:
786 ParseError: If a boolean value couldn't be consumed.
787 """
788 if require_str:
789 if value == 'true':
790 return True
791 elif value == 'false':
792 return False
793 else:
794 raise ParseError('Expected "true" or "false", not {0}.'.format(value))
795
796 if not isinstance(value, bool):
797 raise ParseError('Expected true or false without quotes.')
798 return value
799
800 _WKTJSONMETHODS = {
801 'google.protobuf.Any': ['_AnyMessageToJsonObject',
802 '_ConvertAnyMessage'],
803 'google.protobuf.Duration': ['_GenericMessageToJsonObject',
804 '_ConvertGenericMessage'],
805 'google.protobuf.FieldMask': ['_GenericMessageToJsonObject',
806 '_ConvertGenericMessage'],
807 'google.protobuf.ListValue': ['_ListValueMessageToJsonObject',
808 '_ConvertListValueMessage'],
809 'google.protobuf.Struct': ['_StructMessageToJsonObject',
810 '_ConvertStructMessage'],
811 'google.protobuf.Timestamp': ['_GenericMessageToJsonObject',
812 '_ConvertGenericMessage'],
813 'google.protobuf.Value': ['_ValueMessageToJsonObject',
814 '_ConvertValueMessage']
815 }
816