Extend inline / fixed length bytes array support (issue #244)

[apps/agl-service-can-low-level.git] / generator / nanopb_generator.py

diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py
index 0d7be49..9cce6a5 100755 (executable)
--- a/generator/nanopb_generator.py
+++ b/generator/nanopb_generator.py
@@ -1,9 +1,13 @@
-#!/usr/bin/python
+#!/usr/bin/env python
+
+from __future__ import unicode_literals

 
 '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.'''
 
 '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.'''

-nanopb_version = "nanopb-0.3.2-dev"
+nanopb_version = "nanopb-0.3.8-dev"

 
 import sys
 
 import sys

+import re
+from functools import reduce

 
 try:
     # Add some dummy imports to keep packaging tools happy.
 
 try:
     # Add some dummy imports to keep packaging tools happy.


@@ -70,27 +74,33 @@ intsizes = {
     nanopb_pb2.IS_64:    'int64_t',
 }
 
     nanopb_pb2.IS_64:    'int64_t',
 }
 

+# String types (for python 2 / python 3 compatibility)
+try:
+    strtypes = (unicode, str)
+except NameError:
+    strtypes = (str, )
+

 class Names:
     '''Keeps a set of nested names and formats them to C identifier.'''
     def __init__(self, parts = ()):
         if isinstance(parts, Names):
             parts = parts.parts
         self.parts = tuple(parts)
 class Names:
     '''Keeps a set of nested names and formats them to C identifier.'''
     def __init__(self, parts = ()):
         if isinstance(parts, Names):
             parts = parts.parts
         self.parts = tuple(parts)

-    
+

     def __str__(self):
         return '_'.join(self.parts)
 
     def __add__(self, other):
     def __str__(self):
         return '_'.join(self.parts)
 
     def __add__(self, other):

-        if isinstance(other, (str, unicode)):
+        if isinstance(other, strtypes):

             return Names(self.parts + (other,))
         elif isinstance(other, tuple):
             return Names(self.parts + other)
         else:
             raise ValueError("Name parts should be of type str")
             return Names(self.parts + (other,))
         elif isinstance(other, tuple):
             return Names(self.parts + other)
         else:
             raise ValueError("Name parts should be of type str")

-    
+

     def __eq__(self, other):
         return isinstance(other, Names) and self.parts == other.parts
     def __eq__(self, other):
         return isinstance(other, Names) and self.parts == other.parts

-    
+

 def names_from_type_name(type_name):
     '''Parse Names() from FieldDescriptorProto type_name'''
     if type_name[0] != '.':
 def names_from_type_name(type_name):
     '''Parse Names() from FieldDescriptorProto type_name'''
     if type_name[0] != '.':


@@ -99,11 +109,14 @@ def names_from_type_name(type_name):
 
 def varint_max_size(max_value):
     '''Returns the maximum number of bytes a varint can take when encoded.'''
 
 def varint_max_size(max_value):
     '''Returns the maximum number of bytes a varint can take when encoded.'''

+    if max_value < 0:
+        max_value = 2**64 - max_value

     for i in range(1, 11):
         if (max_value >> (i * 7)) == 0:
             return i
     raise ValueError("Value too large for varint: " + str(max_value))
 
     for i in range(1, 11):
         if (max_value >> (i * 7)) == 0:
             return i
     raise ValueError("Value too large for varint: " + str(max_value))
 

+assert varint_max_size(-1) == 10

 assert varint_max_size(0) == 1
 assert varint_max_size(127) == 1
 assert varint_max_size(128) == 2
 assert varint_max_size(0) == 1
 assert varint_max_size(127) == 1
 assert varint_max_size(128) == 2


@@ -112,16 +125,20 @@ class EncodedSize:
     '''Class used to represent the encoded size of a field or a message.
     Consists of a combination of symbolic sizes and integer sizes.'''
     def __init__(self, value = 0, symbols = []):
     '''Class used to represent the encoded size of a field or a message.
     Consists of a combination of symbolic sizes and integer sizes.'''
     def __init__(self, value = 0, symbols = []):

-        if isinstance(value, (str, Names)):
-            symbols = [str(value)]
-            value = 0
-        self.value = value
-        self.symbols = symbols
-    
+        if isinstance(value, EncodedSize):
+            self.value = value.value
+            self.symbols = value.symbols
+        elif isinstance(value, strtypes + (Names,)):
+            self.symbols = [str(value)]
+            self.value = 0
+        else:
+            self.value = value
+            self.symbols = symbols
+

     def __add__(self, other):
     def __add__(self, other):

-        if isinstance(other, (int, long)):
+        if isinstance(other, int):

             return EncodedSize(self.value + other, self.symbols)
             return EncodedSize(self.value + other, self.symbols)

-        elif isinstance(other, (str, Names)):
+        elif isinstance(other, strtypes + (Names,)):

             return EncodedSize(self.value, self.symbols + [str(other)])
         elif isinstance(other, EncodedSize):
             return EncodedSize(self.value + other.value, self.symbols + other.symbols)
             return EncodedSize(self.value, self.symbols + [str(other)])
         elif isinstance(other, EncodedSize):
             return EncodedSize(self.value + other.value, self.symbols + other.symbols)


@@ -129,7 +146,7 @@ class EncodedSize:
             raise ValueError("Cannot add size: " + repr(other))
 
     def __mul__(self, other):
             raise ValueError("Cannot add size: " + repr(other))
 
     def __mul__(self, other):

-        if isinstance(other, (int, long)):
+        if isinstance(other, int):

             return EncodedSize(self.value * other, [str(other) + '*' + s for s in self.symbols])
         else:
             raise ValueError("Cannot multiply size: " + repr(other))
             return EncodedSize(self.value * other, [str(other) + '*' + s for s in self.symbols])
         else:
             raise ValueError("Cannot multiply size: " + repr(other))


@@ -149,23 +166,88 @@ class EncodedSize:
 class Enum:
     def __init__(self, names, desc, enum_options):
         '''desc is EnumDescriptorProto'''
 class Enum:
     def __init__(self, names, desc, enum_options):
         '''desc is EnumDescriptorProto'''

-        
+

         self.options = enum_options
         self.names = names + desc.name
         self.options = enum_options
         self.names = names + desc.name

-        
+

         if enum_options.long_names:
         if enum_options.long_names:

-            self.values = [(self.names + x.name, x.number) for x in desc.value]            
+            self.values = [(self.names + x.name, x.number) for x in desc.value]

         else:
         else:

-            self.values = [(names + x.name, x.number) for x in desc.value] 
-        
+            self.values = [(names + x.name, x.number) for x in desc.value]
+

         self.value_longnames = [self.names + x.name for x in desc.value]
         self.value_longnames = [self.names + x.name for x in desc.value]

-    
+        self.packed = enum_options.packed_enum
+
+    def has_negative(self):
+        for n, v in self.values:
+            if v < 0:
+                return True
+        return False
+
+    def encoded_size(self):
+        return max([varint_max_size(v) for n,v in self.values])
+

     def __str__(self):
         result = 'typedef enum _%s {\n' % self.names
         result += ',\n'.join(["    %s = %d" % x for x in self.values])
     def __str__(self):
         result = 'typedef enum _%s {\n' % self.names
         result += ',\n'.join(["    %s = %d" % x for x in self.values])

-        result += '\n} %s;' % self.names
+        result += '\n}'
+
+        if self.packed:
+            result += ' pb_packed'
+
+        result += ' %s;' % self.names
+
+        result += '\n#define _%s_MIN %s' % (self.names, self.values[0][0])
+        result += '\n#define _%s_MAX %s' % (self.names, self.values[-1][0])
+        result += '\n#define _%s_ARRAYSIZE ((%s)(%s+1))' % (self.names, self.names, self.values[-1][0])
+
+        if not self.options.long_names:
+            # Define the long names always so that enum value references
+            # from other files work properly.
+            for i, x in enumerate(self.values):
+                result += '\n#define %s %s' % (self.value_longnames[i], x[0])
+
+        if self.options.enum_to_string:
+            result += '\nconst char *%s_name(%s v);\n' % (self.names, self.names)
+

         return result
 
         return result
 

+    def enum_to_string_definition(self):
+        if not self.options.enum_to_string:
+            return ""
+
+        result = 'const char *%s_name(%s v) {\n' % (self.names, self.names)
+        result += '    switch (v) {\n'
+
+        for ((enumname, _), strname) in zip(self.values, self.value_longnames):
+            # Strip off the leading type name from the string value.
+            strval = str(strname)[len(str(self.names)) + 1:]
+            result += '        case %s: return "%s";\n' % (enumname, strval)
+
+        result += '    }\n'
+        result += '    return "unknown";\n'
+        result += '}\n'
+
+        return result
+
+class FieldMaxSize:
+    def __init__(self, worst = 0, checks = [], field_name = 'undefined'):
+        if isinstance(worst, list):
+            self.worst = max(i for i in worst if i is not None)
+        else:
+            self.worst = worst
+
+        self.worst_field = field_name
+        self.checks = list(checks)
+
+    def extend(self, extend, field_name = None):
+        self.worst = max(self.worst, extend.worst)
+
+        if self.worst == extend.worst:
+            self.worst_field = extend.worst_field
+
+        self.checks.extend(extend.checks)
+

 class Field:
     def __init__(self, struct_name, desc, field_options):
         '''desc is FieldDescriptorProto'''
 class Field:
     def __init__(self, struct_name, desc, field_options):
         '''desc is FieldDescriptorProto'''


@@ -179,51 +261,64 @@ class Field:
         self.array_decl = ""
         self.enc_size = None
         self.ctype = None
         self.array_decl = ""
         self.enc_size = None
         self.ctype = None

-        
+
+        if field_options.type == nanopb_pb2.FT_INLINE:
+            # Before nanopb-0.3.8, fixed length bytes arrays were specified
+            # by setting type to FT_INLINE. But to handle pointer typed fields,
+            # it makes sense to have it as a separate option.
+            field_options.type = nanopb_pb2.FT_STATIC
+            field_options.fixed_length = True
+

         # Parse field options
         if field_options.HasField("max_size"):
             self.max_size = field_options.max_size
         
         # Parse field options
         if field_options.HasField("max_size"):
             self.max_size = field_options.max_size
         

+        if desc.type == FieldD.TYPE_STRING and field_options.HasField("max_length"):
+            # max_length overrides max_size for strings
+            self.max_size = field_options.max_length + 1
+

         if field_options.HasField("max_count"):
             self.max_count = field_options.max_count
         if field_options.HasField("max_count"):
             self.max_count = field_options.max_count

-        
+

         if desc.HasField('default_value'):
             self.default = desc.default_value
         if desc.HasField('default_value'):
             self.default = desc.default_value

-           
+

         # Check field rules, i.e. required/optional/repeated.
         can_be_static = True
         # Check field rules, i.e. required/optional/repeated.
         can_be_static = True

-        if desc.label == FieldD.LABEL_REQUIRED:
-            self.rules = 'REQUIRED'
-        elif desc.label == FieldD.LABEL_OPTIONAL:
-            self.rules = 'OPTIONAL'
-        elif desc.label == FieldD.LABEL_REPEATED:
+        if desc.label == FieldD.LABEL_REPEATED:

             self.rules = 'REPEATED'
             if self.max_count is None:
                 can_be_static = False
             else:
                 self.array_decl = '[%d]' % self.max_count
             self.rules = 'REPEATED'
             if self.max_count is None:
                 can_be_static = False
             else:
                 self.array_decl = '[%d]' % self.max_count

+        elif field_options.proto3:
+            self.rules = 'SINGULAR'
+        elif desc.label == FieldD.LABEL_REQUIRED:
+            self.rules = 'REQUIRED'
+        elif desc.label == FieldD.LABEL_OPTIONAL:
+            self.rules = 'OPTIONAL'

         else:
             raise NotImplementedError(desc.label)
         else:
             raise NotImplementedError(desc.label)

-        
+

         # Check if the field can be implemented with static allocation
         # i.e. whether the data size is known.
         if desc.type == FieldD.TYPE_STRING and self.max_size is None:
             can_be_static = False
         # Check if the field can be implemented with static allocation
         # i.e. whether the data size is known.
         if desc.type == FieldD.TYPE_STRING and self.max_size is None:
             can_be_static = False

-        
+

         if desc.type == FieldD.TYPE_BYTES and self.max_size is None:
             can_be_static = False
         if desc.type == FieldD.TYPE_BYTES and self.max_size is None:
             can_be_static = False

-        
+

         # Decide how the field data will be allocated
         if field_options.type == nanopb_pb2.FT_DEFAULT:
             if can_be_static:
                 field_options.type = nanopb_pb2.FT_STATIC
             else:
                 field_options.type = nanopb_pb2.FT_CALLBACK
         # Decide how the field data will be allocated
         if field_options.type == nanopb_pb2.FT_DEFAULT:
             if can_be_static:
                 field_options.type = nanopb_pb2.FT_STATIC
             else:
                 field_options.type = nanopb_pb2.FT_CALLBACK

-        
+

         if field_options.type == nanopb_pb2.FT_STATIC and not can_be_static:
             raise Exception("Field %s is defined as static, but max_size or "
                             "max_count is not given." % self.name)
         if field_options.type == nanopb_pb2.FT_STATIC and not can_be_static:
             raise Exception("Field %s is defined as static, but max_size or "
                             "max_count is not given." % self.name)

-        
+

         if field_options.type == nanopb_pb2.FT_STATIC:
             self.allocation = 'STATIC'
         elif field_options.type == nanopb_pb2.FT_POINTER:
         if field_options.type == nanopb_pb2.FT_STATIC:
             self.allocation = 'STATIC'
         elif field_options.type == nanopb_pb2.FT_POINTER:


@@ -232,9 +327,9 @@ class Field:
             self.allocation = 'CALLBACK'
         else:
             raise NotImplementedError(field_options.type)
             self.allocation = 'CALLBACK'
         else:
             raise NotImplementedError(field_options.type)

-        
+

         # Decide the C data type to use in the struct.
         # Decide the C data type to use in the struct.

-        if datatypes.has_key(desc.type):
+        if desc.type in datatypes:

             self.ctype, self.pbtype, self.enc_size, isa = datatypes[desc.type]
 
             # Override the field size if user wants to use smaller integers
             self.ctype, self.pbtype, self.enc_size, isa = datatypes[desc.type]
 
             # Override the field size if user wants to use smaller integers


@@ -247,7 +342,7 @@ class Field:
             self.ctype = names_from_type_name(desc.type_name)
             if self.default is not None:
                 self.default = self.ctype + self.default
             self.ctype = names_from_type_name(desc.type_name)
             if self.default is not None:
                 self.default = self.ctype + self.default

-            self.enc_size = 5 # protoc rejects enum values > 32 bits
+            self.enc_size = None # Needs to be filled in when enum values are known

         elif desc.type == FieldD.TYPE_STRING:
             self.pbtype = 'STRING'
             self.ctype = 'char'
         elif desc.type == FieldD.TYPE_STRING:
             self.pbtype = 'STRING'
             self.ctype = 'char'


@@ -256,31 +351,39 @@ class Field:
                 self.array_decl += '[%d]' % self.max_size
                 self.enc_size = varint_max_size(self.max_size) + self.max_size
         elif desc.type == FieldD.TYPE_BYTES:
                 self.array_decl += '[%d]' % self.max_size
                 self.enc_size = varint_max_size(self.max_size) + self.max_size
         elif desc.type == FieldD.TYPE_BYTES:

-            self.pbtype = 'BYTES'
-            if self.allocation == 'STATIC':
-                self.ctype = self.struct_name + self.name + 't'
+            if field_options.fixed_length:
+                self.pbtype = 'FIXED_LENGTH_BYTES'

                 self.enc_size = varint_max_size(self.max_size) + self.max_size
                 self.enc_size = varint_max_size(self.max_size) + self.max_size

-            elif self.allocation == 'POINTER':
+                self.ctype = 'pb_byte_t'
+                self.array_decl += '[%d]' % self.max_size
+            else:
+                self.pbtype = 'BYTES'

                 self.ctype = 'pb_bytes_array_t'
                 self.ctype = 'pb_bytes_array_t'

+                if self.allocation == 'STATIC':
+                    self.ctype = self.struct_name + self.name + 't'
+                    self.enc_size = varint_max_size(self.max_size) + self.max_size

         elif desc.type == FieldD.TYPE_MESSAGE:
             self.pbtype = 'MESSAGE'
             self.ctype = self.submsgname = names_from_type_name(desc.type_name)
             self.enc_size = None # Needs to be filled in after the message type is available
         else:
             raise NotImplementedError(desc.type)
         elif desc.type == FieldD.TYPE_MESSAGE:
             self.pbtype = 'MESSAGE'
             self.ctype = self.submsgname = names_from_type_name(desc.type_name)
             self.enc_size = None # Needs to be filled in after the message type is available
         else:
             raise NotImplementedError(desc.type)

-        
-    def __cmp__(self, other):
-        return cmp(self.tag, other.tag)
-    
+
+    def __lt__(self, other):
+        return self.tag < other.tag
+

     def __str__(self):
         result = ''
         if self.allocation == 'POINTER':
             if self.rules == 'REPEATED':
                 result += '    pb_size_t ' + self.name + '_count;\n'
     def __str__(self):
         result = ''
         if self.allocation == 'POINTER':
             if self.rules == 'REPEATED':
                 result += '    pb_size_t ' + self.name + '_count;\n'

-            
+

             if self.pbtype == 'MESSAGE':
                 # Use struct definition, so recursive submessages are possible
                 result += '    struct _%s *%s;' % (self.ctype, self.name)
             if self.pbtype == 'MESSAGE':
                 # Use struct definition, so recursive submessages are possible
                 result += '    struct _%s *%s;' % (self.ctype, self.name)

+            elif self.pbtype == 'FIXED_LENGTH_BYTES':
+                # Pointer to fixed size array
+                result += '    %s (*%s)%s;' % (self.ctype, self.name, self.array_decl)

             elif self.rules == 'REPEATED' and self.pbtype in ['STRING', 'BYTES']:
                 # String/bytes arrays need to be defined as pointers to pointers
                 result += '    %s **%s;' % (self.ctype, self.name)
             elif self.rules == 'REPEATED' and self.pbtype in ['STRING', 'BYTES']:
                 # String/bytes arrays need to be defined as pointers to pointers
                 result += '    %s **%s;' % (self.ctype, self.name)


@@ -295,7 +398,7 @@ class Field:
                 result += '    pb_size_t ' + self.name + '_count;\n'
             result += '    %s %s%s;' % (self.ctype, self.name, self.array_decl)
         return result
                 result += '    pb_size_t ' + self.name + '_count;\n'
             result += '    %s %s%s;' % (self.ctype, self.name, self.array_decl)
         return result

-    
+

     def types(self):
         '''Return definitions for any special types this field might need.'''
         if self.pbtype == 'BYTES' and self.allocation == 'STATIC':
     def types(self):
         '''Return definitions for any special types this field might need.'''
         if self.pbtype == 'BYTES' and self.allocation == 'STATIC':


@@ -303,7 +406,7 @@ class Field:
         else:
             result = ''
         return result
         else:
             result = ''
         return result

-    
+

     def get_dependencies(self):
         '''Get list of type names used by this field.'''
         if self.allocation == 'STATIC':
     def get_dependencies(self):
         '''Get list of type names used by this field.'''
         if self.allocation == 'STATIC':


@@ -328,22 +431,28 @@ class Field:
                 inner_init = '""'
             elif self.pbtype == 'BYTES':
                 inner_init = '{0, {0}}'
                 inner_init = '""'
             elif self.pbtype == 'BYTES':
                 inner_init = '{0, {0}}'

-            elif self.pbtype == 'ENUM':
+            elif self.pbtype == 'FIXED_LENGTH_BYTES':
+                inner_init = '{0}'
+            elif self.pbtype in ('ENUM', 'UENUM'):

                 inner_init = '(%s)0' % self.ctype
             else:
                 inner_init = '0'
         else:
             if self.pbtype == 'STRING':
                 inner_init = '(%s)0' % self.ctype
             else:
                 inner_init = '0'
         else:
             if self.pbtype == 'STRING':

-                inner_init = self.default.encode('utf-8').encode('string_escape')
-                inner_init = inner_init.replace('"', '\\"')
+                inner_init = self.default.replace('"', '\\"')

                 inner_init = '"' + inner_init + '"'
             elif self.pbtype == 'BYTES':
                 inner_init = '"' + inner_init + '"'
             elif self.pbtype == 'BYTES':

-                data = str(self.default).decode('string_escape')
-                data = ['0x%02x' % ord(c) for c in data]
+                data = ['0x%02x' % ord(c) for c in self.default]

                 if len(data) == 0:
                     inner_init = '{0, {0}}'
                 else:
                     inner_init = '{%d, {%s}}' % (len(data), ','.join(data))
                 if len(data) == 0:
                     inner_init = '{0, {0}}'
                 else:
                     inner_init = '{%d, {%s}}' % (len(data), ','.join(data))

+            elif self.pbtype == 'FIXED_LENGTH_BYTES':
+                data = ['0x%02x' % ord(c) for c in self.default]
+                if len(data) == 0:
+                    inner_init = '{0}'
+                else:
+                    inner_init = '{%s}' % ','.join(data)

             elif self.pbtype in ['FIXED32', 'UINT32']:
                 inner_init = str(self.default) + 'u'
             elif self.pbtype in ['FIXED64', 'UINT64']:
             elif self.pbtype in ['FIXED32', 'UINT32']:
                 inner_init = str(self.default) + 'u'
             elif self.pbtype in ['FIXED64', 'UINT64']:


@@ -352,7 +461,7 @@ class Field:
                 inner_init = str(self.default) + 'll'
             else:
                 inner_init = str(self.default)
                 inner_init = str(self.default) + 'll'
             else:
                 inner_init = str(self.default)

-        
+

         if inner_init_only:
             return inner_init
 
         if inner_init_only:
             return inner_init
 


@@ -387,7 +496,7 @@ class Field:
         ctype = self.ctype
         default = self.get_initializer(False, True)
         array_decl = ''
         ctype = self.ctype
         default = self.get_initializer(False, True)
         array_decl = ''

-        
+

         if self.pbtype == 'STRING':
             if self.allocation != 'STATIC':
                 return None # Not implemented
         if self.pbtype == 'STRING':
             if self.allocation != 'STATIC':
                 return None # Not implemented


@@ -395,24 +504,32 @@ class Field:
         elif self.pbtype == 'BYTES':
             if self.allocation != 'STATIC':
                 return None # Not implemented
         elif self.pbtype == 'BYTES':
             if self.allocation != 'STATIC':
                 return None # Not implemented

-        
+        elif self.pbtype == 'FIXED_LENGTH_BYTES':
+            if self.allocation != 'STATIC':
+                return None # Not implemented
+            array_decl = '[%d]' % self.max_size
+

         if declaration_only:
             return 'extern const %s %s_default%s;' % (ctype, self.struct_name + self.name, array_decl)
         else:
             return 'const %s %s_default%s = %s;' % (ctype, self.struct_name + self.name, array_decl, default)
         if declaration_only:
             return 'extern const %s %s_default%s;' % (ctype, self.struct_name + self.name, array_decl)
         else:
             return 'const %s %s_default%s = %s;' % (ctype, self.struct_name + self.name, array_decl, default)

-    
+

     def tags(self):
         '''Return the #define for the tag number of this field.'''
         identifier = '%s_%s_tag' % (self.struct_name, self.name)
         return '#define %-40s %d\n' % (identifier, self.tag)
     def tags(self):
         '''Return the #define for the tag number of this field.'''
         identifier = '%s_%s_tag' % (self.struct_name, self.name)
         return '#define %-40s %d\n' % (identifier, self.tag)

-    
-    def pb_field_t(self, prev_field_name):
+
+    def pb_field_t(self, prev_field_name, union_index = None):

         '''Return the pb_field_t initializer to use in the constant array.
         '''Return the pb_field_t initializer to use in the constant array.

-        prev_field_name is the name of the previous field or None.
+        prev_field_name is the name of the previous field or None. For OneOf
+        unions, union_index is the index of this field inside the OneOf.

         '''
 
         if self.rules == 'ONEOF':
         '''
 
         if self.rules == 'ONEOF':

-            result = '    PB_ONEOF_FIELD(%s, ' % self.union_name
+            if self.anonymous:
+                result = '    PB_ANONYMOUS_ONEOF_FIELD(%s, ' % self.union_name
+            else:
+                result = '    PB_ONEOF_FIELD(%s, ' % self.union_name

         else:
             result = '    PB_FIELD('
 
         else:
             result = '    PB_FIELD('
 


@@ -420,58 +537,77 @@ class Field:
         result += '%-8s, ' % self.pbtype
         result += '%s, ' % self.rules
         result += '%-8s, ' % self.allocation
         result += '%-8s, ' % self.pbtype
         result += '%s, ' % self.rules
         result += '%-8s, ' % self.allocation

-        result += '%s, ' % ("FIRST" if not prev_field_name else "OTHER")
+        
+        if union_index is not None and union_index > 0:
+            result += 'UNION, '
+        elif prev_field_name is None:
+            result += 'FIRST, '
+        else:
+            result += 'OTHER, '
+        

         result += '%s, ' % self.struct_name
         result += '%s, ' % self.name
         result += '%s, ' % (prev_field_name or self.name)
         result += '%s, ' % self.struct_name
         result += '%s, ' % self.name
         result += '%s, ' % (prev_field_name or self.name)

-        
+

         if self.pbtype == 'MESSAGE':
             result += '&%s_fields)' % self.submsgname
         elif self.default is None:
             result += '0)'
         if self.pbtype == 'MESSAGE':
             result += '&%s_fields)' % self.submsgname
         elif self.default is None:
             result += '0)'

-        elif self.pbtype in ['BYTES', 'STRING'] and self.allocation != 'STATIC':
+        elif self.pbtype in ['BYTES', 'STRING', 'FIXED_LENGTH_BYTES'] and self.allocation != 'STATIC':

             result += '0)' # Arbitrary size default values not implemented
         elif self.rules == 'OPTEXT':
             result += '0)' # Default value for extensions is not implemented
         else:
             result += '&%s_default)' % (self.struct_name + self.name)
             result += '0)' # Arbitrary size default values not implemented
         elif self.rules == 'OPTEXT':
             result += '0)' # Default value for extensions is not implemented
         else:
             result += '&%s_default)' % (self.struct_name + self.name)

-        
+

         return result
         return result

-    
+
+    def get_last_field_name(self):
+        return self.name
+

     def largest_field_value(self):
         '''Determine if this field needs 16bit or 32bit pb_field_t structure to compile properly.
         Returns numeric value or a C-expression for assert.'''
     def largest_field_value(self):
         '''Determine if this field needs 16bit or 32bit pb_field_t structure to compile properly.
         Returns numeric value or a C-expression for assert.'''

-        if self.pbtype == 'MESSAGE':
-            if self.rules == 'REPEATED' and self.allocation == 'STATIC':
-                return 'pb_membersize(%s, %s[0])' % (self.struct_name, self.name)
+        check = []
+        if self.pbtype == 'MESSAGE' and self.allocation == 'STATIC':
+            if self.rules == 'REPEATED':
+                check.append('pb_membersize(%s, %s[0])' % (self.struct_name, self.name))

             elif self.rules == 'ONEOF':
             elif self.rules == 'ONEOF':

-                return 'pb_membersize(%s, %s.%s)' % (self.struct_name, self.union_name, self.name)
+                if self.anonymous:
+                    check.append('pb_membersize(%s, %s)' % (self.struct_name, self.name))
+                else:
+                    check.append('pb_membersize(%s, %s.%s)' % (self.struct_name, self.union_name, self.name))

             else:
             else:

-                return 'pb_membersize(%s, %s)' % (self.struct_name, self.name)
+                check.append('pb_membersize(%s, %s)' % (self.struct_name, self.name))
+        elif self.pbtype == 'BYTES' and self.allocation == 'STATIC':
+            if self.max_size > 251:
+                check.append('pb_membersize(%s, %s)' % (self.struct_name, self.name))

 
 

-        return max(self.tag, self.max_size, self.max_count)        
+        return FieldMaxSize([self.tag, self.max_size, self.max_count],
+                            check,
+                            ('%s.%s' % (self.struct_name, self.name)))

 
 

-    def encoded_size(self, allmsgs):
+    def encoded_size(self, dependencies):

         '''Return the maximum size that this field can take when encoded,
         including the field tag. If the size cannot be determined, returns
         None.'''
         '''Return the maximum size that this field can take when encoded,
         including the field tag. If the size cannot be determined, returns
         None.'''

-        
+

         if self.allocation != 'STATIC':
             return None
         if self.allocation != 'STATIC':
             return None

-        
+

         if self.pbtype == 'MESSAGE':
         if self.pbtype == 'MESSAGE':

-            for msg in allmsgs:
-                if msg.name == self.submsgname:
-                    encsize = msg.encoded_size(allmsgs)
-                    if encsize is None:
-                        return None # Submessage size is indeterminate
-                        
+            encsize = None
+            if str(self.submsgname) in dependencies:
+                submsg = dependencies[str(self.submsgname)]
+                encsize = submsg.encoded_size(dependencies)
+                if encsize is not None:

                     # Include submessage length prefix
                     encsize += varint_max_size(encsize.upperlimit())
                     # Include submessage length prefix
                     encsize += varint_max_size(encsize.upperlimit())

-                    break
-            else:
-                # Submessage cannot be found, this currently occurs when
-                # the submessage type is defined in a different file.
+
+            if encsize is None:
+                # Submessage or its size cannot be found.
+                # This can occur if submessage is defined in different
+                # file, and it or its .options could not be found.

                 # Instead of direct numeric value, reference the size that
                 # has been #defined in the other file.
                 encsize = EncodedSize(self.submsgname + 'size')
                 # Instead of direct numeric value, reference the size that
                 # has been #defined in the other file.
                 encsize = EncodedSize(self.submsgname + 'size')


@@ -480,12 +616,20 @@ class Field:
                 # prefix size, though.
                 encsize += 5
 
                 # prefix size, though.
                 encsize += 5
 

+        elif self.pbtype in ['ENUM', 'UENUM']:
+            if str(self.ctype) in dependencies:
+                enumtype = dependencies[str(self.ctype)]
+                encsize = enumtype.encoded_size()
+            else:
+                # Conservative assumption
+                encsize = 10
+

         elif self.enc_size is None:
             raise RuntimeError("Could not determine encoded size for %s.%s"
                                % (self.struct_name, self.name))
         else:
             encsize = EncodedSize(self.enc_size)
         elif self.enc_size is None:
             raise RuntimeError("Could not determine encoded size for %s.%s"
                                % (self.struct_name, self.name))
         else:
             encsize = EncodedSize(self.enc_size)

-        
+

         encsize += varint_max_size(self.tag << 3) # Tag + wire type
 
         if self.rules == 'REPEATED':
         encsize += varint_max_size(self.tag << 3) # Tag + wire type
 
         if self.rules == 'REPEATED':


@@ -493,7 +637,7 @@ class Field:
             # Therefore we have to reserve space for it, even though
             # we emit packed arrays ourselves.
             encsize *= self.max_count
             # Therefore we have to reserve space for it, even though
             # we emit packed arrays ourselves.
             encsize *= self.max_count

-        
+

         return encsize
 
 
         return encsize
 
 


@@ -515,17 +659,17 @@ class ExtensionRange(Field):
         self.default = None
         self.max_size = 0
         self.max_count = 0
         self.default = None
         self.max_size = 0
         self.max_count = 0

-        
+

     def __str__(self):
         return '    pb_extension_t *extensions;'
     def __str__(self):
         return '    pb_extension_t *extensions;'

-    
+

     def types(self):
         return ''
     def types(self):
         return ''

-    
+

     def tags(self):
         return ''
     def tags(self):
         return ''

-    
-    def encoded_size(self, allmsgs):
+
+    def encoded_size(self, dependencies):

         # We exclude extensions from the count, because they cannot be known
         # until runtime. Other option would be to return None here, but this
         # way the value remains useful if extensions are not used.
         # We exclude extensions from the count, because they cannot be known
         # until runtime. Other option would be to return None here, but this
         # way the value remains useful if extensions are not used.


@@ -536,7 +680,7 @@ class ExtensionField(Field):
         self.fullname = struct_name + desc.name
         self.extendee_name = names_from_type_name(desc.extendee)
         Field.__init__(self, self.fullname + 'struct', desc, field_options)
         self.fullname = struct_name + desc.name
         self.extendee_name = names_from_type_name(desc.extendee)
         Field.__init__(self, self.fullname + 'struct', desc, field_options)

-        
+

         if self.rules != 'OPTIONAL':
             self.skip = True
         else:
         if self.rules != 'OPTIONAL':
             self.skip = True
         else:


@@ -554,7 +698,7 @@ class ExtensionField(Field):
             msg = '/* Extension field %s was skipped because only "optional"\n' % self.fullname
             msg +='   type of extension fields is currently supported. */\n'
             return msg
             msg = '/* Extension field %s was skipped because only "optional"\n' % self.fullname
             msg +='   type of extension fields is currently supported. */\n'
             return msg

-        
+

         return ('extern const pb_extension_type_t %s; /* field type: %s */\n' %
             (self.fullname, str(self).strip()))
 
         return ('extern const pb_extension_type_t %s; /* field type: %s */\n' %
             (self.fullname, str(self).strip()))
 


@@ -586,7 +730,12 @@ class OneOf(Field):
         self.struct_name = struct_name
         self.name = oneof_desc.name
         self.ctype = 'union'
         self.struct_name = struct_name
         self.name = oneof_desc.name
         self.ctype = 'union'

+        self.pbtype = 'oneof'

         self.fields = []
         self.fields = []

+        self.allocation = 'ONEOF'
+        self.default = None
+        self.rules = 'ONEOF'
+        self.anonymous = False

 
     def add_field(self, field):
         if field.allocation == 'CALLBACK':
 
     def add_field(self, field):
         if field.allocation == 'CALLBACK':


@@ -595,15 +744,13 @@ class OneOf(Field):
 
         field.union_name = self.name
         field.rules = 'ONEOF'
 
         field.union_name = self.name
         field.rules = 'ONEOF'

+        field.anonymous = self.anonymous

         self.fields.append(field)
         self.fields.sort(key = lambda f: f.tag)
 
         # Sort by the lowest tag number inside union
         self.tag = min([f.tag for f in self.fields])
 
         self.fields.append(field)
         self.fields.sort(key = lambda f: f.tag)
 
         # Sort by the lowest tag number inside union
         self.tag = min([f.tag for f in self.fields])
 

-    def __cmp__(self, other):
-        return cmp(self.tag, other.tag)
-

     def __str__(self):
         result = ''
         if self.fields:
     def __str__(self):
         result = ''
         if self.fields:


@@ -611,7 +758,10 @@ class OneOf(Field):
             result += '    union {\n'
             for f in self.fields:
                 result += '    ' + str(f).replace('\n', '\n    ') + '\n'
             result += '    union {\n'
             for f in self.fields:
                 result += '    ' + str(f).replace('\n', '\n    ') + '\n'

-            result += '    } ' + self.name + ';'
+            if self.anonymous:
+                result += '    };'
+            else:
+                result += '    } ' + self.name + ';'

         return result
 
     def types(self):
         return result
 
     def types(self):


@@ -630,18 +780,39 @@ class OneOf(Field):
         return None
 
     def tags(self):
         return None
 
     def tags(self):

-        return '\n'.join([f.tags() for f in self.fields])
+        return ''.join([f.tags() for f in self.fields])

 
     def pb_field_t(self, prev_field_name):
 
     def pb_field_t(self, prev_field_name):

-        prev_field_name = prev_field_name or self.name
-        result = ',\n'.join([f.pb_field_t(prev_field_name) for f in self.fields])
-        return result
+        parts = []
+        for union_index, field in enumerate(self.fields):
+            parts.append(field.pb_field_t(prev_field_name, union_index))
+        return ',\n'.join(parts)
+
+    def get_last_field_name(self):
+        if self.anonymous:
+            return self.fields[-1].name
+        else:
+            return self.name + '.' + self.fields[-1].name

 
     def largest_field_value(self):
 
     def largest_field_value(self):

-        return max([f.largest_field_value() for f in self.fields])
+        largest = FieldMaxSize()
+        for f in self.fields:
+            largest.extend(f.largest_field_value())
+        return largest

 
 

-    def encoded_size(self, allmsgs):
-        return max([f.encoded_size(allmsgs) for f in self.fields])
+    def encoded_size(self, dependencies):
+        '''Returns the size of the largest oneof field.'''
+        largest = EncodedSize(0)
+        for f in self.fields:
+            size = EncodedSize(f.encoded_size(dependencies))
+            if size.value is None:
+                return None
+            elif size.symbols:
+                return None # Cannot resolve maximum of symbols
+            elif size.value > largest.value:
+                largest = size
+
+        return largest

 
 # ---------------------------------------------------------------------------
 #                   Generation of messages (structures)
 
 # ---------------------------------------------------------------------------
 #                   Generation of messages (structures)


@@ -655,6 +826,9 @@ class Message:
         self.oneofs = {}
         no_unions = []
 
         self.oneofs = {}
         no_unions = []
 

+        if message_options.msgid:
+            self.msgid = message_options.msgid
+

         if hasattr(desc, 'oneof_decl'):
             for i, f in enumerate(desc.oneof_decl):
                 oneof_options = get_nanopb_suboptions(desc, message_options, self.name + f.name)
         if hasattr(desc, 'oneof_decl'):
             for i, f in enumerate(desc.oneof_decl):
                 oneof_options = get_nanopb_suboptions(desc, message_options, self.name + f.name)


@@ -664,6 +838,8 @@ class Message:
                     pass # No union and skip fields also
                 else:
                     oneof = OneOf(self.name, f)
                     pass # No union and skip fields also
                 else:
                     oneof = OneOf(self.name, f)

+                    if oneof_options.anonymous_oneof:
+                        oneof.anonymous = True

                     self.oneofs[i] = oneof
                     self.fields.append(oneof)
 
                     self.oneofs[i] = oneof
                     self.fields.append(oneof)
 


@@ -680,13 +856,13 @@ class Message:
                     self.oneofs[f.oneof_index].add_field(field)
             else:
                 self.fields.append(field)
                     self.oneofs[f.oneof_index].add_field(field)
             else:
                 self.fields.append(field)

-        
+

         if len(desc.extension_range) > 0:
             field_options = get_nanopb_suboptions(desc, message_options, self.name + 'extensions')
             range_start = min([r.start for r in desc.extension_range])
             if field_options.type != nanopb_pb2.FT_IGNORE:
                 self.fields.append(ExtensionRange(self.name, range_start, field_options))
         if len(desc.extension_range) > 0:
             field_options = get_nanopb_suboptions(desc, message_options, self.name + 'extensions')
             range_start = min([r.start for r in desc.extension_range])
             if field_options.type != nanopb_pb2.FT_IGNORE:
                 self.fields.append(ExtensionRange(self.name, range_start, field_options))

-        
+

         self.packed = message_options.packed_struct
         self.ordered_fields = self.fields[:]
         self.ordered_fields.sort()
         self.packed = message_options.packed_struct
         self.ordered_fields = self.fields[:]
         self.ordered_fields.sort()


@@ -697,41 +873,42 @@ class Message:
         for f in self.fields:
             deps += f.get_dependencies()
         return deps
         for f in self.fields:
             deps += f.get_dependencies()
         return deps

-    
+

     def __str__(self):
         result = 'typedef struct _%s {\n' % self.name
 
         if not self.ordered_fields:
             # Empty structs are not allowed in C standard.
             # Therefore add a dummy field if an empty message occurs.
     def __str__(self):
         result = 'typedef struct _%s {\n' % self.name
 
         if not self.ordered_fields:
             # Empty structs are not allowed in C standard.
             # Therefore add a dummy field if an empty message occurs.

-            result += '    uint8_t dummy_field;'
+            result += '    char dummy_field;'

 
         result += '\n'.join([str(f) for f in self.ordered_fields])
 
         result += '\n'.join([str(f) for f in self.ordered_fields])

+        result += '\n/* @@protoc_insertion_point(struct:%s) */' % self.name

         result += '\n}'
         result += '\n}'

-        
+

         if self.packed:
             result += ' pb_packed'
         if self.packed:
             result += ' pb_packed'

-        
+

         result += ' %s;' % self.name
         result += ' %s;' % self.name

-        
+

         if self.packed:
             result = 'PB_PACKED_STRUCT_START\n' + result
             result += '\nPB_PACKED_STRUCT_END'
         if self.packed:
             result = 'PB_PACKED_STRUCT_START\n' + result
             result += '\nPB_PACKED_STRUCT_END'

-        
+

         return result
         return result

-    
+

     def types(self):
         return ''.join([f.types() for f in self.fields])
 
     def get_initializer(self, null_init):
         if not self.ordered_fields:
             return '{0}'
     def types(self):
         return ''.join([f.types() for f in self.fields])
 
     def get_initializer(self, null_init):
         if not self.ordered_fields:
             return '{0}'

-    
+

         parts = []
         for field in self.ordered_fields:
             parts.append(field.get_initializer(null_init))
         return '{' + ', '.join(parts) + '}'
         parts = []
         for field in self.ordered_fields:
             parts.append(field.get_initializer(null_init))
         return '{' + ', '.join(parts) + '}'

-    
+

     def default_decl(self, declaration_only = False):
         result = ""
         for field in self.fields:
     def default_decl(self, declaration_only = False):
         result = ""
         for field in self.fields:


@@ -764,30 +941,27 @@ class Message:
 
     def fields_definition(self):
         result = 'const pb_field_t %s_fields[%d] = {\n' % (self.name, self.count_all_fields() + 1)
 
     def fields_definition(self):
         result = 'const pb_field_t %s_fields[%d] = {\n' % (self.name, self.count_all_fields() + 1)

-        
+

         prev = None
         for field in self.ordered_fields:
             result += field.pb_field_t(prev)
             result += ',\n'
         prev = None
         for field in self.ordered_fields:
             result += field.pb_field_t(prev)
             result += ',\n'

-            if isinstance(field, OneOf):
-                prev = field.name + '.' + field.fields[-1].name
-            else:
-                prev = field.name
-        
+            prev = field.get_last_field_name()
+

         result += '    PB_LAST_FIELD\n};'
         return result
 
         result += '    PB_LAST_FIELD\n};'
         return result
 

-    def encoded_size(self, allmsgs):
+    def encoded_size(self, dependencies):

         '''Return the maximum size that this message can take when encoded.
         If the size cannot be determined, returns None.
         '''
         size = EncodedSize(0)
         for field in self.fields:
         '''Return the maximum size that this message can take when encoded.
         If the size cannot be determined, returns None.
         '''
         size = EncodedSize(0)
         for field in self.fields:

-            fsize = field.encoded_size(allmsgs)
+            fsize = field.encoded_size(dependencies)

             if fsize is None:
                 return None
             size += fsize
             if fsize is None:
                 return None
             size += fsize

-        
+

         return size
 
 
         return size
 
 


@@ -795,18 +969,17 @@ class Message:
 #                    Processing of entire .proto files
 # ---------------------------------------------------------------------------
 
 #                    Processing of entire .proto files
 # ---------------------------------------------------------------------------
 

-

 def iterate_messages(desc, names = Names()):
     '''Recursively find all messages. For each, yield name, DescriptorProto.'''
     if hasattr(desc, 'message_type'):
         submsgs = desc.message_type
     else:
         submsgs = desc.nested_type
 def iterate_messages(desc, names = Names()):
     '''Recursively find all messages. For each, yield name, DescriptorProto.'''
     if hasattr(desc, 'message_type'):
         submsgs = desc.message_type
     else:
         submsgs = desc.nested_type

-    
+

     for submsg in submsgs:
         sub_names = names + submsg.name
         yield sub_names, submsg
     for submsg in submsgs:
         sub_names = names + submsg.name
         yield sub_names, submsg

-        
+

         for x in iterate_messages(submsg, sub_names):
             yield x
 
         for x in iterate_messages(submsg, sub_names):
             yield x
 


@@ -821,65 +994,22 @@ def iterate_extensions(desc, names = Names()):
         for extension in subdesc.extension:
             yield subname, extension
 
         for extension in subdesc.extension:
             yield subname, extension
 

-def parse_file(fdesc, file_options):
-    '''Takes a FileDescriptorProto and returns tuple (enums, messages, extensions).'''
-    
-    enums = []
-    messages = []
-    extensions = []
-    
-    if fdesc.package:
-        base_name = Names(fdesc.package.split('.'))
-    else:
-        base_name = Names()
-    
-    for enum in fdesc.enum_type:
-        enum_options = get_nanopb_suboptions(enum, file_options, base_name + enum.name)
-        enums.append(Enum(base_name, enum, enum_options))
-    
-    for names, message in iterate_messages(fdesc, base_name):
-        message_options = get_nanopb_suboptions(message, file_options, names)
-        
-        if message_options.skip_message:
-            continue
-        
-        messages.append(Message(names, message, message_options))
-        for enum in message.enum_type:
-            enum_options = get_nanopb_suboptions(enum, message_options, names + enum.name)
-            enums.append(Enum(names, enum, enum_options))
-    
-    for names, extension in iterate_extensions(fdesc, base_name):
-        field_options = get_nanopb_suboptions(extension, file_options, names + extension.name)
-        if field_options.type != nanopb_pb2.FT_IGNORE:
-            extensions.append(ExtensionField(names, extension, field_options))
-    
-    # Fix field default values where enum short names are used.
-    for enum in enums:
-        if not enum.options.long_names:
-            for message in messages:
-                for field in message.fields:
-                    if field.default in enum.value_longnames:
-                        idx = enum.value_longnames.index(field.default)
-                        field.default = enum.values[idx][0]
-    
-    return enums, messages, extensions
-

 def toposort2(data):
     '''Topological sort.
     From http://code.activestate.com/recipes/577413-topological-sort/
     This function is under the MIT license.
     '''
 def toposort2(data):
     '''Topological sort.
     From http://code.activestate.com/recipes/577413-topological-sort/
     This function is under the MIT license.
     '''

-    for k, v in data.items():
+    for k, v in list(data.items()):

         v.discard(k) # Ignore self dependencies
         v.discard(k) # Ignore self dependencies

-    extra_items_in_deps = reduce(set.union, data.values(), set()) - set(data.keys())
+    extra_items_in_deps = reduce(set.union, list(data.values()), set()) - set(data.keys())

     data.update(dict([(item, set()) for item in extra_items_in_deps]))
     while True:
     data.update(dict([(item, set()) for item in extra_items_in_deps]))
     while True:

-        ordered = set(item for item,dep in data.items() if not dep)
+        ordered = set(item for item,dep in list(data.items()) if not dep)

         if not ordered:
             break
         for item in sorted(ordered):
             yield item
         if not ordered:
             break
         for item in sorted(ordered):
             yield item

-        data = dict([(item, (dep - ordered)) for item,dep in data.items()
+        data = dict([(item, (dep - ordered)) for item,dep in list(data.items())

                 if item not in ordered])
     assert not data, "A cyclic dependency exists amongst %r" % data
 
                 if item not in ordered])
     assert not data, "A cyclic dependency exists amongst %r" % data
 


@@ -890,7 +1020,7 @@ def sort_dependencies(messages):
     for message in messages:
         dependencies[str(message.name)] = set(message.get_dependencies())
         message_by_name[str(message.name)] = message
     for message in messages:
         dependencies[str(message.name)] = set(message.get_dependencies())
         message_by_name[str(message.name)] = message

-    
+

     for msgname in toposort2(dependencies):
         if msgname in message_by_name:
             yield message_by_name[msgname]
     for msgname in toposort2(dependencies):
         if msgname in message_by_name:
             yield message_by_name[msgname]


@@ -905,203 +1035,309 @@ def make_identifier(headername):
             result += '_'
     return result
 
             result += '_'
     return result
 

-def generate_header(dependencies, headername, enums, messages, extensions, options):
-    '''Generate content for a header file.
-    Generates strings, which should be concatenated and stored to file.
-    '''
-    
-    yield '/* Automatically generated nanopb header */\n'
-    if options.notimestamp:
-        yield '/* Generated by %s */\n\n' % (nanopb_version)
-    else:
-        yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime())
-    
-    symbol = make_identifier(headername)
-    yield '#ifndef PB_%s_INCLUDED\n' % symbol
-    yield '#define PB_%s_INCLUDED\n' % symbol
-    try:
-        yield options.libformat % ('pb.h')
-    except TypeError:
-        # no %s specified - use whatever was passed in as options.libformat
-        yield options.libformat
-    yield '\n'
-    
-    for dependency in dependencies:
-        noext = os.path.splitext(dependency)[0]
-        yield options.genformat % (noext + options.extension + '.h')
+class ProtoFile:
+    def __init__(self, fdesc, file_options):
+        '''Takes a FileDescriptorProto and parses it.'''
+        self.fdesc = fdesc
+        self.file_options = file_options
+        self.dependencies = {}
+        self.parse()
+
+        # Some of types used in this file probably come from the file itself.
+        # Thus it has implicit dependency on itself.
+        self.add_dependency(self)
+
+    def parse(self):
+        self.enums = []
+        self.messages = []
+        self.extensions = []
+
+        if self.fdesc.package:
+            base_name = Names(self.fdesc.package.split('.'))
+        else:
+            base_name = Names()
+
+        for enum in self.fdesc.enum_type:
+            enum_options = get_nanopb_suboptions(enum, self.file_options, base_name + enum.name)
+            self.enums.append(Enum(base_name, enum, enum_options))
+
+        for names, message in iterate_messages(self.fdesc, base_name):
+            message_options = get_nanopb_suboptions(message, self.file_options, names)
+
+            if message_options.skip_message:
+                continue
+
+            self.messages.append(Message(names, message, message_options))
+            for enum in message.enum_type:
+                enum_options = get_nanopb_suboptions(enum, message_options, names + enum.name)
+                self.enums.append(Enum(names, enum, enum_options))
+
+        for names, extension in iterate_extensions(self.fdesc, base_name):
+            field_options = get_nanopb_suboptions(extension, self.file_options, names + extension.name)
+            if field_options.type != nanopb_pb2.FT_IGNORE:
+                self.extensions.append(ExtensionField(names, extension, field_options))
+
+    def add_dependency(self, other):
+        for enum in other.enums:
+            self.dependencies[str(enum.names)] = enum
+
+        for msg in other.messages:
+            self.dependencies[str(msg.name)] = msg
+
+        # Fix field default values where enum short names are used.
+        for enum in other.enums:
+            if not enum.options.long_names:
+                for message in self.messages:
+                    for field in message.fields:
+                        if field.default in enum.value_longnames:
+                            idx = enum.value_longnames.index(field.default)
+                            field.default = enum.values[idx][0]
+
+        # Fix field data types where enums have negative values.
+        for enum in other.enums:
+            if not enum.has_negative():
+                for message in self.messages:
+                    for field in message.fields:
+                        if field.pbtype == 'ENUM' and field.ctype == enum.names:
+                            field.pbtype = 'UENUM'
+
+    def generate_header(self, includes, headername, options):
+        '''Generate content for a header file.
+        Generates strings, which should be concatenated and stored to file.
+        '''
+
+        yield '/* Automatically generated nanopb header */\n'
+        if options.notimestamp:
+            yield '/* Generated by %s */\n\n' % (nanopb_version)
+        else:
+            yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime())
+
+        if self.fdesc.package:
+            symbol = make_identifier(self.fdesc.package + '_' + headername)
+        else:
+            symbol = make_identifier(headername)
+        yield '#ifndef PB_%s_INCLUDED\n' % symbol
+        yield '#define PB_%s_INCLUDED\n' % symbol
+        try:
+            yield options.libformat % ('pb.h')
+        except TypeError:
+            # no %s specified - use whatever was passed in as options.libformat
+            yield options.libformat

         yield '\n'
 
         yield '\n'
 

-    yield '#if PB_PROTO_HEADER_VERSION != 30\n'
-    yield '#error Regenerate this file with the current version of nanopb generator.\n'
-    yield '#endif\n'
-    yield '\n'
-
-    yield '#ifdef __cplusplus\n'
-    yield 'extern "C" {\n'
-    yield '#endif\n\n'
-    
-    yield '/* Enum definitions */\n'
-    for enum in enums:
-        yield str(enum) + '\n\n'
-    
-    yield '/* Struct definitions */\n'
-    for msg in sort_dependencies(messages):
-        yield msg.types()
-        yield str(msg) + '\n\n'
-    
-    if extensions:
-        yield '/* Extensions */\n'
-        for extension in extensions:
-            yield extension.extension_decl()
+        for incfile in includes:
+            noext = os.path.splitext(incfile)[0]
+            yield options.genformat % (noext + options.extension + '.h')
+            yield '\n'
+
+        yield '/* @@protoc_insertion_point(includes) */\n'
+
+        yield '#if PB_PROTO_HEADER_VERSION != 30\n'
+        yield '#error Regenerate this file with the current version of nanopb generator.\n'
+        yield '#endif\n'

         yield '\n'
         yield '\n'

-        
-    yield '/* Default values for struct fields */\n'
-    for msg in messages:
-        yield msg.default_decl(True)
-    yield '\n'
-    
-    yield '/* Initializer values for message structs */\n'
-    for msg in messages:
-        identifier = '%s_init_default' % msg.name
-        yield '#define %-40s %s\n' % (identifier, msg.get_initializer(False))
-    for msg in messages:
-        identifier = '%s_init_zero' % msg.name
-        yield '#define %-40s %s\n' % (identifier, msg.get_initializer(True))
-    yield '\n'
-    
-    yield '/* Field tags (for use in manual encoding/decoding) */\n'
-    for msg in sort_dependencies(messages):
-        for field in msg.fields:
-            yield field.tags()
-    for extension in extensions:
-        yield extension.tags()
-    yield '\n'
-    
-    yield '/* Struct field encoding specification for nanopb */\n'
-    for msg in messages:
-        yield msg.fields_declaration() + '\n'
-    yield '\n'
-    
-    yield '/* Maximum encoded size of messages (where known) */\n'
-    for msg in messages:
-        msize = msg.encoded_size(messages)
-        if msize is not None:
-            identifier = '%s_size' % msg.name
-            yield '#define %-40s %s\n' % (identifier, msize)
-    yield '\n'
-    
-    yield '#ifdef __cplusplus\n'
-    yield '} /* extern "C" */\n'
-    yield '#endif\n'
-    
-    # End of header
-    yield '\n#endif\n'
-
-def generate_source(headername, enums, messages, extensions, options):
-    '''Generate content for a source file.'''
-    
-    yield '/* Automatically generated nanopb constant definitions */\n'
-    if options.notimestamp:
-        yield '/* Generated by %s */\n\n' % (nanopb_version)
-    else:
-        yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime())
-    yield options.genformat % (headername)
-    yield '\n'
-    
-    yield '#if PB_PROTO_HEADER_VERSION != 30\n'
-    yield '#error Regenerate this file with the current version of nanopb generator.\n'
-    yield '#endif\n'
-    yield '\n'
-    
-    for msg in messages:
-        yield msg.default_decl(False)
-    
-    yield '\n\n'
-    
-    for msg in messages:
-        yield msg.fields_definition() + '\n\n'
-    
-    for ext in extensions:
-        yield ext.extension_def() + '\n'
-        
-    # Add checks for numeric limits
-    if messages:
-        largest_msg = max(messages, key = lambda m: m.count_required_fields())
-        largest_count = largest_msg.count_required_fields()
-        if largest_count > 64:
-            yield '\n/* Check that missing required fields will be properly detected */\n'
-            yield '#if PB_MAX_REQUIRED_FIELDS < %d\n' % largest_count
-            yield '#error Properly detecting missing required fields in %s requires \\\n' % largest_msg.name
-            yield '       setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count
-            yield '#endif\n'
-    
-    worst = 0
-    worst_field = ''
-    checks = []
-    checks_msgnames = []
-    for msg in messages:
-        checks_msgnames.append(msg.name)
-        for field in msg.fields:
-            status = field.largest_field_value()
-            if isinstance(status, (str, unicode)):
-                checks.append(status)
-            elif status > worst:
-                worst = status
-                worst_field = str(field.struct_name) + '.' + str(field.name)
-
-    if worst > 255 or checks:
-        yield '\n/* Check that field information fits in pb_field_t */\n'
-        
-        if worst > 65535 or checks:
-            yield '#if !defined(PB_FIELD_32BIT)\n'
-            if worst > 65535:
-                yield '#error Field descriptor for %s is too large. Define PB_FIELD_32BIT to fix this.\n' % worst_field
-            else:
-                assertion = ' && '.join(str(c) + ' < 65536' for c in checks)
-                msgs = '_'.join(str(n) for n in checks_msgnames)
-                yield '/* If you get an error here, it means that you need to define PB_FIELD_32BIT\n'
-                yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n'
-                yield ' * \n'
-                yield ' * The reason you need to do this is that some of your messages contain tag\n'
-                yield ' * numbers or field sizes that are larger than what can fit in 8 or 16 bit\n'
-                yield ' * field descriptors.\n'
-                yield ' */\n'
-                yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs)
-            yield '#endif\n\n'
-        
-        if worst < 65536:
-            yield '#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)\n'
-            if worst > 255:
-                yield '#error Field descriptor for %s is too large. Define PB_FIELD_16BIT to fix this.\n' % worst_field
-            else:
-                assertion = ' && '.join(str(c) + ' < 256' for c in checks)
-                msgs = '_'.join(str(n) for n in checks_msgnames)
-                yield '/* If you get an error here, it means that you need to define PB_FIELD_16BIT\n'
-                yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n'
-                yield ' * \n'
-                yield ' * The reason you need to do this is that some of your messages contain tag\n'
-                yield ' * numbers or field sizes that are larger than what can fit in the default\n'
-                yield ' * 8 bit descriptors.\n'
-                yield ' */\n'
-                yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs)
+
+        yield '#ifdef __cplusplus\n'
+        yield 'extern "C" {\n'
+        yield '#endif\n\n'
+
+        if self.enums:
+            yield '/* Enum definitions */\n'
+            for enum in self.enums:
+                yield str(enum) + '\n\n'
+
+        if self.messages:
+            yield '/* Struct definitions */\n'
+            for msg in sort_dependencies(self.messages):
+                yield msg.types()
+                yield str(msg) + '\n\n'
+
+        if self.extensions:
+            yield '/* Extensions */\n'
+            for extension in self.extensions:
+                yield extension.extension_decl()
+            yield '\n'
+
+        if self.messages:
+            yield '/* Default values for struct fields */\n'
+            for msg in self.messages:
+                yield msg.default_decl(True)
+            yield '\n'
+
+            yield '/* Initializer values for message structs */\n'
+            for msg in self.messages:
+                identifier = '%s_init_default' % msg.name
+                yield '#define %-40s %s\n' % (identifier, msg.get_initializer(False))
+            for msg in self.messages:
+                identifier = '%s_init_zero' % msg.name
+                yield '#define %-40s %s\n' % (identifier, msg.get_initializer(True))
+            yield '\n'
+
+            yield '/* Field tags (for use in manual encoding/decoding) */\n'
+            for msg in sort_dependencies(self.messages):
+                for field in msg.fields:
+                    yield field.tags()
+            for extension in self.extensions:
+                yield extension.tags()
+            yield '\n'
+
+            yield '/* Struct field encoding specification for nanopb */\n'
+            for msg in self.messages:
+                yield msg.fields_declaration() + '\n'
+            yield '\n'
+
+            yield '/* Maximum encoded size of messages (where known) */\n'
+            for msg in self.messages:
+                msize = msg.encoded_size(self.dependencies)
+                identifier = '%s_size' % msg.name
+                if msize is not None:
+                    yield '#define %-40s %s\n' % (identifier, msize)
+                else:
+                    yield '/* %s depends on runtime parameters */\n' % identifier
+            yield '\n'
+
+            yield '/* Message IDs (where set with "msgid" option) */\n'
+
+            yield '#ifdef PB_MSGID\n'
+            for msg in self.messages:
+                if hasattr(msg,'msgid'):
+                    yield '#define PB_MSG_%d %s\n' % (msg.msgid, msg.name)
+            yield '\n'
+
+            symbol = make_identifier(headername.split('.')[0])
+            yield '#define %s_MESSAGES \\\n' % symbol
+
+            for msg in self.messages:
+                m = "-1"
+                msize = msg.encoded_size(self.dependencies)
+                if msize is not None:
+                    m = msize
+                if hasattr(msg,'msgid'):
+                    yield '\tPB_MSG(%d,%s,%s) \\\n' % (msg.msgid, m, msg.name)
+            yield '\n'
+
+            for msg in self.messages:
+                if hasattr(msg,'msgid'):
+                    yield '#define %s_msgid %d\n' % (msg.name, msg.msgid)
+            yield '\n'
+

             yield '#endif\n\n'
             yield '#endif\n\n'

-    
-    # Add check for sizeof(double)
-    has_double = False
-    for msg in messages:
-        for field in msg.fields:
-            if field.ctype == 'double':
-                has_double = True
-    
-    if has_double:
+
+        yield '#ifdef __cplusplus\n'
+        yield '} /* extern "C" */\n'
+        yield '#endif\n'
+
+        # End of header
+        yield '/* @@protoc_insertion_point(eof) */\n'
+        yield '\n#endif\n'
+
+    def generate_source(self, headername, options):
+        '''Generate content for a source file.'''
+
+        yield '/* Automatically generated nanopb constant definitions */\n'
+        if options.notimestamp:
+            yield '/* Generated by %s */\n\n' % (nanopb_version)
+        else:
+            yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime())
+        yield options.genformat % (headername)
+        yield '\n'
+        yield '/* @@protoc_insertion_point(includes) */\n'
+
+        yield '#if PB_PROTO_HEADER_VERSION != 30\n'
+        yield '#error Regenerate this file with the current version of nanopb generator.\n'
+        yield '#endif\n'

         yield '\n'
         yield '\n'

-        yield '/* On some platforms (such as AVR), double is really float.\n'
-        yield ' * These are not directly supported by nanopb, but see example_avr_double.\n'
-        yield ' * To get rid of this error, remove any double fields from your .proto.\n'
-        yield ' */\n'
-        yield 'PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)\n'
-    
-    yield '\n'
+
+        for msg in self.messages:
+            yield msg.default_decl(False)
+
+        yield '\n\n'
+
+        for msg in self.messages:
+            yield msg.fields_definition() + '\n\n'
+
+        for ext in self.extensions:
+            yield ext.extension_def() + '\n'
+
+        for enum in self.enums:
+            yield enum.enum_to_string_definition() + '\n'
+
+        # Add checks for numeric limits
+        if self.messages:
+            largest_msg = max(self.messages, key = lambda m: m.count_required_fields())
+            largest_count = largest_msg.count_required_fields()
+            if largest_count > 64:
+                yield '\n/* Check that missing required fields will be properly detected */\n'
+                yield '#if PB_MAX_REQUIRED_FIELDS < %d\n' % largest_count
+                yield '#error Properly detecting missing required fields in %s requires \\\n' % largest_msg.name
+                yield '       setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count
+                yield '#endif\n'
+
+        max_field = FieldMaxSize()
+        checks_msgnames = []
+        for msg in self.messages:
+            checks_msgnames.append(msg.name)
+            for field in msg.fields:
+                max_field.extend(field.largest_field_value())
+
+        worst = max_field.worst
+        worst_field = max_field.worst_field
+        checks = max_field.checks
+
+        if worst > 255 or checks:
+            yield '\n/* Check that field information fits in pb_field_t */\n'
+
+            if worst > 65535 or checks:
+                yield '#if !defined(PB_FIELD_32BIT)\n'
+                if worst > 65535:
+                    yield '#error Field descriptor for %s is too large. Define PB_FIELD_32BIT to fix this.\n' % worst_field
+                else:
+                    assertion = ' && '.join(str(c) + ' < 65536' for c in checks)
+                    msgs = '_'.join(str(n) for n in checks_msgnames)
+                    yield '/* If you get an error here, it means that you need to define PB_FIELD_32BIT\n'
+                    yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n'
+                    yield ' * \n'
+                    yield ' * The reason you need to do this is that some of your messages contain tag\n'
+                    yield ' * numbers or field sizes that are larger than what can fit in 8 or 16 bit\n'
+                    yield ' * field descriptors.\n'
+                    yield ' */\n'
+                    yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs)
+                yield '#endif\n\n'
+
+            if worst < 65536:
+                yield '#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)\n'
+                if worst > 255:
+                    yield '#error Field descriptor for %s is too large. Define PB_FIELD_16BIT to fix this.\n' % worst_field
+                else:
+                    assertion = ' && '.join(str(c) + ' < 256' for c in checks)
+                    msgs = '_'.join(str(n) for n in checks_msgnames)
+                    yield '/* If you get an error here, it means that you need to define PB_FIELD_16BIT\n'
+                    yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n'
+                    yield ' * \n'
+                    yield ' * The reason you need to do this is that some of your messages contain tag\n'
+                    yield ' * numbers or field sizes that are larger than what can fit in the default\n'
+                    yield ' * 8 bit descriptors.\n'
+                    yield ' */\n'
+                    yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs)
+                yield '#endif\n\n'
+
+        # Add check for sizeof(double)
+        has_double = False
+        for msg in self.messages:
+            for field in msg.fields:
+                if field.ctype == 'double':
+                    has_double = True
+
+        if has_double:
+            yield '\n'
+            yield '/* On some platforms (such as AVR), double is really float.\n'
+            yield ' * These are not directly supported by nanopb, but see example_avr_double.\n'
+            yield ' * To get rid of this error, remove any double fields from your .proto.\n'
+            yield ' */\n'
+            yield 'PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)\n'
+
+        yield '\n'
+        yield '/* @@protoc_insertion_point(eof) */\n'

 
 # ---------------------------------------------------------------------------
 #                    Options parsing for the .proto files
 
 # ---------------------------------------------------------------------------
 #                    Options parsing for the .proto files


@@ -1114,14 +1350,32 @@ def read_options_file(infile):
         [(namemask, options), ...]
     '''
     results = []
         [(namemask, options), ...]
     '''
     results = []

-    for line in infile:
+    data = infile.read()
+    data = re.sub('/\*.*?\*/', '', data, flags = re.MULTILINE)
+    data = re.sub('//.*?$', '', data, flags = re.MULTILINE)
+    data = re.sub('#.*?$', '', data, flags = re.MULTILINE)
+    for i, line in enumerate(data.split('\n')):

         line = line.strip()
         line = line.strip()

-        if not line or line.startswith('//') or line.startswith('#'):
+        if not line:

             continue
             continue

-        
+

         parts = line.split(None, 1)
         parts = line.split(None, 1)

+
+        if len(parts) < 2:
+            sys.stderr.write("%s:%d: " % (infile.name, i + 1) +
+                             "Option lines should have space between field name and options. " +
+                             "Skipping line: '%s'\n" % line)
+            continue
+

         opts = nanopb_pb2.NanoPBOptions()
         opts = nanopb_pb2.NanoPBOptions()

-        text_format.Merge(parts[1], opts)
+
+        try:
+            text_format.Merge(parts[1], opts)
+        except Exception as e:
+            sys.stderr.write("%s:%d: " % (infile.name, i + 1) +
+                             "Unparseable option line: '%s'. " % line +
+                             "Error: %s\n" % str(e))
+            continue

         results.append((parts[0], opts))
 
     return results
         results.append((parts[0], opts))
 
     return results


@@ -1136,14 +1390,17 @@ def get_nanopb_suboptions(subdesc, options, name):
     '''Get copy of options, and merge information from subdesc.'''
     new_options = nanopb_pb2.NanoPBOptions()
     new_options.CopyFrom(options)
     '''Get copy of options, and merge information from subdesc.'''
     new_options = nanopb_pb2.NanoPBOptions()
     new_options.CopyFrom(options)

-    
+
+    if hasattr(subdesc, 'syntax') and subdesc.syntax == "proto3":
+        new_options.proto3 = True
+

     # Handle options defined in a separate file
     dotname = '.'.join(name.parts)
     for namemask, options in Globals.separate_options:
         if fnmatch(dotname, namemask):
             Globals.matched_namemasks.add(namemask)
             new_options.MergeFrom(options)
     # Handle options defined in a separate file
     dotname = '.'.join(name.parts)
     for namemask, options in Globals.separate_options:
         if fnmatch(dotname, namemask):
             Globals.matched_namemasks.add(namemask)
             new_options.MergeFrom(options)

-    
+

     # Handle options defined in .proto
     if isinstance(subdesc.options, descriptor.FieldOptions):
         ext_type = nanopb_pb2.nanopb
     # Handle options defined in .proto
     if isinstance(subdesc.options, descriptor.FieldOptions):
         ext_type = nanopb_pb2.nanopb


@@ -1155,15 +1412,15 @@ def get_nanopb_suboptions(subdesc, options, name):
         ext_type = nanopb_pb2.nanopb_enumopt
     else:
         raise Exception("Unknown options type")
         ext_type = nanopb_pb2.nanopb_enumopt
     else:
         raise Exception("Unknown options type")

-    
+

     if subdesc.options.HasExtension(ext_type):
         ext = subdesc.options.Extensions[ext_type]
         new_options.MergeFrom(ext)
     if subdesc.options.HasExtension(ext_type):
         ext = subdesc.options.Extensions[ext_type]
         new_options.MergeFrom(ext)

-    
+

     if Globals.verbose_options:
         sys.stderr.write("Options for " + dotname + ": ")
         sys.stderr.write(text_format.MessageToString(new_options) + "\n")
     if Globals.verbose_options:
         sys.stderr.write("Options for " + dotname + ": ")
         sys.stderr.write(text_format.MessageToString(new_options) + "\n")

-    
+

     return new_options
 
 
     return new_options
 
 


@@ -1172,7 +1429,7 @@ def get_nanopb_suboptions(subdesc, options, name):
 # ---------------------------------------------------------------------------
 
 import sys
 # ---------------------------------------------------------------------------
 
 import sys

-import os.path    
+import os.path

 from optparse import OptionParser
 
 optparser = OptionParser(
 from optparse import OptionParser
 
 optparser = OptionParser(


@@ -1185,6 +1442,12 @@ optparser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION",
     help="Set extension to use instead of '.pb' for generated files. [default: %default]")
 optparser.add_option("-f", "--options-file", dest="options_file", metavar="FILE", default="%s.options",
     help="Set name of a separate generator options file.")
     help="Set extension to use instead of '.pb' for generated files. [default: %default]")
 optparser.add_option("-f", "--options-file", dest="options_file", metavar="FILE", default="%s.options",
     help="Set name of a separate generator options file.")

+optparser.add_option("-I", "--options-path", dest="options_path", metavar="DIR",
+    action="append", default = [],
+    help="Search for .options files additionally in this path")
+optparser.add_option("-D", "--output-dir", dest="output_dir",
+                     metavar="OUTPUTDIR", default=None,
+                     help="Output directory of .pb.h and .pb.c files")

 optparser.add_option("-Q", "--generated-include-format", dest="genformat",
     metavar="FORMAT", default='#include "%s"\n',
     help="Set format string to use for including other .pb.h files. [default: %default]")
 optparser.add_option("-Q", "--generated-include-format", dest="genformat",
     metavar="FORMAT", default='#include "%s"\n',
     help="Set format string to use for including other .pb.h files. [default: %default]")


@@ -1200,27 +1463,16 @@ optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", def
 optparser.add_option("-s", dest="settings", metavar="OPTION:VALUE", action="append", default=[],
     help="Set generator option (max_size, max_count etc.).")
 
 optparser.add_option("-s", dest="settings", metavar="OPTION:VALUE", action="append", default=[],
     help="Set generator option (max_size, max_count etc.).")
 

-def process_file(filename, fdesc, options):
-    '''Process a single file.
-    filename: The full path to the .proto or .pb source file, as string.
-    fdesc: The loaded FileDescriptorSet, or None to read from the input file.
-    options: Command line options as they come from OptionsParser.
-    
-    Returns a dict:
-        {'headername': Name of header file,
-         'headerdata': Data for the .h header file,
-         'sourcename': Name of the source code file,
-         'sourcedata': Data for the .c source code file
-        }
-    '''
+def parse_file(filename, fdesc, options):
+    '''Parse a single file. Returns a ProtoFile instance.'''

     toplevel_options = nanopb_pb2.NanoPBOptions()
     for s in options.settings:
         text_format.Merge(s, toplevel_options)
     toplevel_options = nanopb_pb2.NanoPBOptions()
     for s in options.settings:
         text_format.Merge(s, toplevel_options)

-    
+

     if not fdesc:
         data = open(filename, 'rb').read()
         fdesc = descriptor.FileDescriptorSet.FromString(data).file[0]
     if not fdesc:
         data = open(filename, 'rb').read()
         fdesc = descriptor.FileDescriptorSet.FromString(data).file[0]

-    
+

     # Check if there is a separate .options file
     had_abspath = False
     try:
     # Check if there is a separate .options file
     had_abspath = False
     try:


@@ -1230,113 +1482,167 @@ def process_file(filename, fdesc, options):
         optfilename = options.options_file
         had_abspath = True
 
         optfilename = options.options_file
         had_abspath = True
 

-    if os.path.isfile(optfilename):
-        if options.verbose:
-            sys.stderr.write('Reading options from ' + optfilename + '\n')
-
-        Globals.separate_options = read_options_file(open(optfilename, "rU"))
+    paths = ['.'] + options.options_path
+    for p in paths:
+        if os.path.isfile(os.path.join(p, optfilename)):
+            optfilename = os.path.join(p, optfilename)
+            if options.verbose:
+                sys.stderr.write('Reading options from ' + optfilename + '\n')
+            Globals.separate_options = read_options_file(open(optfilename, "rU"))
+            break

     else:
         # If we are given a full filename and it does not exist, give an error.
         # However, don't give error when we automatically look for .options file
         # with the same name as .proto.
         if options.verbose or had_abspath:
     else:
         # If we are given a full filename and it does not exist, give an error.
         # However, don't give error when we automatically look for .options file
         # with the same name as .proto.
         if options.verbose or had_abspath:

-            sys.stderr.write('Options file not found: ' + optfilename)
-
+            sys.stderr.write('Options file not found: ' + optfilename + '\n')

         Globals.separate_options = []
 
     Globals.matched_namemasks = set()
         Globals.separate_options = []
 
     Globals.matched_namemasks = set()

-    
+

     # Parse the file
     file_options = get_nanopb_suboptions(fdesc, toplevel_options, Names([filename]))
     # Parse the file
     file_options = get_nanopb_suboptions(fdesc, toplevel_options, Names([filename]))

-    enums, messages, extensions = parse_file(fdesc, file_options)
+    f = ProtoFile(fdesc, file_options)
+    f.optfilename = optfilename
+
+    return f
+
+def process_file(filename, fdesc, options, other_files = {}):
+    '''Process a single file.
+    filename: The full path to the .proto or .pb source file, as string.
+    fdesc: The loaded FileDescriptorSet, or None to read from the input file.
+    options: Command line options as they come from OptionsParser.
+
+    Returns a dict:
+        {'headername': Name of header file,
+         'headerdata': Data for the .h header file,
+         'sourcename': Name of the source code file,
+         'sourcedata': Data for the .c source code file
+        }
+    '''
+    f = parse_file(filename, fdesc, options)
+
+    # Provide dependencies if available
+    for dep in f.fdesc.dependency:
+        if dep in other_files:
+            f.add_dependency(other_files[dep])

 
     # Decide the file names
     noext = os.path.splitext(filename)[0]
     headername = noext + options.extension + '.h'
     sourcename = noext + options.extension + '.c'
     headerbasename = os.path.basename(headername)
 
     # Decide the file names
     noext = os.path.splitext(filename)[0]
     headername = noext + options.extension + '.h'
     sourcename = noext + options.extension + '.c'
     headerbasename = os.path.basename(headername)

-    
+

     # List of .proto files that should not be included in the C header file
     # even if they are mentioned in the source .proto.
     excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude
     # List of .proto files that should not be included in the C header file
     # even if they are mentioned in the source .proto.
     excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude

-    dependencies = [d for d in fdesc.dependency if d not in excludes]
-    
-    headerdata = ''.join(generate_header(dependencies, headerbasename, enums,
-                                         messages, extensions, options))
+    includes = [d for d in f.fdesc.dependency if d not in excludes]

 
 

-    sourcedata = ''.join(generate_source(headerbasename, enums,
-                                         messages, extensions, options))
+    headerdata = ''.join(f.generate_header(includes, headerbasename, options))
+    sourcedata = ''.join(f.generate_source(headerbasename, options))

 
     # Check if there were any lines in .options that did not match a member
     unmatched = [n for n,o in Globals.separate_options if n not in Globals.matched_namemasks]
     if unmatched and not options.quiet:
 
     # Check if there were any lines in .options that did not match a member
     unmatched = [n for n,o in Globals.separate_options if n not in Globals.matched_namemasks]
     if unmatched and not options.quiet:

-        sys.stderr.write("Following patterns in " + optfilename + " did not match any fields: "
+        sys.stderr.write("Following patterns in " + f.optfilename + " did not match any fields: "

                          + ', '.join(unmatched) + "\n")
         if not Globals.verbose_options:
             sys.stderr.write("Use  protoc --nanopb-out=-v:.   to see a list of the field names.\n")
 
     return {'headername': headername, 'headerdata': headerdata,
             'sourcename': sourcename, 'sourcedata': sourcedata}
                          + ', '.join(unmatched) + "\n")
         if not Globals.verbose_options:
             sys.stderr.write("Use  protoc --nanopb-out=-v:.   to see a list of the field names.\n")
 
     return {'headername': headername, 'headerdata': headerdata,
             'sourcename': sourcename, 'sourcedata': sourcedata}

-    
+

 def main_cli():
     '''Main function when invoked directly from the command line.'''
 def main_cli():
     '''Main function when invoked directly from the command line.'''

-    
+

     options, filenames = optparser.parse_args()
     options, filenames = optparser.parse_args()

-    
+

     if not filenames:
         optparser.print_help()
         sys.exit(1)
     if not filenames:
         optparser.print_help()
         sys.exit(1)

-    
+

     if options.quiet:
         options.verbose = False
 
     if options.quiet:
         options.verbose = False
 

+    if options.output_dir and not os.path.exists(options.output_dir):
+        optparser.print_help()
+        sys.stderr.write("\noutput_dir does not exist: %s\n" % options.output_dir)
+        sys.exit(1)
+
+

     Globals.verbose_options = options.verbose
     Globals.verbose_options = options.verbose

-    

     for filename in filenames:
         results = process_file(filename, None, options)
     for filename in filenames:
         results = process_file(filename, None, options)

-        
+
+        base_dir = options.output_dir or ''
+        to_write = [
+            (os.path.join(base_dir, results['headername']), results['headerdata']),
+            (os.path.join(base_dir, results['sourcename']), results['sourcedata']),
+        ]
+

         if not options.quiet:
         if not options.quiet:

-            sys.stderr.write("Writing to " + results['headername'] + " and "
-                             + results['sourcename'] + "\n")
-    
-        open(results['headername'], 'w').write(results['headerdata'])
-        open(results['sourcename'], 'w').write(results['sourcedata'])        
+            paths = " and ".join([x[0] for x in to_write])
+            sys.stderr.write("Writing to %s\n" % paths)
+
+        for path, data in to_write:
+            with open(path, 'w') as f:
+                f.write(data)

 
 def main_plugin():
     '''Main function when invoked as a protoc plugin.'''
 
 
 def main_plugin():
     '''Main function when invoked as a protoc plugin.'''
 

-    import sys
+    import io, sys

     if sys.platform == "win32":
         import os, msvcrt
         # Set stdin and stdout to binary mode
         msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
         msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
     if sys.platform == "win32":
         import os, msvcrt
         # Set stdin and stdout to binary mode
         msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
         msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)

-    
-    data = sys.stdin.read()
+
+    data = io.open(sys.stdin.fileno(), "rb").read()
+

     request = plugin_pb2.CodeGeneratorRequest.FromString(data)
     request = plugin_pb2.CodeGeneratorRequest.FromString(data)

-    
+
+    try:
+        # Versions of Python prior to 2.7.3 do not support unicode
+        # input to shlex.split(). Try to convert to str if possible.
+        params = str(request.parameter)
+    except UnicodeEncodeError:
+        params = request.parameter
+

     import shlex
     import shlex

-    args = shlex.split(request.parameter)
+    args = shlex.split(params)

     options, dummy = optparser.parse_args(args)
     options, dummy = optparser.parse_args(args)

-    
+

     Globals.verbose_options = options.verbose
     Globals.verbose_options = options.verbose

-    
+

     response = plugin_pb2.CodeGeneratorResponse()
     response = plugin_pb2.CodeGeneratorResponse()

-    
+
+    # Google's protoc does not currently indicate the full path of proto files.
+    # Instead always add the main file path to the search dirs, that works for
+    # the common case.
+    import os.path
+    options.options_path.append(os.path.dirname(request.file_to_generate[0]))
+
+    # Process any include files first, in order to have them
+    # available as dependencies
+    other_files = {}
+    for fdesc in request.proto_file:
+        other_files[fdesc.name] = parse_file(fdesc.name, fdesc, options)
+

     for filename in request.file_to_generate:
         for fdesc in request.proto_file:
             if fdesc.name == filename:
     for filename in request.file_to_generate:
         for fdesc in request.proto_file:
             if fdesc.name == filename:

-                results = process_file(filename, fdesc, options)
-                
+                results = process_file(filename, fdesc, options, other_files)
+

                 f = response.file.add()
                 f.name = results['headername']
                 f.content = results['headerdata']
 
                 f = response.file.add()
                 f.name = results['sourcename']
                 f = response.file.add()
                 f.name = results['headername']
                 f.content = results['headerdata']
 
                 f = response.file.add()
                 f.name = results['sourcename']

-                f.content = results['sourcedata']    
-    
-    sys.stdout.write(response.SerializeToString())
+                f.content = results['sourcedata']
+
+    io.open(sys.stdout.fileno(), "wb").write(response.SerializeToString())

 
 if __name__ == '__main__':
     # Check if we are running as a plugin under protoc
 
 if __name__ == '__main__':
     # Check if we are running as a plugin under protoc


@@ -1344,4 +1650,3 @@ if __name__ == '__main__':
         main_plugin()
     else:
         main_cli()
         main_plugin()
     else:
         main_cli()

-