Merge pull request #169 from kylemanna/python3

[apps/agl-service-can-low-level.git] / generator / nanopb_generator.py

diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py
index 5010814..7fe0db9 100755 (executable)
--- a/generator/nanopb_generator.py
+++ b/generator/nanopb_generator.py
@@ -1,9 +1,13 @@
-#!/usr/bin/python
+#!/usr/bin/env python
+
+from __future__ import unicode_literals

 
 '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.'''
 
 '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.'''

-nanopb_version = "nanopb-0.3.0-dev"
+nanopb_version = "nanopb-0.3.5-dev"

 
 import sys
 
 import sys

+import re
+from functools import reduce

 
 try:
     # Add some dummy imports to keep packaging tools happy.
 
 try:
     # Add some dummy imports to keep packaging tools happy.


@@ -44,22 +48,30 @@ except:
 import time
 import os.path
 
 import time
 import os.path
 

-# Values are tuple (c type, pb type, encoded size)
+# Values are tuple (c type, pb type, encoded size, int_size_allowed)

 FieldD = descriptor.FieldDescriptorProto
 datatypes = {
 FieldD = descriptor.FieldDescriptorProto
 datatypes = {

-    FieldD.TYPE_BOOL:       ('bool',     'BOOL',        1),
-    FieldD.TYPE_DOUBLE:     ('double',   'DOUBLE',      8),
-    FieldD.TYPE_FIXED32:    ('uint32_t', 'FIXED32',     4),
-    FieldD.TYPE_FIXED64:    ('uint64_t', 'FIXED64',     8),
-    FieldD.TYPE_FLOAT:      ('float',    'FLOAT',       4),
-    FieldD.TYPE_INT32:      ('int32_t',  'INT32',      10),
-    FieldD.TYPE_INT64:      ('int64_t',  'INT64',      10),
-    FieldD.TYPE_SFIXED32:   ('int32_t',  'SFIXED32',    4),
-    FieldD.TYPE_SFIXED64:   ('int64_t',  'SFIXED64',    8),
-    FieldD.TYPE_SINT32:     ('int32_t',  'SINT32',      5),
-    FieldD.TYPE_SINT64:     ('int64_t',  'SINT64',     10),
-    FieldD.TYPE_UINT32:     ('uint32_t', 'UINT32',      5),
-    FieldD.TYPE_UINT64:     ('uint64_t', 'UINT64',     10)
+    FieldD.TYPE_BOOL:       ('bool',     'BOOL',        1,  False),
+    FieldD.TYPE_DOUBLE:     ('double',   'DOUBLE',      8,  False),
+    FieldD.TYPE_FIXED32:    ('uint32_t', 'FIXED32',     4,  False),
+    FieldD.TYPE_FIXED64:    ('uint64_t', 'FIXED64',     8,  False),
+    FieldD.TYPE_FLOAT:      ('float',    'FLOAT',       4,  False),
+    FieldD.TYPE_INT32:      ('int32_t',  'INT32',      10,  True),
+    FieldD.TYPE_INT64:      ('int64_t',  'INT64',      10,  True),
+    FieldD.TYPE_SFIXED32:   ('int32_t',  'SFIXED32',    4,  False),
+    FieldD.TYPE_SFIXED64:   ('int64_t',  'SFIXED64',    8,  False),
+    FieldD.TYPE_SINT32:     ('int32_t',  'SINT32',      5,  True),
+    FieldD.TYPE_SINT64:     ('int64_t',  'SINT64',     10,  True),
+    FieldD.TYPE_UINT32:     ('uint32_t', 'UINT32',      5,  True),
+    FieldD.TYPE_UINT64:     ('uint64_t', 'UINT64',     10,  True)
+}
+
+# Integer size overrides (from .proto settings)
+intsizes = {
+    nanopb_pb2.IS_8:     'int8_t',
+    nanopb_pb2.IS_16:    'int16_t',
+    nanopb_pb2.IS_32:    'int32_t',
+    nanopb_pb2.IS_64:    'int64_t',

 }
 
 class Names:
 }
 
 class Names:


@@ -73,7 +85,14 @@ class Names:
         return '_'.join(self.parts)
 
     def __add__(self, other):
         return '_'.join(self.parts)
 
     def __add__(self, other):

-        if isinstance(other, (str, unicode)):
+        # The fdesc names are unicode and need to be handled for
+        # python2 and python3
+        try:
+              realstr = unicode
+        except NameError:
+              realstr = str
+
+        if isinstance(other, realstr):

             return Names(self.parts + (other,))
         elif isinstance(other, tuple):
             return Names(self.parts + other)
             return Names(self.parts + (other,))
         elif isinstance(other, tuple):
             return Names(self.parts + other)


@@ -91,11 +110,14 @@ def names_from_type_name(type_name):
 
 def varint_max_size(max_value):
     '''Returns the maximum number of bytes a varint can take when encoded.'''
 
 def varint_max_size(max_value):
     '''Returns the maximum number of bytes a varint can take when encoded.'''

+    if max_value < 0:
+        max_value = 2**64 - max_value

     for i in range(1, 11):
         if (max_value >> (i * 7)) == 0:
             return i
     raise ValueError("Value too large for varint: " + str(max_value))
 
     for i in range(1, 11):
         if (max_value >> (i * 7)) == 0:
             return i
     raise ValueError("Value too large for varint: " + str(max_value))
 

+assert varint_max_size(-1) == 10

 assert varint_max_size(0) == 1
 assert varint_max_size(127) == 1
 assert varint_max_size(128) == 2
 assert varint_max_size(0) == 1
 assert varint_max_size(127) == 1
 assert varint_max_size(128) == 2


@@ -111,7 +133,7 @@ class EncodedSize:
         self.symbols = symbols
     
     def __add__(self, other):
         self.symbols = symbols
     
     def __add__(self, other):

-        if isinstance(other, (int, long)):
+        if isinstance(other, int):

             return EncodedSize(self.value + other, self.symbols)
         elif isinstance(other, (str, Names)):
             return EncodedSize(self.value, self.symbols + [str(other)])
             return EncodedSize(self.value + other, self.symbols)
         elif isinstance(other, (str, Names)):
             return EncodedSize(self.value, self.symbols + [str(other)])


@@ -121,7 +143,7 @@ class EncodedSize:
             raise ValueError("Cannot add size: " + repr(other))
 
     def __mul__(self, other):
             raise ValueError("Cannot add size: " + repr(other))
 
     def __mul__(self, other):

-        if isinstance(other, (int, long)):
+        if isinstance(other, int):

             return EncodedSize(self.value * other, [str(other) + '*' + s for s in self.symbols])
         else:
             raise ValueError("Cannot multiply size: " + repr(other))
             return EncodedSize(self.value * other, [str(other) + '*' + s for s in self.symbols])
         else:
             raise ValueError("Cannot multiply size: " + repr(other))


@@ -151,18 +173,59 @@ class Enum:
             self.values = [(names + x.name, x.number) for x in desc.value] 
         
         self.value_longnames = [self.names + x.name for x in desc.value]
             self.values = [(names + x.name, x.number) for x in desc.value] 
         
         self.value_longnames = [self.names + x.name for x in desc.value]

+        self.packed = enum_options.packed_enum
+    
+    def has_negative(self):
+        for n, v in self.values:
+            if v < 0:
+                return True
+        return False
+    
+    def encoded_size(self):
+        return max([varint_max_size(v) for n,v in self.values])

     
     def __str__(self):
         result = 'typedef enum _%s {\n' % self.names
         result += ',\n'.join(["    %s = %d" % x for x in self.values])
     
     def __str__(self):
         result = 'typedef enum _%s {\n' % self.names
         result += ',\n'.join(["    %s = %d" % x for x in self.values])

-        result += '\n} %s;' % self.names
+        result += '\n}'
+        
+        if self.packed:
+            result += ' pb_packed'
+        
+        result += ' %s;' % self.names
+        
+        if not self.options.long_names:
+            # Define the long names always so that enum value references
+            # from other files work properly.
+            for i, x in enumerate(self.values):
+                result += '\n#define %s %s' % (self.value_longnames[i], x[0])
+        

         return result
 
         return result
 

+class FieldMaxSize:
+    def __init__(self, worst = 0, checks = [], field_name = 'undefined'):
+        if isinstance(worst, list):
+            self.worst = max(i for i in worst if i is not None)
+        else:
+            self.worst = worst
+
+        self.worst_field = field_name
+        self.checks = checks
+
+    def extend(self, extend, field_name = None):
+        self.worst = max(self.worst, extend.worst)
+
+        if self.worst == extend.worst:
+            self.worst_field = extend.worst_field
+
+        self.checks.extend(extend.checks)
+

 class Field:
     def __init__(self, struct_name, desc, field_options):
         '''desc is FieldDescriptorProto'''
         self.tag = desc.number
         self.struct_name = struct_name
 class Field:
     def __init__(self, struct_name, desc, field_options):
         '''desc is FieldDescriptorProto'''
         self.tag = desc.number
         self.struct_name = struct_name

+        self.union_name = None

         self.name = desc.name
         self.default = None
         self.max_size = None
         self.name = desc.name
         self.default = None
         self.max_size = None


@@ -225,14 +288,20 @@ class Field:
             raise NotImplementedError(field_options.type)
         
         # Decide the C data type to use in the struct.
             raise NotImplementedError(field_options.type)
         
         # Decide the C data type to use in the struct.

-        if datatypes.has_key(desc.type):
-            self.ctype, self.pbtype, self.enc_size = datatypes[desc.type]
+        if desc.type in datatypes:
+            self.ctype, self.pbtype, self.enc_size, isa = datatypes[desc.type]
+
+            # Override the field size if user wants to use smaller integers
+            if isa and field_options.int_size != nanopb_pb2.IS_DEFAULT:
+                self.ctype = intsizes[field_options.int_size]
+                if desc.type == FieldD.TYPE_UINT32 or desc.type == FieldD.TYPE_UINT64:
+                    self.ctype = 'u' + self.ctype;

         elif desc.type == FieldD.TYPE_ENUM:
             self.pbtype = 'ENUM'
             self.ctype = names_from_type_name(desc.type_name)
             if self.default is not None:
                 self.default = self.ctype + self.default
         elif desc.type == FieldD.TYPE_ENUM:
             self.pbtype = 'ENUM'
             self.ctype = names_from_type_name(desc.type_name)
             if self.default is not None:
                 self.default = self.ctype + self.default

-            self.enc_size = 5 # protoc rejects enum values > 32 bits
+            self.enc_size = None # Needs to be filled in when enum values are known

         elif desc.type == FieldD.TYPE_STRING:
             self.pbtype = 'STRING'
             self.ctype = 'char'
         elif desc.type == FieldD.TYPE_STRING:
             self.pbtype = 'STRING'
             self.ctype = 'char'


@@ -254,8 +323,8 @@ class Field:
         else:
             raise NotImplementedError(desc.type)
         
         else:
             raise NotImplementedError(desc.type)
         

-    def __cmp__(self, other):
-        return cmp(self.tag, other.tag)
+    def __lt__(self, other):
+        return self.tag < other.tag

     
     def __str__(self):
         result = ''
     
     def __str__(self):
         result = ''


@@ -286,57 +355,89 @@ class Field:
         if self.pbtype == 'BYTES' and self.allocation == 'STATIC':
             result = 'typedef PB_BYTES_ARRAY_T(%d) %s;\n' % (self.max_size, self.ctype)
         else:
         if self.pbtype == 'BYTES' and self.allocation == 'STATIC':
             result = 'typedef PB_BYTES_ARRAY_T(%d) %s;\n' % (self.max_size, self.ctype)
         else:

-            result = None
+            result = ''

         return result
     
         return result
     

-    def get_initializer(self, null_init):
-        '''Return literal expression for this field's default value.'''
-        
+    def get_dependencies(self):
+        '''Get list of type names used by this field.'''
+        if self.allocation == 'STATIC':
+            return [str(self.ctype)]
+        else:
+            return []
+
+    def get_initializer(self, null_init, inner_init_only = False):
+        '''Return literal expression for this field's default value.
+        null_init: If True, initialize to a 0 value instead of default from .proto
+        inner_init_only: If True, exclude initialization for any count/has fields
+        '''
+
+        inner_init = None

         if self.pbtype == 'MESSAGE':
             if null_init:
         if self.pbtype == 'MESSAGE':
             if null_init:

-                return '%s_init_zero' % self.ctype
+                inner_init = '%s_init_zero' % self.ctype

             else:
             else:

-                return '%s_init_default' % self.ctype
-        
-        if self.default is None or null_init:
+                inner_init = '%s_init_default' % self.ctype
+        elif self.default is None or null_init:

             if self.pbtype == 'STRING':
             if self.pbtype == 'STRING':

-                return '""'
+                inner_init = '""'

             elif self.pbtype == 'BYTES':
             elif self.pbtype == 'BYTES':

-                return '{0, {0}}'
-            elif self.pbtype == 'ENUM':
-                return '(%s)0' % self.ctype
+                inner_init = '{0, {0}}'
+            elif self.pbtype in ('ENUM', 'UENUM'):
+                inner_init = '(%s)0' % self.ctype

             else:
             else:

-                return '0'
-        
-        default = str(self.default)
+                inner_init = '0'
+        else:
+            if self.pbtype == 'STRING':
+                inner_init = self.default.replace('"', '\\"')
+                inner_init = '"' + inner_init + '"'
+            elif self.pbtype == 'BYTES':
+                data = ['0x%02x' % ord(c) for c in self.default]
+                if len(data) == 0:
+                    inner_init = '{0, {0}}'
+                else:
+                    inner_init = '{%d, {%s}}' % (len(data), ','.join(data))
+            elif self.pbtype in ['FIXED32', 'UINT32']:
+                inner_init = str(self.default) + 'u'
+            elif self.pbtype in ['FIXED64', 'UINT64']:
+                inner_init = str(self.default) + 'ull'
+            elif self.pbtype in ['SFIXED64', 'INT64']:
+                inner_init = str(self.default) + 'll'
+            else:
+                inner_init = str(self.default)

         
         

-        if self.pbtype == 'STRING':
-            default = default.encode('utf-8').encode('string_escape')
-            default = default.replace('"', '\\"')
-            default = '"' + default + '"'
-        elif self.pbtype == 'BYTES':
-            data = default.decode('string_escape')
-            data = ['0x%02x' % ord(c) for c in data]
-            if len(data) == 0:
-                default = '{0, {0}}'
+        if inner_init_only:
+            return inner_init
+
+        outer_init = None
+        if self.allocation == 'STATIC':
+            if self.rules == 'REPEATED':
+                outer_init = '0, {'
+                outer_init += ', '.join([inner_init] * self.max_count)
+                outer_init += '}'
+            elif self.rules == 'OPTIONAL':
+                outer_init = 'false, ' + inner_init

             else:
             else:

-                default = '{%d, {%s}}' % (len(data), ','.join(data))
-        elif self.pbtype in ['FIXED32', 'UINT32']:
-            default += 'u'
-        elif self.pbtype in ['FIXED64', 'UINT64']:
-            default += 'ull'
-        elif self.pbtype in ['SFIXED64', 'INT64']:
-            default += 'll'
-        
-        return default
-    
+                outer_init = inner_init
+        elif self.allocation == 'POINTER':
+            if self.rules == 'REPEATED':
+                outer_init = '0, NULL'
+            else:
+                outer_init = 'NULL'
+        elif self.allocation == 'CALLBACK':
+            if self.pbtype == 'EXTENSION':
+                outer_init = 'NULL'
+            else:
+                outer_init = '{{NULL}, NULL}'
+
+        return outer_init
+

     def default_decl(self, declaration_only = False):
         '''Return definition for this field's default value.'''
         if self.default is None:
             return None
 
         ctype = self.ctype
     def default_decl(self, declaration_only = False):
         '''Return definition for this field's default value.'''
         if self.default is None:
             return None
 
         ctype = self.ctype

-        default = self.get_initializer(False)
+        default = self.get_initializer(False, True)

         array_decl = ''
         
         if self.pbtype == 'STRING':
         array_decl = ''
         
         if self.pbtype == 'STRING':


@@ -361,7 +462,13 @@ class Field:
         '''Return the pb_field_t initializer to use in the constant array.
         prev_field_name is the name of the previous field or None.
         '''
         '''Return the pb_field_t initializer to use in the constant array.
         prev_field_name is the name of the previous field or None.
         '''

-        result  = '    PB_FIELD2(%3d, ' % self.tag
+
+        if self.rules == 'ONEOF':
+            result = '    PB_ONEOF_FIELD(%s, ' % self.union_name
+        else:
+            result = '    PB_FIELD('
+
+        result += '%3d, ' % self.tag

         result += '%-8s, ' % self.pbtype
         result += '%s, ' % self.rules
         result += '%-8s, ' % self.allocation
         result += '%-8s, ' % self.pbtype
         result += '%s, ' % self.rules
         result += '%-8s, ' % self.allocation


@@ -386,15 +493,20 @@ class Field:
     def largest_field_value(self):
         '''Determine if this field needs 16bit or 32bit pb_field_t structure to compile properly.
         Returns numeric value or a C-expression for assert.'''
     def largest_field_value(self):
         '''Determine if this field needs 16bit or 32bit pb_field_t structure to compile properly.
         Returns numeric value or a C-expression for assert.'''

+        check = []

         if self.pbtype == 'MESSAGE':
             if self.rules == 'REPEATED' and self.allocation == 'STATIC':
         if self.pbtype == 'MESSAGE':
             if self.rules == 'REPEATED' and self.allocation == 'STATIC':

-                return 'pb_membersize(%s, %s[0])' % (self.struct_name, self.name)
+                check.append('pb_membersize(%s, %s[0])' % (self.struct_name, self.name))
+            elif self.rules == 'ONEOF':
+                check.append('pb_membersize(%s, %s.%s)' % (self.struct_name, self.union_name, self.name))

             else:
             else:

-                return 'pb_membersize(%s, %s)' % (self.struct_name, self.name)
+                check.append('pb_membersize(%s, %s)' % (self.struct_name, self.name))

 
 

-        return max(self.tag, self.max_size, self.max_count)        
+        return FieldMaxSize([self.tag, self.max_size, self.max_count],
+                            check,
+                            ('%s.%s' % (self.struct_name, self.name)))

 
 

-    def encoded_size(self, allmsgs):
+    def encoded_size(self, dependencies):

         '''Return the maximum size that this field can take when encoded,
         including the field tag. If the size cannot be determined, returns
         None.'''
         '''Return the maximum size that this field can take when encoded,
         including the field tag. If the size cannot be determined, returns
         None.'''


@@ -403,18 +515,18 @@ class Field:
             return None
         
         if self.pbtype == 'MESSAGE':
             return None
         
         if self.pbtype == 'MESSAGE':

-            for msg in allmsgs:
-                if msg.name == self.submsgname:
-                    encsize = msg.encoded_size(allmsgs)
-                    if encsize is None:
-                        return None # Submessage size is indeterminate
-                        
-                    # Include submessage length prefix
-                    encsize += varint_max_size(encsize.upperlimit())
-                    break
+            if str(self.submsgname) in dependencies:
+                submsg = dependencies[str(self.submsgname)]
+                encsize = submsg.encoded_size(dependencies)
+                if encsize is None:
+                    return None # Submessage size is indeterminate
+                    
+                # Include submessage length prefix
+                encsize += varint_max_size(encsize.upperlimit())

             else:
                 # Submessage cannot be found, this currently occurs when
             else:
                 # Submessage cannot be found, this currently occurs when

-                # the submessage type is defined in a different file.
+                # the submessage type is defined in a different file and
+                # not using the protoc plugin.

                 # Instead of direct numeric value, reference the size that
                 # has been #defined in the other file.
                 encsize = EncodedSize(self.submsgname + 'size')
                 # Instead of direct numeric value, reference the size that
                 # has been #defined in the other file.
                 encsize = EncodedSize(self.submsgname + 'size')


@@ -423,6 +535,14 @@ class Field:
                 # prefix size, though.
                 encsize += 5
 
                 # prefix size, though.
                 encsize += 5
 

+        elif self.pbtype in ['ENUM', 'UENUM']:
+            if str(self.ctype) in dependencies:
+                enumtype = dependencies[str(self.ctype)]
+                encsize = enumtype.encoded_size()
+            else:
+                # Conservative assumption
+                encsize = 10
+

         elif self.enc_size is None:
             raise RuntimeError("Could not determine encoded size for %s.%s"
                                % (self.struct_name, self.name))
         elif self.enc_size is None:
             raise RuntimeError("Could not determine encoded size for %s.%s"
                                % (self.struct_name, self.name))


@@ -463,12 +583,12 @@ class ExtensionRange(Field):
         return '    pb_extension_t *extensions;'
     
     def types(self):
         return '    pb_extension_t *extensions;'
     
     def types(self):

-        return None
+        return ''

     
     def tags(self):
         return ''
     
     
     def tags(self):
         return ''
     

-    def encoded_size(self, allmsgs):
+    def encoded_size(self, dependencies):

         # We exclude extensions from the count, because they cannot be known
         # until runtime. Other option would be to return None here, but this
         # way the value remains useful if extensions are not used.
         # We exclude extensions from the count, because they cannot be known
         # until runtime. Other option would be to return None here, but this
         # way the value remains useful if extensions are not used.


@@ -498,7 +618,8 @@ class ExtensionField(Field):
             msg +='   type of extension fields is currently supported. */\n'
             return msg
         
             msg +='   type of extension fields is currently supported. */\n'
             return msg
         

-        return 'extern const pb_extension_type_t %s;\n' % self.fullname
+        return ('extern const pb_extension_type_t %s; /* field type: %s */\n' %
+            (self.fullname, str(self).strip()))

 
     def extension_def(self):
         '''Definition of the extension type in the .pb.c file'''
 
     def extension_def(self):
         '''Definition of the extension type in the .pb.c file'''


@@ -519,6 +640,85 @@ class ExtensionField(Field):
         return result
 
 
         return result
 
 

+# ---------------------------------------------------------------------------
+#                   Generation of oneofs (unions)
+# ---------------------------------------------------------------------------
+
+class OneOf(Field):
+    def __init__(self, struct_name, oneof_desc):
+        self.struct_name = struct_name
+        self.name = oneof_desc.name
+        self.ctype = 'union'
+        self.pbtype = 'oneof'
+        self.fields = []
+        self.allocation = 'ONEOF'
+        self.default = None
+        self.rules = 'ONEOF'
+
+    def add_field(self, field):
+        if field.allocation == 'CALLBACK':
+            raise Exception("Callback fields inside of oneof are not supported"
+                            + " (field %s)" % field.name)
+
+        field.union_name = self.name
+        field.rules = 'ONEOF'
+        self.fields.append(field)
+        self.fields.sort(key = lambda f: f.tag)
+
+        # Sort by the lowest tag number inside union
+        self.tag = min([f.tag for f in self.fields])
+
+    def __str__(self):
+        result = ''
+        if self.fields:
+            result += '    pb_size_t which_' + self.name + ";\n"
+            result += '    union {\n'
+            for f in self.fields:
+                result += '    ' + str(f).replace('\n', '\n    ') + '\n'
+            result += '    } ' + self.name + ';'
+        return result
+
+    def types(self):
+        return ''.join([f.types() for f in self.fields])
+
+    def get_dependencies(self):
+        deps = []
+        for f in self.fields:
+            deps += f.get_dependencies()
+        return deps
+
+    def get_initializer(self, null_init):
+        return '0, {' + self.fields[0].get_initializer(null_init) + '}'
+
+    def default_decl(self, declaration_only = False):
+        return None
+
+    def tags(self):
+        return '\n'.join([f.tags() for f in self.fields])
+
+    def pb_field_t(self, prev_field_name):
+        result = ',\n'.join([f.pb_field_t(prev_field_name) for f in self.fields])
+        return result
+
+    def largest_field_value(self):
+        largest = FieldMaxSize()
+        for f in self.fields:
+            largest.extend(f.largest_field_value())
+        return largest
+
+    def encoded_size(self, dependencies):
+        largest = EncodedSize(0)
+        for f in self.fields:
+            size = f.encoded_size(dependencies)
+            if size is None:
+                return None
+            elif size.symbols:
+                return None # Cannot resolve maximum of symbols
+            elif size.value > largest.value:
+                largest = size
+
+        return largest
+

 # ---------------------------------------------------------------------------
 #                   Generation of messages (structures)
 # ---------------------------------------------------------------------------
 # ---------------------------------------------------------------------------
 #                   Generation of messages (structures)
 # ---------------------------------------------------------------------------


@@ -528,11 +728,37 @@ class Message:
     def __init__(self, names, desc, message_options):
         self.name = names
         self.fields = []
     def __init__(self, names, desc, message_options):
         self.name = names
         self.fields = []

-        
+        self.oneofs = {}
+        no_unions = []
+
+        if message_options.msgid:
+            self.msgid = message_options.msgid
+
+        if hasattr(desc, 'oneof_decl'):
+            for i, f in enumerate(desc.oneof_decl):
+                oneof_options = get_nanopb_suboptions(desc, message_options, self.name + f.name)
+                if oneof_options.no_unions:
+                    no_unions.append(i) # No union, but add fields normally
+                elif oneof_options.type == nanopb_pb2.FT_IGNORE:
+                    pass # No union and skip fields also
+                else:
+                    oneof = OneOf(self.name, f)
+                    self.oneofs[i] = oneof
+                    self.fields.append(oneof)
+

         for f in desc.field:
             field_options = get_nanopb_suboptions(f, message_options, self.name + f.name)
         for f in desc.field:
             field_options = get_nanopb_suboptions(f, message_options, self.name + f.name)

-            if field_options.type != nanopb_pb2.FT_IGNORE:
-                self.fields.append(Field(self.name, f, field_options))
+            if field_options.type == nanopb_pb2.FT_IGNORE:
+                continue
+
+            field = Field(self.name, f, field_options)
+            if (hasattr(f, 'oneof_index') and
+                f.HasField('oneof_index') and
+                f.oneof_index not in no_unions):
+                if f.oneof_index in self.oneofs:
+                    self.oneofs[f.oneof_index].add_field(field)
+            else:
+                self.fields.append(field)

         
         if len(desc.extension_range) > 0:
             field_options = get_nanopb_suboptions(desc, message_options, self.name + 'extensions')
         
         if len(desc.extension_range) > 0:
             field_options = get_nanopb_suboptions(desc, message_options, self.name + 'extensions')


@@ -546,7 +772,10 @@ class Message:
 
     def get_dependencies(self):
         '''Get list of type names that this structure refers to.'''
 
     def get_dependencies(self):
         '''Get list of type names that this structure refers to.'''

-        return [str(field.ctype) for field in self.fields]
+        deps = []
+        for f in self.fields:
+            deps += f.get_dependencies()
+        return deps

     
     def __str__(self):
         result = 'typedef struct _%s {\n' % self.name
     
     def __str__(self):
         result = 'typedef struct _%s {\n' % self.name


@@ -571,37 +800,15 @@ class Message:
         return result
     
     def types(self):
         return result
     
     def types(self):

-        result = ""
-        for field in self.fields:
-            types = field.types()
-            if types is not None:
-                result += types + '\n'
-        return result
-    
+        return ''.join([f.types() for f in self.fields])
+

     def get_initializer(self, null_init):
         if not self.ordered_fields:
             return '{0}'
     
         parts = []
         for field in self.ordered_fields:
     def get_initializer(self, null_init):
         if not self.ordered_fields:
             return '{0}'
     
         parts = []
         for field in self.ordered_fields:

-            if field.allocation == 'STATIC':
-                if field.rules == 'REPEATED':
-                    parts.append('0')
-                    parts.append('{'
-                                 + ', '.join([field.get_initializer(null_init)] * field.max_count)
-                                 + '}')
-                elif field.rules == 'OPTIONAL':
-                    parts.append('false')
-                    parts.append(field.get_initializer(null_init))
-                else:
-                    parts.append(field.get_initializer(null_init))
-            elif field.allocation == 'POINTER':
-                parts.append('NULL')
-            elif field.allocation == 'CALLBACK':
-                if field.pbtype == 'EXTENSION':
-                    parts.append('NULL')
-                else:
-                    parts.append('{{NULL}, NULL}')
+            parts.append(field.get_initializer(null_init))

         return '{' + ', '.join(parts) + '}'
     
     def default_decl(self, declaration_only = False):
         return '{' + ', '.join(parts) + '}'
     
     def default_decl(self, declaration_only = False):


@@ -612,29 +819,50 @@ class Message:
                 result += default + '\n'
         return result
 
                 result += default + '\n'
         return result
 

+    def count_required_fields(self):
+        '''Returns number of required fields inside this message'''
+        count = 0
+        for f in self.fields:
+            if not isinstance(f, OneOf):
+                if f.rules == 'REQUIRED':
+                    count += 1
+        return count
+
+    def count_all_fields(self):
+        count = 0
+        for f in self.fields:
+            if isinstance(f, OneOf):
+                count += len(f.fields)
+            else:
+                count += 1
+        return count
+

     def fields_declaration(self):
     def fields_declaration(self):

-        result = 'extern const pb_field_t %s_fields[%d];' % (self.name, len(self.fields) + 1)
+        result = 'extern const pb_field_t %s_fields[%d];' % (self.name, self.count_all_fields() + 1)

         return result
 
     def fields_definition(self):
         return result
 
     def fields_definition(self):

-        result = 'const pb_field_t %s_fields[%d] = {\n' % (self.name, len(self.fields) + 1)
+        result = 'const pb_field_t %s_fields[%d] = {\n' % (self.name, self.count_all_fields() + 1)

         
         prev = None
         for field in self.ordered_fields:
             result += field.pb_field_t(prev)
             result += ',\n'
         
         prev = None
         for field in self.ordered_fields:
             result += field.pb_field_t(prev)
             result += ',\n'

-            prev = field.name
+            if isinstance(field, OneOf):
+                prev = field.name + '.' + field.fields[-1].name
+            else:
+                prev = field.name

         
         result += '    PB_LAST_FIELD\n};'
         return result
 
         
         result += '    PB_LAST_FIELD\n};'
         return result
 

-    def encoded_size(self, allmsgs):
+    def encoded_size(self, dependencies):

         '''Return the maximum size that this message can take when encoded.
         If the size cannot be determined, returns None.
         '''
         size = EncodedSize(0)
         for field in self.fields:
         '''Return the maximum size that this message can take when encoded.
         If the size cannot be determined, returns None.
         '''
         size = EncodedSize(0)
         for field in self.fields:

-            fsize = field.encoded_size(allmsgs)
+            fsize = field.encoded_size(dependencies)

             if fsize is None:
                 return None
             size += fsize
             if fsize is None:
                 return None
             size += fsize


@@ -646,7 +874,6 @@ class Message:
 #                    Processing of entire .proto files
 # ---------------------------------------------------------------------------
 
 #                    Processing of entire .proto files
 # ---------------------------------------------------------------------------
 

-

 def iterate_messages(desc, names = Names()):
     '''Recursively find all messages. For each, yield name, DescriptorProto.'''
     if hasattr(desc, 'message_type'):
 def iterate_messages(desc, names = Names()):
     '''Recursively find all messages. For each, yield name, DescriptorProto.'''
     if hasattr(desc, 'message_type'):


@@ -672,65 +899,22 @@ def iterate_extensions(desc, names = Names()):
         for extension in subdesc.extension:
             yield subname, extension
 
         for extension in subdesc.extension:
             yield subname, extension
 

-def parse_file(fdesc, file_options):
-    '''Takes a FileDescriptorProto and returns tuple (enums, messages, extensions).'''
-    
-    enums = []
-    messages = []
-    extensions = []
-    
-    if fdesc.package:
-        base_name = Names(fdesc.package.split('.'))
-    else:
-        base_name = Names()
-    
-    for enum in fdesc.enum_type:
-        enum_options = get_nanopb_suboptions(enum, file_options, base_name + enum.name)
-        enums.append(Enum(base_name, enum, enum_options))
-    
-    for names, message in iterate_messages(fdesc, base_name):
-        message_options = get_nanopb_suboptions(message, file_options, names)
-        
-        if message_options.skip_message:
-            continue
-        
-        messages.append(Message(names, message, message_options))
-        for enum in message.enum_type:
-            enum_options = get_nanopb_suboptions(enum, message_options, names + enum.name)
-            enums.append(Enum(names, enum, enum_options))
-    
-    for names, extension in iterate_extensions(fdesc, base_name):
-        field_options = get_nanopb_suboptions(extension, file_options, names + extension.name)
-        if field_options.type != nanopb_pb2.FT_IGNORE:
-            extensions.append(ExtensionField(names, extension, field_options))
-    
-    # Fix field default values where enum short names are used.
-    for enum in enums:
-        if not enum.options.long_names:
-            for message in messages:
-                for field in message.fields:
-                    if field.default in enum.value_longnames:
-                        idx = enum.value_longnames.index(field.default)
-                        field.default = enum.values[idx][0]
-    
-    return enums, messages, extensions
-

 def toposort2(data):
     '''Topological sort.
     From http://code.activestate.com/recipes/577413-topological-sort/
     This function is under the MIT license.
     '''
 def toposort2(data):
     '''Topological sort.
     From http://code.activestate.com/recipes/577413-topological-sort/
     This function is under the MIT license.
     '''

-    for k, v in data.items():
+    for k, v in list(data.items()):

         v.discard(k) # Ignore self dependencies
         v.discard(k) # Ignore self dependencies

-    extra_items_in_deps = reduce(set.union, data.values(), set()) - set(data.keys())
+    extra_items_in_deps = reduce(set.union, list(data.values()), set()) - set(data.keys())

     data.update(dict([(item, set()) for item in extra_items_in_deps]))
     while True:
     data.update(dict([(item, set()) for item in extra_items_in_deps]))
     while True:

-        ordered = set(item for item,dep in data.items() if not dep)
+        ordered = set(item for item,dep in list(data.items()) if not dep)

         if not ordered:
             break
         for item in sorted(ordered):
             yield item
         if not ordered:
             break
         for item in sorted(ordered):
             yield item

-        data = dict([(item, (dep - ordered)) for item,dep in data.items()
+        data = dict([(item, (dep - ordered)) for item,dep in list(data.items())

                 if item not in ordered])
     assert not data, "A cyclic dependency exists amongst %r" % data
 
                 if item not in ordered])
     assert not data, "A cyclic dependency exists amongst %r" % data
 


@@ -756,194 +940,296 @@ def make_identifier(headername):
             result += '_'
     return result
 
             result += '_'
     return result
 

-def generate_header(dependencies, headername, enums, messages, extensions, options):
-    '''Generate content for a header file.
-    Generates strings, which should be concatenated and stored to file.
-    '''
-    
-    yield '/* Automatically generated nanopb header */\n'
-    if options.notimestamp:
-        yield '/* Generated by %s */\n\n' % (nanopb_version)
-    else:
-        yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime())
+class ProtoFile:
+    def __init__(self, fdesc, file_options):
+        '''Takes a FileDescriptorProto and parses it.'''
+        self.fdesc = fdesc
+        self.file_options = file_options
+        self.dependencies = {}
+        self.parse()
+        
+        # Some of types used in this file probably come from the file itself.
+        # Thus it has implicit dependency on itself.
+        self.add_dependency(self)
+
+    def parse(self):
+        self.enums = []
+        self.messages = []
+        self.extensions = []
+        
+        if self.fdesc.package:
+            base_name = Names(self.fdesc.package.split('.'))
+        else:
+            base_name = Names()

     
     

-    symbol = make_identifier(headername)
-    yield '#ifndef PB_%s_INCLUDED\n' % symbol
-    yield '#define PB_%s_INCLUDED\n' % symbol
-    try:
-        yield options.libformat % ('pb.h')
-    except TypeError:
-        # no %s specified - use whatever was passed in as options.libformat
-        yield options.libformat
-    yield '\n'
+        for enum in self.fdesc.enum_type:
+            enum_options = get_nanopb_suboptions(enum, self.file_options, base_name + enum.name)
+            self.enums.append(Enum(base_name, enum, enum_options))
+        
+        for names, message in iterate_messages(self.fdesc, base_name):
+            message_options = get_nanopb_suboptions(message, self.file_options, names)
+            
+            if message_options.skip_message:
+                continue
+       
+            self.messages.append(Message(names, message, message_options))
+            for enum in message.enum_type:
+                enum_options = get_nanopb_suboptions(enum, message_options, names + enum.name)
+                self.enums.append(Enum(names, enum, enum_options))
+        
+        for names, extension in iterate_extensions(self.fdesc, base_name):
+            field_options = get_nanopb_suboptions(extension, self.file_options, names + extension.name)
+            if field_options.type != nanopb_pb2.FT_IGNORE:
+                self.extensions.append(ExtensionField(names, extension, field_options))

     
     

-    for dependency in dependencies:
-        noext = os.path.splitext(dependency)[0]
-        yield options.genformat % (noext + options.extension + '.h')
+    def add_dependency(self, other):
+        for enum in other.enums:
+            self.dependencies[str(enum.names)] = enum
+        
+        for msg in other.messages:
+            self.dependencies[str(msg.name)] = msg
+        
+        # Fix field default values where enum short names are used.
+        for enum in other.enums:
+            if not enum.options.long_names:
+                for message in self.messages:
+                    for field in message.fields:
+                        if field.default in enum.value_longnames:
+                            idx = enum.value_longnames.index(field.default)
+                            field.default = enum.values[idx][0]
+        
+        # Fix field data types where enums have negative values.
+        for enum in other.enums:
+            if not enum.has_negative():
+                for message in self.messages:
+                    for field in message.fields:
+                        if field.pbtype == 'ENUM' and field.ctype == enum.names:
+                            field.pbtype = 'UENUM'
+
+    def generate_header(self, includes, headername, options):
+        '''Generate content for a header file.
+        Generates strings, which should be concatenated and stored to file.
+        '''
+        
+        yield '/* Automatically generated nanopb header */\n'
+        if options.notimestamp:
+            yield '/* Generated by %s */\n\n' % (nanopb_version)
+        else:
+            yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime())
+        
+        symbol = make_identifier(headername)
+        yield '#ifndef PB_%s_INCLUDED\n' % symbol
+        yield '#define PB_%s_INCLUDED\n' % symbol
+        try:
+            yield options.libformat % ('pb.h')
+        except TypeError:
+            # no %s specified - use whatever was passed in as options.libformat
+            yield options.libformat

         yield '\n'
         yield '\n'

-
-    yield '#ifdef __cplusplus\n'
-    yield 'extern "C" {\n'
-    yield '#endif\n\n'
-    
-    yield '/* Enum definitions */\n'
-    for enum in enums:
-        yield str(enum) + '\n\n'
-    
-    yield '/* Struct definitions */\n'
-    for msg in sort_dependencies(messages):
-        yield msg.types()
-        yield str(msg) + '\n\n'
-    
-    if extensions:
-        yield '/* Extensions */\n'
-        for extension in extensions:
-            yield extension.extension_decl()
+        
+        for incfile in includes:
+            noext = os.path.splitext(incfile)[0]
+            yield options.genformat % (noext + options.extension + '.h')
+            yield '\n'
+
+        yield '#if PB_PROTO_HEADER_VERSION != 30\n'
+        yield '#error Regenerate this file with the current version of nanopb generator.\n'
+        yield '#endif\n'

         yield '\n'
         yield '\n'

+
+        yield '#ifdef __cplusplus\n'
+        yield 'extern "C" {\n'
+        yield '#endif\n\n'

         
         

-    yield '/* Default values for struct fields */\n'
-    for msg in messages:
-        yield msg.default_decl(True)
-    yield '\n'
-    
-    yield '/* Initializer values for message structs */\n'
-    for msg in messages:
-        identifier = '%s_init_default' % msg.name
-        yield '#define %-40s %s\n' % (identifier, msg.get_initializer(False))
-    for msg in messages:
-        identifier = '%s_init_zero' % msg.name
-        yield '#define %-40s %s\n' % (identifier, msg.get_initializer(True))
-    yield '\n'
-    
-    yield '/* Field tags (for use in manual encoding/decoding) */\n'
-    for msg in sort_dependencies(messages):
-        for field in msg.fields:
-            yield field.tags()
-    for extension in extensions:
-        yield extension.tags()
-    yield '\n'
-    
-    yield '/* Struct field encoding specification for nanopb */\n'
-    for msg in messages:
-        yield msg.fields_declaration() + '\n'
-    yield '\n'
-    
-    yield '/* Maximum encoded size of messages (where known) */\n'
-    for msg in messages:
-        msize = msg.encoded_size(messages)
-        if msize is not None:
-            identifier = '%s_size' % msg.name
-            yield '#define %-40s %s\n' % (identifier, msize)
-    yield '\n'
-    
-    yield '#ifdef __cplusplus\n'
-    yield '} /* extern "C" */\n'
-    yield '#endif\n'
-    
-    # End of header
-    yield '\n#endif\n'
+        if self.enums:
+            yield '/* Enum definitions */\n'
+            for enum in self.enums:
+                yield str(enum) + '\n\n'
+        
+        if self.messages:
+            yield '/* Struct definitions */\n'
+            for msg in sort_dependencies(self.messages):
+                yield msg.types()
+                yield str(msg) + '\n\n'
+        
+        if self.extensions:
+            yield '/* Extensions */\n'
+            for extension in self.extensions:
+                yield extension.extension_decl()
+            yield '\n'
+        
+        if self.messages:
+            yield '/* Default values for struct fields */\n'
+            for msg in self.messages:
+                yield msg.default_decl(True)
+            yield '\n'
+        
+            yield '/* Initializer values for message structs */\n'
+            for msg in self.messages:
+                identifier = '%s_init_default' % msg.name
+                yield '#define %-40s %s\n' % (identifier, msg.get_initializer(False))
+            for msg in self.messages:
+                identifier = '%s_init_zero' % msg.name
+                yield '#define %-40s %s\n' % (identifier, msg.get_initializer(True))
+            yield '\n'
+        
+            yield '/* Field tags (for use in manual encoding/decoding) */\n'
+            for msg in sort_dependencies(self.messages):
+                for field in msg.fields:
+                    yield field.tags()
+            for extension in self.extensions:
+                yield extension.tags()
+            yield '\n'
+        
+            yield '/* Struct field encoding specification for nanopb */\n'
+            for msg in self.messages:
+                yield msg.fields_declaration() + '\n'
+            yield '\n'
+        
+            yield '/* Maximum encoded size of messages (where known) */\n'
+            for msg in self.messages:
+                msize = msg.encoded_size(self.dependencies)
+                if msize is not None:
+                    identifier = '%s_size' % msg.name
+                    yield '#define %-40s %s\n' % (identifier, msize)
+            yield '\n'
+
+            yield '/* Message IDs (where set with "msgid" option) */\n'
+            
+            yield '#ifdef PB_MSGID\n'
+            for msg in self.messages:
+                if hasattr(msg,'msgid'):
+                    yield '#define PB_MSG_%d %s\n' % (msg.msgid, msg.name)
+            yield '\n'
+
+            symbol = make_identifier(headername.split('.')[0])
+            yield '#define %s_MESSAGES \\\n' % symbol
+
+            for msg in self.messages:
+                m = "-1"
+                msize = msg.encoded_size(self.dependencies)
+                if msize is not None:
+                    m = msize
+                if hasattr(msg,'msgid'):
+                    yield '\tPB_MSG(%d,%s,%s) \\\n' % (msg.msgid, m, msg.name)
+            yield '\n'
+
+            for msg in self.messages:
+                if hasattr(msg,'msgid'):
+                    yield '#define %s_msgid %d\n' % (msg.name, msg.msgid)
+            yield '\n'

 
 

-def generate_source(headername, enums, messages, extensions, options):
-    '''Generate content for a source file.'''
-    
-    yield '/* Automatically generated nanopb constant definitions */\n'
-    if options.notimestamp:
-        yield '/* Generated by %s */\n\n' % (nanopb_version)
-    else:
-        yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime())
-    yield options.genformat % (headername)
-    yield '\n'
-    
-    for msg in messages:
-        yield msg.default_decl(False)
-    
-    yield '\n\n'
-    
-    for msg in messages:
-        yield msg.fields_definition() + '\n\n'
-    
-    for ext in extensions:
-        yield ext.extension_def() + '\n'
-        
-    # Add checks for numeric limits
-    if messages:
-        count_required_fields = lambda m: len([f for f in msg.fields if f.rules == 'REQUIRED'])
-        largest_msg = max(messages, key = count_required_fields)
-        largest_count = count_required_fields(largest_msg)
-        if largest_count > 64:
-            yield '\n/* Check that missing required fields will be properly detected */\n'
-            yield '#if PB_MAX_REQUIRED_FIELDS < %d\n' % largest_count
-            yield '#error Properly detecting missing required fields in %s requires \\\n' % largest_msg.name
-            yield '       setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count
-            yield '#endif\n'
-    
-    worst = 0
-    worst_field = ''
-    checks = []
-    checks_msgnames = []
-    for msg in messages:
-        checks_msgnames.append(msg.name)
-        for field in msg.fields:
-            status = field.largest_field_value()
-            if isinstance(status, (str, unicode)):
-                checks.append(status)
-            elif status > worst:
-                worst = status
-                worst_field = str(field.struct_name) + '.' + str(field.name)
-
-    if worst > 255 or checks:
-        yield '\n/* Check that field information fits in pb_field_t */\n'
-        
-        if worst > 65535 or checks:
-            yield '#if !defined(PB_FIELD_32BIT)\n'
-            if worst > 65535:
-                yield '#error Field descriptor for %s is too large. Define PB_FIELD_32BIT to fix this.\n' % worst_field
-            else:
-                assertion = ' && '.join(str(c) + ' < 65536' for c in checks)
-                msgs = '_'.join(str(n) for n in checks_msgnames)
-                yield '/* If you get an error here, it means that you need to define PB_FIELD_32BIT\n'
-                yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n'
-                yield ' * \n'
-                yield ' * The reason you need to do this is that some of your messages contain tag\n'
-                yield ' * numbers or field sizes that are larger than what can fit in 8 or 16 bit\n'
-                yield ' * field descriptors.\n'
-                yield ' */\n'
-                yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs)

             yield '#endif\n\n'
             yield '#endif\n\n'

+
+        yield '#ifdef __cplusplus\n'
+        yield '} /* extern "C" */\n'
+        yield '#endif\n'
+        
+        # End of header
+        yield '\n#endif\n'
+
+    def generate_source(self, headername, options):
+        '''Generate content for a source file.'''
+        
+        yield '/* Automatically generated nanopb constant definitions */\n'
+        if options.notimestamp:
+            yield '/* Generated by %s */\n\n' % (nanopb_version)
+        else:
+            yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime())
+        yield options.genformat % (headername)
+        yield '\n'
+        
+        yield '#if PB_PROTO_HEADER_VERSION != 30\n'
+        yield '#error Regenerate this file with the current version of nanopb generator.\n'
+        yield '#endif\n'
+        yield '\n'
+        
+        for msg in self.messages:
+            yield msg.default_decl(False)
+        
+        yield '\n\n'
+        
+        for msg in self.messages:
+            yield msg.fields_definition() + '\n\n'
+        
+        for ext in self.extensions:
+            yield ext.extension_def() + '\n'
+            
+        # Add checks for numeric limits
+        if self.messages:
+            largest_msg = max(self.messages, key = lambda m: m.count_required_fields())
+            largest_count = largest_msg.count_required_fields()
+            if largest_count > 64:
+                yield '\n/* Check that missing required fields will be properly detected */\n'
+                yield '#if PB_MAX_REQUIRED_FIELDS < %d\n' % largest_count
+                yield '#error Properly detecting missing required fields in %s requires \\\n' % largest_msg.name
+                yield '       setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count
+                yield '#endif\n'
+
+        max_field = FieldMaxSize()
+        checks_msgnames = []
+        for msg in self.messages:
+            checks_msgnames.append(msg.name)
+            for field in msg.fields:
+                max_field.extend(field.largest_field_value())
+
+        worst = max_field.worst
+        worst_field = max_field.worst_field
+        checks = max_field.checks
+
+        if worst > 255 or checks:
+            yield '\n/* Check that field information fits in pb_field_t */\n'
+            
+            if worst > 65535 or checks:
+                yield '#if !defined(PB_FIELD_32BIT)\n'
+                if worst > 65535:
+                    yield '#error Field descriptor for %s is too large. Define PB_FIELD_32BIT to fix this.\n' % worst_field
+                else:
+                    assertion = ' && '.join(str(c) + ' < 65536' for c in checks)
+                    msgs = '_'.join(str(n) for n in checks_msgnames)
+                    yield '/* If you get an error here, it means that you need to define PB_FIELD_32BIT\n'
+                    yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n'
+                    yield ' * \n'
+                    yield ' * The reason you need to do this is that some of your messages contain tag\n'
+                    yield ' * numbers or field sizes that are larger than what can fit in 8 or 16 bit\n'
+                    yield ' * field descriptors.\n'
+                    yield ' */\n'
+                    yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs)
+                yield '#endif\n\n'
+            
+            if worst < 65536:
+                yield '#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)\n'
+                if worst > 255:
+                    yield '#error Field descriptor for %s is too large. Define PB_FIELD_16BIT to fix this.\n' % worst_field
+                else:
+                    assertion = ' && '.join(str(c) + ' < 256' for c in checks)
+                    msgs = '_'.join(str(n) for n in checks_msgnames)
+                    yield '/* If you get an error here, it means that you need to define PB_FIELD_16BIT\n'
+                    yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n'
+                    yield ' * \n'
+                    yield ' * The reason you need to do this is that some of your messages contain tag\n'
+                    yield ' * numbers or field sizes that are larger than what can fit in the default\n'
+                    yield ' * 8 bit descriptors.\n'
+                    yield ' */\n'
+                    yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs)
+                yield '#endif\n\n'
+        
+        # Add check for sizeof(double)
+        has_double = False
+        for msg in self.messages:
+            for field in msg.fields:
+                if field.ctype == 'double':
+                    has_double = True
+        
+        if has_double:
+            yield '\n'
+            yield '/* On some platforms (such as AVR), double is really float.\n'
+            yield ' * These are not directly supported by nanopb, but see example_avr_double.\n'
+            yield ' * To get rid of this error, remove any double fields from your .proto.\n'
+            yield ' */\n'
+            yield 'PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)\n'

         
         

-        if worst < 65536:
-            yield '#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)\n'
-            if worst > 255:
-                yield '#error Field descriptor for %s is too large. Define PB_FIELD_16BIT to fix this.\n' % worst_field
-            else:
-                assertion = ' && '.join(str(c) + ' < 256' for c in checks)
-                msgs = '_'.join(str(n) for n in checks_msgnames)
-                yield '/* If you get an error here, it means that you need to define PB_FIELD_16BIT\n'
-                yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n'
-                yield ' * \n'
-                yield ' * The reason you need to do this is that some of your messages contain tag\n'
-                yield ' * numbers or field sizes that are larger than what can fit in the default\n'
-                yield ' * 8 bit descriptors.\n'
-                yield ' */\n'
-                yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs)
-            yield '#endif\n\n'
-    
-    # Add check for sizeof(double)
-    has_double = False
-    for msg in messages:
-        for field in msg.fields:
-            if field.ctype == 'double':
-                has_double = True
-    
-    if has_double:

         yield '\n'
         yield '\n'

-        yield '/* On some platforms (such as AVR), double is really float.\n'
-        yield ' * These are not directly supported by nanopb, but see example_avr_double.\n'
-        yield ' * To get rid of this error, remove any double fields from your .proto.\n'
-        yield ' */\n'
-        yield 'PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)\n'
-    
-    yield '\n'

 
 # ---------------------------------------------------------------------------
 #                    Options parsing for the .proto files
 
 # ---------------------------------------------------------------------------
 #                    Options parsing for the .proto files


@@ -956,14 +1242,32 @@ def read_options_file(infile):
         [(namemask, options), ...]
     '''
     results = []
         [(namemask, options), ...]
     '''
     results = []

-    for line in infile:
+    data = infile.read()
+    data = re.sub('/\*.*?\*/', '', data, flags = re.MULTILINE)
+    data = re.sub('//.*?$', '', data, flags = re.MULTILINE)
+    data = re.sub('#.*?$', '', data, flags = re.MULTILINE)
+    for i, line in enumerate(data.split('\n')):

         line = line.strip()
         line = line.strip()

-        if not line or line.startswith('//') or line.startswith('#'):
+        if not line:

             continue
         
         parts = line.split(None, 1)
             continue
         
         parts = line.split(None, 1)

+        
+        if len(parts) < 2:
+            sys.stderr.write("%s:%d: " % (infile.name, i + 1) +
+                             "Option lines should have space between field name and options. " +
+                             "Skipping line: '%s'\n" % line)
+            continue
+        

         opts = nanopb_pb2.NanoPBOptions()
         opts = nanopb_pb2.NanoPBOptions()

-        text_format.Merge(parts[1], opts)
+        
+        try:
+            text_format.Merge(parts[1], opts)
+        except Exception as e:
+            sys.stderr.write("%s:%d: " % (infile.name, i + 1) +
+                             "Unparseable option line: '%s'. " % line +
+                             "Error: %s\n" % str(e))
+            continue

         results.append((parts[0], opts))
 
     return results
         results.append((parts[0], opts))
 
     return results


@@ -1027,6 +1331,9 @@ optparser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION",
     help="Set extension to use instead of '.pb' for generated files. [default: %default]")
 optparser.add_option("-f", "--options-file", dest="options_file", metavar="FILE", default="%s.options",
     help="Set name of a separate generator options file.")
     help="Set extension to use instead of '.pb' for generated files. [default: %default]")
 optparser.add_option("-f", "--options-file", dest="options_file", metavar="FILE", default="%s.options",
     help="Set name of a separate generator options file.")

+optparser.add_option("-I", "--options-path", dest="options_path", metavar="DIR",
+    action="append", default = [],
+    help="Search for .options files additionally in this path")

 optparser.add_option("-Q", "--generated-include-format", dest="genformat",
     metavar="FORMAT", default='#include "%s"\n',
     help="Set format string to use for including other .pb.h files. [default: %default]")
 optparser.add_option("-Q", "--generated-include-format", dest="genformat",
     metavar="FORMAT", default='#include "%s"\n',
     help="Set format string to use for including other .pb.h files. [default: %default]")


@@ -1042,19 +1349,8 @@ optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", def
 optparser.add_option("-s", dest="settings", metavar="OPTION:VALUE", action="append", default=[],
     help="Set generator option (max_size, max_count etc.).")
 
 optparser.add_option("-s", dest="settings", metavar="OPTION:VALUE", action="append", default=[],
     help="Set generator option (max_size, max_count etc.).")
 

-def process_file(filename, fdesc, options):
-    '''Process a single file.
-    filename: The full path to the .proto or .pb source file, as string.
-    fdesc: The loaded FileDescriptorSet, or None to read from the input file.
-    options: Command line options as they come from OptionsParser.
-    
-    Returns a dict:
-        {'headername': Name of header file,
-         'headerdata': Data for the .h header file,
-         'sourcename': Name of the source code file,
-         'sourcedata': Data for the .c source code file
-        }
-    '''
+def parse_file(filename, fdesc, options):
+    '''Parse a single file. Returns a ProtoFile instance.'''

     toplevel_options = nanopb_pb2.NanoPBOptions()
     for s in options.settings:
         text_format.Merge(s, toplevel_options)
     toplevel_options = nanopb_pb2.NanoPBOptions()
     for s in options.settings:
         text_format.Merge(s, toplevel_options)


@@ -1072,25 +1368,50 @@ def process_file(filename, fdesc, options):
         optfilename = options.options_file
         had_abspath = True
 
         optfilename = options.options_file
         had_abspath = True
 

-    if os.path.isfile(optfilename):
-        if options.verbose:
-            sys.stderr.write('Reading options from ' + optfilename + '\n')
-
-        Globals.separate_options = read_options_file(open(optfilename, "rU"))
+    paths = ['.'] + options.options_path
+    for p in paths:
+        if os.path.isfile(os.path.join(p, optfilename)):
+            optfilename = os.path.join(p, optfilename)
+            if options.verbose:
+                sys.stderr.write('Reading options from ' + optfilename + '\n')
+            Globals.separate_options = read_options_file(open(optfilename, "rU"))
+            break

     else:
         # If we are given a full filename and it does not exist, give an error.
         # However, don't give error when we automatically look for .options file
         # with the same name as .proto.
         if options.verbose or had_abspath:
     else:
         # If we are given a full filename and it does not exist, give an error.
         # However, don't give error when we automatically look for .options file
         # with the same name as .proto.
         if options.verbose or had_abspath:

-            sys.stderr.write('Options file not found: ' + optfilename)
-
+            sys.stderr.write('Options file not found: ' + optfilename + '\n')

         Globals.separate_options = []
 
     Globals.matched_namemasks = set()
     
     # Parse the file
     file_options = get_nanopb_suboptions(fdesc, toplevel_options, Names([filename]))
         Globals.separate_options = []
 
     Globals.matched_namemasks = set()
     
     # Parse the file
     file_options = get_nanopb_suboptions(fdesc, toplevel_options, Names([filename]))

-    enums, messages, extensions = parse_file(fdesc, file_options)
+    f = ProtoFile(fdesc, file_options)
+    f.optfilename = optfilename
+    
+    return f
+
+def process_file(filename, fdesc, options, other_files = {}):
+    '''Process a single file.
+    filename: The full path to the .proto or .pb source file, as string.
+    fdesc: The loaded FileDescriptorSet, or None to read from the input file.
+    options: Command line options as they come from OptionsParser.
+    
+    Returns a dict:
+        {'headername': Name of header file,
+         'headerdata': Data for the .h header file,
+         'sourcename': Name of the source code file,
+         'sourcedata': Data for the .c source code file
+        }
+    '''
+    f = parse_file(filename, fdesc, options)
+
+    # Provide dependencies if available
+    for dep in f.fdesc.dependency:
+        if dep in other_files:
+            f.add_dependency(other_files[dep])

 
     # Decide the file names
     noext = os.path.splitext(filename)[0]
 
     # Decide the file names
     noext = os.path.splitext(filename)[0]


@@ -1101,18 +1422,15 @@ def process_file(filename, fdesc, options):
     # List of .proto files that should not be included in the C header file
     # even if they are mentioned in the source .proto.
     excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude
     # List of .proto files that should not be included in the C header file
     # even if they are mentioned in the source .proto.
     excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude

-    dependencies = [d for d in fdesc.dependency if d not in excludes]
+    includes = [d for d in f.fdesc.dependency if d not in excludes]

     
     

-    headerdata = ''.join(generate_header(dependencies, headerbasename, enums,
-                                         messages, extensions, options))
-
-    sourcedata = ''.join(generate_source(headerbasename, enums,
-                                         messages, extensions, options))
+    headerdata = ''.join(f.generate_header(includes, headerbasename, options))
+    sourcedata = ''.join(f.generate_source(headerbasename, options))

 
     # Check if there were any lines in .options that did not match a member
     unmatched = [n for n,o in Globals.separate_options if n not in Globals.matched_namemasks]
     if unmatched and not options.quiet:
 
     # Check if there were any lines in .options that did not match a member
     unmatched = [n for n,o in Globals.separate_options if n not in Globals.matched_namemasks]
     if unmatched and not options.quiet:

-        sys.stderr.write("Following patterns in " + optfilename + " did not match any fields: "
+        sys.stderr.write("Following patterns in " + f.optfilename + " did not match any fields: "

                          + ', '.join(unmatched) + "\n")
         if not Globals.verbose_options:
             sys.stderr.write("Use  protoc --nanopb-out=-v:.   to see a list of the field names.\n")
                          + ', '.join(unmatched) + "\n")
         if not Globals.verbose_options:
             sys.stderr.write("Use  protoc --nanopb-out=-v:.   to see a list of the field names.\n")


@@ -1147,28 +1465,48 @@ def main_cli():
 def main_plugin():
     '''Main function when invoked as a protoc plugin.'''
 
 def main_plugin():
     '''Main function when invoked as a protoc plugin.'''
 

-    import sys
+    import io, sys

     if sys.platform == "win32":
         import os, msvcrt
         # Set stdin and stdout to binary mode
         msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
         msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
     
     if sys.platform == "win32":
         import os, msvcrt
         # Set stdin and stdout to binary mode
         msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
         msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
     

-    data = sys.stdin.read()
+    data = io.open(sys.stdin.fileno(), "rb").read()
+

     request = plugin_pb2.CodeGeneratorRequest.FromString(data)
     
     request = plugin_pb2.CodeGeneratorRequest.FromString(data)
     

+    try:
+        # Versions of Python prior to 2.7.3 do not support unicode
+        # input to shlex.split(). Try to convert to str if possible.
+        params = str(request.parameter)
+    except UnicodeEncodeError:
+        params = request.parameter
+    

     import shlex
     import shlex

-    args = shlex.split(request.parameter)
+    args = shlex.split(params)

     options, dummy = optparser.parse_args(args)
     
     Globals.verbose_options = options.verbose
     
     response = plugin_pb2.CodeGeneratorResponse()
     
     options, dummy = optparser.parse_args(args)
     
     Globals.verbose_options = options.verbose
     
     response = plugin_pb2.CodeGeneratorResponse()
     

+    # Google's protoc does not currently indicate the full path of proto files.
+    # Instead always add the main file path to the search dirs, that works for
+    # the common case.
+    import os.path
+    options.options_path.append(os.path.dirname(request.file_to_generate[0]))
+    
+    # Process any include files first, in order to have them
+    # available as dependencies
+    other_files = {}
+    for fdesc in request.proto_file:
+        other_files[fdesc.name] = parse_file(fdesc.name, fdesc, options)
+    

     for filename in request.file_to_generate:
         for fdesc in request.proto_file:
             if fdesc.name == filename:
     for filename in request.file_to_generate:
         for fdesc in request.proto_file:
             if fdesc.name == filename:

-                results = process_file(filename, fdesc, options)
+                results = process_file(filename, fdesc, options, other_files)

                 
                 f = response.file.add()
                 f.name = results['headername']
                 
                 f = response.file.add()
                 f.name = results['headername']


@@ -1178,7 +1516,7 @@ def main_plugin():
                 f.name = results['sourcename']
                 f.content = results['sourcedata']    
     
                 f.name = results['sourcename']
                 f.content = results['sourcedata']    
     

-    sys.stdout.write(response.SerializeToString())
+    io.open(sys.stdout.fileno(), "wb").write(response.SerializeToString())

 
 if __name__ == '__main__':
     # Check if we are running as a plugin under protoc
 
 if __name__ == '__main__':
     # Check if we are running as a plugin under protoc