-Fix deserializing empty BSON structures

-Fix error message when attempting to deserialize custom dictionary with no default constructor
-Fix serializing private base members marked with JsonProperty attribute
-Fix performance issue when deserializing to object member

Src/Newtonsoft.Json.Tests/Bson/BsonReaderTests.cs

@@ -596,5 +596,55 @@ namespace Newtonsoft.Json.Tests.Bson
 
       Assert.IsFalse(reader.Read());
     }
+
+    [Test]
+    public void WriteAndReadEmptyListsAndDictionaries()
+    {
+      MemoryStream ms = new MemoryStream();
+      BsonWriter writer = new BsonWriter(ms);
+
+      writer.WriteStartObject();
+      writer.WritePropertyName("Arguments");
+      writer.WriteStartObject();
+      writer.WriteEndObject();
+      writer.WritePropertyName("List");
+      writer.WriteStartArray();
+      writer.WriteEndArray();
+      writer.WriteEndObject();
+
+      string bson = BitConverter.ToString(ms.ToArray());
+
+      Assert.AreEqual("20-00-00-00-03-41-72-67-75-6D-65-6E-74-73-00-05-00-00-00-00-04-4C-69-73-74-00-05-00-00-00-00-00", bson);
+
+      BsonReader reader = new BsonReader(new MemoryStream(MiscellaneousUtils.HexToBytes(bson)));
+
+      Assert.IsTrue(reader.Read());
+      Assert.AreEqual(JsonToken.StartObject, reader.TokenType);
+
+      Assert.IsTrue(reader.Read());
+      Assert.AreEqual(JsonToken.PropertyName, reader.TokenType);
+      Assert.AreEqual("Arguments", reader.Value.ToString());
+
+      Assert.IsTrue(reader.Read());
+      Assert.AreEqual(JsonToken.StartObject, reader.TokenType);
+
+      Assert.IsTrue(reader.Read());
+      Assert.AreEqual(JsonToken.EndObject, reader.TokenType);
+
+      Assert.IsTrue(reader.Read());
+      Assert.AreEqual(JsonToken.PropertyName, reader.TokenType);
+      Assert.AreEqual("List", reader.Value.ToString());
+
+      Assert.IsTrue(reader.Read());
+      Assert.AreEqual(JsonToken.StartArray, reader.TokenType);
+
+      Assert.IsTrue(reader.Read());
+      Assert.AreEqual(JsonToken.EndArray, reader.TokenType);
+
+      Assert.IsTrue(reader.Read());
+      Assert.AreEqual(JsonToken.EndObject, reader.TokenType);
+
+      Assert.IsFalse(reader.Read());
+    }
   }
 }

Src/Newtonsoft.Json.Tests/JsonTextReaderTest.cs

@@ -647,5 +647,29 @@ bye", reader.Value);
 
       Assert.IsFalse(jsonReader.Read());
     }
+
+    [Test]
+    public void ReadUnicode()
+    {
+      string json = @"{""Message"":""Hi,I\u0092ve send you smth""}";
+
+      JsonTextReader reader = new JsonTextReader(new StringReader(json));
+
+      Assert.IsTrue(reader.Read());
+      Assert.AreEqual(JsonToken.StartObject, reader.TokenType);
+
+      Assert.IsTrue(reader.Read());
+      Assert.AreEqual(JsonToken.PropertyName, reader.TokenType);
+      Assert.AreEqual("Message", reader.Value);
+
+      Assert.IsTrue(reader.Read());
+      Assert.AreEqual(JsonToken.String, reader.TokenType);
+      Assert.AreEqual(@"Hi,I" + '\u0092' + "ve send you smth", reader.Value);
+
+      Assert.IsTrue(reader.Read());
+      Assert.AreEqual(JsonToken.EndObject, reader.TokenType);
+
+      Assert.IsFalse(reader.Read());
+    }
   }
 }

Src/Newtonsoft.Json.Tests/Serialization/JsonSerializerTest.cs

@@ -2232,5 +2232,77 @@ keyword such as type of business.""
       deserialized = JsonConvert.DeserializeObject<JRawValueTestObject>("{value:'3'}");
       Assert.AreEqual(@"""3""", deserialized.Value.ToString());
     }
+
+    [Test]
+    [ExpectedException(typeof(JsonSerializationException), ExpectedMessage = "Unable to find a default constructor to use for type Newtonsoft.Json.Tests.Serialization.JsonSerializerTest+DictionaryWithNoDefaultConstructor.")]
+    public void DeserializeDictionaryWithNoDefaultConstructor()
+    {
+      string json = "{key1:'value',key2:'value',key3:'value'}";
+      JsonConvert.DeserializeObject<DictionaryWithNoDefaultConstructor>(json);
+    }
+
+    public class DictionaryWithNoDefaultConstructor : Dictionary<string, string>
+    {
+      public DictionaryWithNoDefaultConstructor(IEnumerable<KeyValuePair<string, string>> initial)
+      {
+        foreach (KeyValuePair<string, string> pair in initial)
+        {
+          Add(pair.Key, pair.Value);
+        }
+      }
+    }
+
+    [JsonObject(MemberSerialization.OptIn)]
+    public class A
+    {
+      [JsonProperty("A1")]
+      private string _A1;
+      public string A1 { get { return _A1; } set { _A1 = value; } }
+
+      [JsonProperty("A2")]
+      private string A2 { get; set; }
+    }
+
+    [JsonObject(MemberSerialization.OptIn)]
+    public class B : A
+    {
+      public string B1 { get; set; }
+
+      [JsonProperty("B2")]
+      string _B2;
+      public string B2 { get { return _B2; } set { _B2 = value; } }
+
+      [JsonProperty("B3")]
+      private string B3 { get; set; }
+    }
+
+    [Test]
+    public void SerializeNonPublicBaseJsonProperties()
+    {
+      B value = new B();
+      string json = JsonConvert.SerializeObject(value, Formatting.Indented);
+
+      Assert.AreEqual(@"{
+  ""B2"": null,
+  ""A1"": null,
+  ""B3"": null,
+  ""A2"": null
+}", json);
+    }
+
+    public class TestClass
+    {
+      public string Key { get; set; }
+      public object Value { get; set; }
+    }
+
+    [Test]
+    public void DeserializeToObjectProperty()
+    {
+      var json = "{ Key: 'abc', Value: 123 }";
+      var item = JsonConvert.DeserializeObject<TestClass>(json);
+      
+      Assert.AreEqual(123, item.Value);
+    }
   }
 }

Src/Newtonsoft.Json/Bson/BsonReader.cs

@@ -265,11 +265,6 @@ namespace Newtonsoft.Json.Bson
             newContext.Length = ReadInt32();
             return true;
           }
-        case State.ObjectStart:
-          {
-            SetToken(JsonToken.PropertyName, ReadElement());
-            return true;
-          }
         case State.Complete:
         case State.Closed:
           return false;
@@ -278,10 +273,8 @@ namespace Newtonsoft.Json.Bson
             ReadType(_currentElementType);
             return true;
           }
+        case State.ObjectStart:
         case State.ArrayStart:
-          ReadElement();
-          ReadType(_currentElementType);
-          return true;
         case State.PostValue:
           ContainerContext context = _currentContext;
           if (context == null)

Src/Newtonsoft.Json/Serialization/JsonSerializerInternalReader.cs

@@ -465,11 +465,19 @@ namespace Newtonsoft.Json.Serialization
 
     private object CreateAndPopulateDictionary(JsonReader reader, JsonDictionaryContract contract, string id)
     {
-      IWrappedDictionary dictionary = contract.CreateWrapper(contract.DefaultCreator());
+      object dictionary;
 
-      PopulateDictionary(dictionary, reader, contract, id);
+      if (contract.DefaultCreator != null &&
+        (!contract.DefaultCreatorNonPublic || Serializer.ConstructorHandling == ConstructorHandling.AllowNonPublicDefaultConstructor))
+        dictionary = contract.DefaultCreator();
+      else
+        throw new JsonSerializationException("Unable to find a default constructor to use for type {0}.".FormatWith(CultureInfo.InvariantCulture, contract.UnderlyingType));
 
-      return dictionary.UnderlyingDictionary;
+      IWrappedDictionary dictionaryWrapper = contract.CreateWrapper(dictionary);
+
+      PopulateDictionary(dictionaryWrapper, reader, contract, id);
+
+      return dictionaryWrapper.UnderlyingDictionary;
     }
 
     private object PopulateDictionary(IWrappedDictionary dictionary, JsonReader reader, JsonDictionaryContract contract, string id)

Src/Newtonsoft.Json/Utilities/ConvertUtils.cs

@@ -315,6 +315,9 @@ namespace Newtonsoft.Json.Utilities
     {
       object convertedValue;
 
+      if (targetType == typeof(object))
+        return initialValue;
+
       if (initialValue == null && ReflectionUtils.IsNullable(targetType))
         return null;
 

Src/Newtonsoft.Json/Utilities/ReflectionUtils.cs

@@ -25,10 +25,8 @@
 
 using System;
 using System.Collections.Generic;
-using System.Text;
 using System.Reflection;
 using System.Collections;
-using System.ComponentModel;
 using System.Linq;
 using System.Globalization;
 
@@ -240,10 +238,10 @@ namespace Newtonsoft.Json.Utilities
       if (!genericClassDefinition.IsClass || !genericClassDefinition.IsGenericTypeDefinition)
         throw new ArgumentNullException("'{0}' is not a generic class definition.".FormatWith(CultureInfo.InvariantCulture, genericClassDefinition));
 
-      return InheritsGenericDefinitionInternal(type, type, genericClassDefinition, out implementingType);
+      return InheritsGenericDefinitionInternal(type, genericClassDefinition, out implementingType);
     }
 
-    private static bool InheritsGenericDefinitionInternal(Type initialType, Type currentType, Type genericClassDefinition, out Type implementingType)
+    private static bool InheritsGenericDefinitionInternal(Type currentType, Type genericClassDefinition, out Type implementingType)
     {
       if (currentType.IsGenericType)
       {
@@ -262,7 +260,7 @@ namespace Newtonsoft.Json.Utilities
         return false;
       }
 
-      return InheritsGenericDefinitionInternal(initialType, currentType.BaseType, genericClassDefinition, out implementingType);
+      return InheritsGenericDefinitionInternal(currentType.BaseType, genericClassDefinition, out implementingType);
     }
 
     /// <summary>
@@ -538,8 +536,8 @@ namespace Newtonsoft.Json.Utilities
     {
       List<MemberInfo> targetMembers = new List<MemberInfo>();
 
-      targetMembers.AddRange(type.GetFields(bindingAttr));
-      targetMembers.AddRange(type.GetProperties(bindingAttr));
+      targetMembers.AddRange(GetFields(type, bindingAttr));
+      targetMembers.AddRange(GetProperties(type, bindingAttr));
 
       // for some reason .NET returns multiple members when overriding a generic member on a base class
       // http://forums.msdn.microsoft.com/en-US/netfxbcl/thread/b5abbfee-e292-4a64-8907-4e3f0fb90cd9/
@@ -622,12 +620,12 @@ namespace Newtonsoft.Json.Utilities
 
     public static object CreateGeneric(Type genericTypeDefinition, Type innerType, params object[] args)
     {
-      return CreateGeneric(genericTypeDefinition, new Type[] { innerType }, args);
+      return CreateGeneric(genericTypeDefinition, new [] { innerType }, args);
     }
 
     public static object CreateGeneric(Type genericTypeDefinition, IList<Type> innerTypes, params object[] args)
     {
-      return CreateGeneric(genericTypeDefinition, innerTypes, (t, a) => ReflectionUtils.CreateInstance(t, a.ToArray()), args);
+      return CreateGeneric(genericTypeDefinition, innerTypes, (t, a) => CreateInstance(t, a.ToArray()), args);
     }
 
     public static object CreateGeneric(Type genericTypeDefinition, IList<Type> innerTypes, Func<Type, IList<object>, object> instanceCreator, params object[] args)
@@ -641,16 +639,16 @@ namespace Newtonsoft.Json.Utilities
       return instanceCreator(specificType, args);
     }
 
-     public static bool IsCompatibleValue(object value, Type type)
-     {
-       if (value == null && IsNullable(type))
-         return true;
+    public static bool IsCompatibleValue(object value, Type type)
+    {
+      if (value == null)
+        return IsNullable(type);
 
-       if (type.IsAssignableFrom(value.GetType()))
-         return true;
+      if (type.IsAssignableFrom(value.GetType()))
+        return true;
 
-       return false;
-     }
+      return false;
+    }
 
      public static object CreateInstance(Type type, params object[] args)
      {
@@ -730,5 +728,61 @@ namespace Newtonsoft.Json.Utilities
 
       return null;
     }
+
+    public static IEnumerable<FieldInfo> GetFields(Type targetType, BindingFlags bindingAttr)
+    {
+      ValidationUtils.ArgumentNotNull(targetType, "targetType");
+
+      List<MemberInfo> fieldInfos = new List<MemberInfo>(targetType.GetFields(bindingAttr));
+      GetChildPrivateFields(fieldInfos, targetType, bindingAttr);
+
+      return fieldInfos.Cast<FieldInfo>();
+    }
+
+    private static void GetChildPrivateFields(IList<MemberInfo> initialFields, Type targetType, BindingFlags bindingAttr)
+    {
+      // fix weirdness with FieldInfos only being returned for the current Type
+      // find base type fields and add them to result
+      if ((bindingAttr & BindingFlags.NonPublic) != 0)
+      {
+        // modify flags to not search for public fields
+        BindingFlags nonPublicBindingAttr = ((bindingAttr & BindingFlags.Public) == BindingFlags.Public)
+          ? bindingAttr ^ BindingFlags.Public
+          : bindingAttr;
+
+        while ((targetType = targetType.BaseType) != null)
+        {
+          initialFields.AddRange(targetType.GetFields(nonPublicBindingAttr));
+        }
+      }
+    }
+
+    public static IEnumerable<PropertyInfo> GetProperties(Type targetType, BindingFlags bindingAttr)
+    {
+      ValidationUtils.ArgumentNotNull(targetType, "targetType");
+
+      List<MemberInfo> propertyInfos = new List<MemberInfo>(targetType.GetProperties(bindingAttr));
+      GetChildPrivateProperties(propertyInfos, targetType, bindingAttr);
+
+      return propertyInfos.Cast<PropertyInfo>();
+    }
+
+    private static void GetChildPrivateProperties(IList<MemberInfo> initialFields, Type targetType, BindingFlags bindingAttr)
+    {
+      // fix weirdness with FieldInfos only being returned for the current Type
+      // find base type fields and add them to result
+      if ((bindingAttr & BindingFlags.NonPublic) != 0)
+      {
+        // modify flags to not search for public fields
+        BindingFlags nonPublicBindingAttr = ((bindingAttr & BindingFlags.Public) == BindingFlags.Public)
+          ? bindingAttr ^ BindingFlags.Public
+          : bindingAttr;
+
+        while ((targetType = targetType.BaseType) != null)
+        {
+          initialFields.AddRange(targetType.GetProperties(nonPublicBindingAttr));
+        }
+      }
+    }
   }
 }