-Rewrote JsonTextReader to use internal buffer to improve performance

-Improved JsonSerializer to cache contract/converters where possible to improve performance
-Improved JsonTextReader error message when additional JSON content found
-Fixed internal exception thrown when deserializing Decimal/DateTimeOffset/Byte lists
-Fixed various JsonTextReader bugs

Src/Newtonsoft.Json.Tests/JsonConvertTest.cs

@@ -257,7 +257,7 @@ now brown cow?", '"', true);
     }
 
     [Test]
-    [ExpectedException(typeof(JsonReaderException), ExpectedMessage = "After parsing a value an unexpected character was encountered: t. Line 1, position 20.")]
+    [ExpectedException(typeof(JsonReaderException), ExpectedMessage = "Additional text encountered after finished reading JSON content: t. Line 1, position 19.")]
     public void TestInvalidStrings()
     {
       string orig = @"this is a string ""that has quotes"" ";

Src/Newtonsoft.Json.Tests/JsonValidatingReaderTests.cs

@@ -371,7 +371,7 @@ namespace Newtonsoft.Json.Tests
 
       Assert.IsTrue(reader.Read());
       Assert.AreEqual(JsonToken.Integer, reader.TokenType);
-      Assert.AreEqual(@"Value 3 is not defined in enum. Line 1, position 7.", validationEventArgs.Message);
+      Assert.AreEqual(@"Value 3 is not defined in enum. Line 1, position 6.", validationEventArgs.Message);
 
       Assert.IsTrue(reader.Read());
       Assert.AreEqual(JsonToken.EndArray, reader.TokenType);
@@ -457,7 +457,7 @@ namespace Newtonsoft.Json.Tests
 
       Assert.IsTrue(reader.Read());
       Assert.AreEqual(JsonToken.Float, reader.TokenType);
-      Assert.AreEqual(@"Value 3.0 is not defined in enum. Line 1, position 13.", validationEventArgs.Message);
+      Assert.AreEqual(@"Value 3.0 is not defined in enum. Line 1, position 12.", validationEventArgs.Message);
 
       Assert.IsTrue(reader.Read());
       Assert.AreEqual(JsonToken.EndArray, reader.TokenType);
@@ -496,7 +496,7 @@ namespace Newtonsoft.Json.Tests
 
       Assert.IsTrue(reader.Read());
       Assert.AreEqual(JsonToken.Float, reader.TokenType);
-      Assert.AreEqual(@"Float 4.001 is not evenly divisible by 0.1. Line 1, position 15.", validationEventArgs.Message);
+      Assert.AreEqual(@"Float 4.001 is not evenly divisible by 0.1. Line 1, position 14.", validationEventArgs.Message);
 
       Assert.IsTrue(reader.Read());
       Assert.AreEqual(JsonToken.EndArray, reader.TokenType);
@@ -723,7 +723,7 @@ namespace Newtonsoft.Json.Tests
 
       Assert.IsTrue(reader.Read());
       Assert.AreEqual(JsonToken.Float, reader.TokenType);
-      Assert.AreEqual(@"Type Float is disallowed. Line 1, position 11.", validationEventArgs.Message);
+      Assert.AreEqual(@"Type Float is disallowed. Line 1, position 10.", validationEventArgs.Message);
 
       Assert.IsTrue(reader.Read());
       Assert.AreEqual(JsonToken.EndArray, reader.TokenType);
@@ -1029,7 +1029,7 @@ namespace Newtonsoft.Json.Tests
 
       Assert.IsTrue(reader.Read());
       Assert.AreEqual(JsonToken.Integer, reader.TokenType);
-      Assert.AreEqual("Invalid type. Expected String but got Integer. Line 1, position 3.", validationEventArgs.Message);
+      Assert.AreEqual("Invalid type. Expected String but got Integer. Line 1, position 2.", validationEventArgs.Message);
 
       Assert.IsTrue(reader.Read());
       Assert.AreEqual(JsonToken.String, reader.TokenType);
@@ -1097,7 +1097,7 @@ namespace Newtonsoft.Json.Tests
 
       Assert.IsTrue(reader.Read());
       Assert.AreEqual(JsonToken.PropertyName, reader.TokenType);
-      Assert.AreEqual("Property 'hey' has not been defined and the schema does not allow additional properties. Line 5, position 8.", validationEventArgs.Message);
+      Assert.AreEqual("Property 'hey' has not been defined and the schema does not allow additional properties. Line 5, position 9.", validationEventArgs.Message);
 
       Assert.IsTrue(reader.Read());
       Assert.AreEqual(JsonToken.String, reader.TokenType);
@@ -1198,7 +1198,7 @@ namespace Newtonsoft.Json.Tests
       Assert.AreEqual(JsonToken.String, reader.TokenType);
       Assert.AreEqual("secasecasecasecaseca", reader.Value.ToString());
       Assert.AreEqual(1, errors.Count);
-      Assert.AreEqual("String 'secasecasecasecaseca' exceeds maximum length of 10. Line 3, position 41.", errors[0]);
+      Assert.AreEqual("String 'secasecasecasecaseca' exceeds maximum length of 10. Line 3, position 42.", errors[0]);
 
       Assert.IsTrue(reader.Read());
       Assert.AreEqual(JsonToken.PropertyName, reader.TokenType);
@@ -1218,9 +1218,9 @@ namespace Newtonsoft.Json.Tests
       Assert.AreEqual(JsonToken.String, reader.TokenType);
       Assert.AreEqual("aaa", reader.Value.ToString());
       Assert.AreEqual(4, errors.Count);
-      Assert.AreEqual("String 'aaa' is less than minimum length of 7. Line 5, position 39.", errors[1]);
-      Assert.AreEqual("String 'aaa' does not match regex pattern 'hi'. Line 5, position 39.", errors[2]);
-      Assert.AreEqual("String 'aaa' does not match regex pattern 'hi2u'. Line 5, position 39.", errors[3]);
+      Assert.AreEqual("String 'aaa' is less than minimum length of 7. Line 5, position 40.", errors[1]);
+      Assert.AreEqual("String 'aaa' does not match regex pattern 'hi'. Line 5, position 40.", errors[2]);
+      Assert.AreEqual("String 'aaa' does not match regex pattern 'hi2u'. Line 5, position 40.", errors[3]);
 
       Assert.IsTrue(reader.Read());
       Assert.AreEqual(JsonToken.PropertyName, reader.TokenType);
@@ -1231,7 +1231,7 @@ namespace Newtonsoft.Json.Tests
       Assert.AreEqual(JsonToken.String, reader.TokenType);
       Assert.AreEqual("three", reader.Value.ToString());
       Assert.AreEqual(5, errors.Count);
-      Assert.AreEqual("String 'three' is less than minimum length of 6. Line 6, position 24.", errors[4]);
+      Assert.AreEqual("String 'three' is less than minimum length of 6. Line 6, position 25.", errors[4]);
       
       Assert.IsTrue(reader.Read());
       Assert.AreEqual(JsonToken.EndObject, reader.TokenType);

Src/Newtonsoft.Json.Tests/Linq/JArrayTests.cs

@@ -450,7 +450,7 @@ Parameter name: index")]
     }
 
     [Test]
-    [ExpectedException(typeof(JsonReaderException), ExpectedMessage = "Unexpected state: Finished. Line 5, position 3.")]
+    [ExpectedException(typeof(JsonReaderException), ExpectedMessage = "Additional text encountered after finished reading JSON content: ,. Line 5, position 2.")]
     public void ParseAdditionalContent()
     {
       string json = @"[

Src/Newtonsoft.Json.Tests/Linq/JObjectTests.cs

@@ -1626,7 +1626,7 @@ Parameter name: arrayIndex")]
     }
 
     [Test]
-    [ExpectedException(typeof(JsonReaderException), ExpectedMessage = "Unexpected state: Finished. Line 10, position 3.")]
+    [ExpectedException(typeof(JsonReaderException), ExpectedMessage = "Additional text encountered after finished reading JSON content: ,. Line 10, position 2.")]
     public void ParseAdditionalContent()
     {
       string json = @"{

Src/Newtonsoft.Json.Tests/Linq/JTokenReaderTest.cs

@@ -147,41 +147,41 @@ namespace Newtonsoft.Json.Tests.Linq
         Assert.AreEqual(jsonReader.TokenType, JsonToken.PropertyName);
         Assert.AreEqual(jsonReader.Value, "CPU");
         Assert.AreEqual(2, lineInfo.LineNumber);
-        Assert.AreEqual(6, lineInfo.LinePosition);
+        Assert.AreEqual(7, lineInfo.LinePosition);
         Assert.AreEqual(true, lineInfo.HasLineInfo());
 
         jsonReader.Read();
         Assert.AreEqual(jsonReader.TokenType, JsonToken.String);
         Assert.AreEqual(jsonReader.Value, "Intel");
         Assert.AreEqual(2, lineInfo.LineNumber);
-        Assert.AreEqual(14, lineInfo.LinePosition);
+        Assert.AreEqual(15, lineInfo.LinePosition);
         Assert.AreEqual(true, lineInfo.HasLineInfo());
 
         jsonReader.Read();
         Assert.AreEqual(jsonReader.TokenType, JsonToken.PropertyName);
         Assert.AreEqual(jsonReader.Value, "Drives");
         Assert.AreEqual(3, lineInfo.LineNumber);
-        Assert.AreEqual(9, lineInfo.LinePosition);
+        Assert.AreEqual(10, lineInfo.LinePosition);
         Assert.AreEqual(true, lineInfo.HasLineInfo());
 
         jsonReader.Read();
         Assert.AreEqual(jsonReader.TokenType, JsonToken.StartArray);
         Assert.AreEqual(3, lineInfo.LineNumber);
-        Assert.AreEqual(11, lineInfo.LinePosition);
+        Assert.AreEqual(12, lineInfo.LinePosition);
         Assert.AreEqual(true, lineInfo.HasLineInfo());
 
         jsonReader.Read();
         Assert.AreEqual(jsonReader.TokenType, JsonToken.String);
         Assert.AreEqual(jsonReader.Value, "DVD read/writer");
         Assert.AreEqual(4, lineInfo.LineNumber);
-        Assert.AreEqual(21, lineInfo.LinePosition);
+        Assert.AreEqual(22, lineInfo.LinePosition);
         Assert.AreEqual(true, lineInfo.HasLineInfo());
 
         jsonReader.Read();
         Assert.AreEqual(jsonReader.TokenType, JsonToken.String);
         Assert.AreEqual(jsonReader.Value, "500 gigabyte hard drive");
         Assert.AreEqual(5, lineInfo.LineNumber);
-        Assert.AreEqual(29, lineInfo.LinePosition);
+        Assert.AreEqual(30, lineInfo.LinePosition);
         Assert.AreEqual(true, lineInfo.HasLineInfo());
 
         jsonReader.Read();

Src/Newtonsoft.Json.Tests/Linq/JTokenTests.cs

@@ -741,14 +741,14 @@ namespace Newtonsoft.Json.Tests.Linq
     }
 
     [Test]
-    [ExpectedException(typeof(JsonReaderException), ExpectedMessage = "Unexpected state: Finished. Line 5, position 3.")]
+    [ExpectedException(typeof(JsonReaderException), ExpectedMessage = "Additional text encountered after finished reading JSON content: ,. Line 5, position 2.")]
     public void ParseAdditionalContent()
     {
       string json = @"[
 ""Small"",
 ""Medium"",
 ""Large""
-], 987987";
+],";
 
       JToken.Parse(json);
     }

Src/Newtonsoft.Json.Tests/Properties/AssemblyInfo.cs

@@ -64,7 +64,7 @@ using System.Runtime.InteropServices;
 //
 // You can specify all the values or you can default the Revision and Build Numbers 
 // by using the '*' as shown below:
-[assembly: AssemblyVersion("4.0.3.0")]
+[assembly: AssemblyVersion("4.0.5.0")]
 #if !PocketPC
-[assembly: AssemblyFileVersion("4.0.3.14319")]
+[assembly: AssemblyFileVersion("4.0.5.14516")]
 #endif

Src/Newtonsoft.Json.Tests/Schema/ExtensionsTests.cs

@@ -108,7 +108,7 @@ namespace Newtonsoft.Json.Tests.Schema
       List<string> errors = new List<string>();
       o.Validate(schema, (sender, args) => errors.Add(args.Message));
 
-      Assert.AreEqual("Invalid type. Expected String but got Integer. Line 1, position 9.", errors[0]);
+      Assert.AreEqual("Invalid type. Expected String but got Integer. Line 1, position 8.", errors[0]);
       Assert.AreEqual(1, errors.Count);
     }
 

Src/Newtonsoft.Json.Tests/Serialization/JsonSerializerTest.cs

@@ -3263,7 +3263,7 @@ keyword such as type of business.""
           Converters = new List<JsonConverter> {new StringAppenderConverter()}
         });
 
-      Assert.AreEqual(p.Name, "Existing,Appended");
+      Assert.AreEqual("Existing,Appended", p.Name);
     }
 
     [Test]
@@ -5073,6 +5073,45 @@ keyword such as type of business.""
       }
     }
 #endif
+
+    [Test]
+    public void ReadForTypeHackFixDecimal()
+    {
+      IList<decimal> d1 = new List<decimal> { 1.1m};
+
+      string json = JsonConvert.SerializeObject(d1);
+
+      IList<decimal> d2 = JsonConvert.DeserializeObject<IList<decimal>>(json);
+
+      Assert.AreEqual(d1.Count, d2.Count);
+      Assert.AreEqual(d1[0], d2[0]);
+    }
+
+    [Test]
+    public void ReadForTypeHackFixDateTimeOffset()
+    {
+      IList<DateTimeOffset?> d1 = new List<DateTimeOffset?> { null };
+
+      string json = JsonConvert.SerializeObject(d1);
+
+      IList<DateTimeOffset?> d2 = JsonConvert.DeserializeObject<IList<DateTimeOffset?>>(json);
+
+      Assert.AreEqual(d1.Count, d2.Count);
+      Assert.AreEqual(d1[0], d2[0]);
+    }
+
+    [Test]
+    public void ReadForTypeHackFixByteArray()
+    {
+      IList<byte[]> d1 = new List<byte[]> { null };
+
+      string json = JsonConvert.SerializeObject(d1);
+
+      IList<byte[]> d2 = JsonConvert.DeserializeObject<IList<byte[]>>(json);
+
+      Assert.AreEqual(d1.Count, d2.Count);
+      Assert.AreEqual(d1[0], d2[0]);
+    }
   }
 
   public class DecimalTestClass

Src/Newtonsoft.Json/Bson/BsonReader.cs

@@ -171,6 +171,9 @@ namespace Newtonsoft.Json.Bson
       if (TokenType == JsonToken.Bytes)
         return (byte[])Value;
 
+      if (ReaderIsSerializerInArray())
+        return null;
+
       throw new JsonReaderException("Error reading bytes. Expected bytes but got {0}.".FormatWith(CultureInfo.InvariantCulture, TokenType));
     }
 
@@ -214,6 +217,9 @@ namespace Newtonsoft.Json.Bson
         return (decimal)Value;
       }
 
+      if (ReaderIsSerializerInArray())
+        return null;
+
       throw new JsonReaderException("Error reading decimal. Expected a number but got {0}.".FormatWith(CultureInfo.InvariantCulture, TokenType));
     }
 
@@ -235,7 +241,10 @@ namespace Newtonsoft.Json.Bson
         SetToken(JsonToken.Date, new DateTimeOffset((DateTime)Value));
         return (DateTimeOffset)Value;
       }
-      
+
+      if (ReaderIsSerializerInArray())
+        return null;
+
       throw new JsonReaderException("Error reading date. Expected bytes but got {0}.".FormatWith(CultureInfo.InvariantCulture, TokenType));
     }
 #endif

Src/Newtonsoft.Json/JsonConvert.cs

@@ -619,7 +619,7 @@ namespace Newtonsoft.Json
     {
       JsonSerializer jsonSerializer = JsonSerializer.Create(settings);
 
-      StringBuilder sb = new StringBuilder(128);
+      StringBuilder sb = new StringBuilder(256);
       StringWriter sw = new StringWriter(sb, CultureInfo.InvariantCulture);
       using (JsonTextWriter jsonWriter = new JsonTextWriter(sw))
       {

Src/Newtonsoft.Json/JsonReader.cs

@@ -40,7 +40,7 @@ namespace Newtonsoft.Json
     /// <summary>
     /// Specifies the state of the reader.
     /// </summary>
-    protected enum State
+    protected internal enum State
     {
       /// <summary>
       /// The Read method has not been called.
@@ -99,10 +99,15 @@ namespace Newtonsoft.Json
     // current Token data
     private JsonToken _token;
     private object _value;
-    private Type _valueType;
     private char _quoteChar;
-    private State _currentState;
+    internal State _currentState;
     private JTokenType _currentTypeContext;
+    private bool _serializerInArray;
+
+    internal void SetSerializeInArray(bool serializerInArray)
+    {
+      _serializerInArray = serializerInArray;
+    }
 
     /// <summary>
     /// Gets the current reader state.
@@ -157,7 +162,7 @@ namespace Newtonsoft.Json
     /// </summary>
     public virtual Type ValueType
     {
-      get { return _valueType; }
+      get { return (_value != null) ? _value.GetType() : null; }
     }
 
     /// <summary>
@@ -242,6 +247,9 @@ namespace Newtonsoft.Json
     /// </summary>
     public void Skip()
     {
+      if (TokenType == JsonToken.PropertyName)
+        Read();
+
       if (IsStartToken(TokenType))
       {
         int depth = Depth;
@@ -266,7 +274,7 @@ namespace Newtonsoft.Json
     /// </summary>
     /// <param name="newToken">The new token.</param>
     /// <param name="value">The value.</param>
-    protected virtual void SetToken(JsonToken newToken, object value)
+    protected void SetToken(JsonToken newToken, object value)
     {
       _token = newToken;
 
@@ -286,19 +294,15 @@ namespace Newtonsoft.Json
           break;
         case JsonToken.EndObject:
           ValidateEnd(JsonToken.EndObject);
-          _currentState = State.PostValue;
           break;
         case JsonToken.EndArray:
           ValidateEnd(JsonToken.EndArray);
-          _currentState = State.PostValue;
           break;
         case JsonToken.EndConstructor:
           ValidateEnd(JsonToken.EndConstructor);
-          _currentState = State.PostValue;
           break;
         case JsonToken.PropertyName:
           _currentState = State.Property;
-          Push(JTokenType.Property);
           break;
         case JsonToken.Undefined:
         case JsonToken.Integer:
@@ -309,24 +313,11 @@ namespace Newtonsoft.Json
         case JsonToken.String:
         case JsonToken.Raw:
         case JsonToken.Bytes:
-          _currentState = State.PostValue;
+          _currentState = (Peek() != JTokenType.None) ? State.PostValue : State.Finished;
           break;
       }
 
-      JTokenType current = Peek();
-      if (current == JTokenType.Property && _currentState == State.PostValue)
-        Pop();
-
-      if (value != null)
-      {
-        _value = value;
-        _valueType = value.GetType();
-      }
-      else
-      {
-        _value = null;
-        _valueType = null;
-      }
+      _value = value;
     }
 
     private void ValidateEnd(JsonToken endToken)
@@ -335,6 +326,8 @@ namespace Newtonsoft.Json
 
       if (GetTypeForCloseToken(endToken) != currentObject)
         throw new JsonReaderException("JsonToken {0} is not valid for closing JsonType {1}.".FormatWith(CultureInfo.InvariantCulture, endToken, currentObject));
+
+      _currentState = (Peek() != JTokenType.None) ? State.PostValue : State.Finished;
     }
 
     /// <summary>
@@ -388,8 +381,8 @@ namespace Newtonsoft.Json
         case JsonToken.StartObject:
         case JsonToken.StartArray:
         case JsonToken.StartConstructor:
-        case JsonToken.PropertyName:
           return true;
+        case JsonToken.PropertyName:
         case JsonToken.None:
         case JsonToken.Comment:
         case JsonToken.Integer:
@@ -425,6 +418,11 @@ namespace Newtonsoft.Json
       }
     }
 
+    internal bool ReaderIsSerializerInArray()
+    {
+      return (TokenType == JsonToken.EndArray && _serializerInArray);
+    }
+
     /// <summary>
     /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
     /// </summary>
@@ -451,7 +449,6 @@ namespace Newtonsoft.Json
       _currentState = State.Closed;
       _token = JsonToken.None;
       _value = null;
-      _valueType = null;
     }
   }
 }

Src/Newtonsoft.Json/JsonValidatingReader.cs

@@ -131,7 +131,7 @@ namespace Newtonsoft.Json
     }
 
     /// <summary>
-    /// Gets The Common Language Runtime (CLR) type for the current Json token.
+    /// Gets the Common Language Runtime (CLR) type for the current Json token.
     /// </summary>
     /// <value></value>
     public override Type ValueType

Src/Newtonsoft.Json/Linq/JToken.cs

@@ -1212,7 +1212,6 @@ namespace Newtonsoft.Json.Linq
       if (reader.TokenType == JsonToken.StartConstructor)
         return JConstructor.Load(reader);
 
-      // hack. change to look at TokenType rather than using value
       if (!JsonReader.IsStartToken(reader.TokenType))
         return new JValue(reader.Value);
 

Src/Newtonsoft.Json/Linq/JTokenReader.cs

@@ -59,6 +59,9 @@ namespace Newtonsoft.Json.Linq
       if (TokenType == JsonToken.Bytes)
         return (byte[])Value;
 
+      if (ReaderIsSerializerInArray())
+        return null;
+
       throw new JsonReaderException("Error reading bytes. Expected bytes but got {0}.".FormatWith(CultureInfo.InvariantCulture, TokenType));
     }
 
@@ -102,6 +105,9 @@ namespace Newtonsoft.Json.Linq
         return (decimal) Value;
       }
 
+      if (ReaderIsSerializerInArray())
+        return null;
+
       throw new JsonReaderException("Error reading decimal. Expected a number but got {0}.".FormatWith(CultureInfo.InvariantCulture, TokenType));
     }
 
@@ -122,6 +128,9 @@ namespace Newtonsoft.Json.Linq
         return (DateTimeOffset)Value;
       }
 
+      if (ReaderIsSerializerInArray())
+        return null;
+
       throw new JsonReaderException("Error reading date. Expected bytes but got {0}.".FormatWith(CultureInfo.InvariantCulture, TokenType));
     }
 #endif

Src/Newtonsoft.Json/Newtonsoft.Json.Net20.csproj

@@ -195,6 +195,7 @@
     <Compile Include="Utilities\LateBoundReflectionDelegateFactory.cs" />
     <Compile Include="Utilities\MethodCall.cs" />
     <Compile Include="Utilities\ReflectionDelegateFactory.cs" />
+    <Compile Include="Utilities\StringReference.cs" />
     <Compile Include="Utilities\ThreadSafeStore.cs" />
     <Compile Include="TypeNameHandling.cs" />
     <Compile Include="Utilities\BidirectionalDictionary.cs" />

Src/Newtonsoft.Json/Newtonsoft.Json.Net35.csproj

@@ -210,6 +210,7 @@
     <Compile Include="Utilities\ReflectionDelegateFactory.cs" />
     <Compile Include="Utilities\LateBoundReflectionDelegateFactory.cs" />
     <Compile Include="Utilities\MethodCall.cs" />
+    <Compile Include="Utilities\StringReference.cs" />
     <Compile Include="Utilities\ThreadSafeStore.cs" />
     <Compile Include="TypeNameHandling.cs" />
     <Compile Include="Utilities\BidirectionalDictionary.cs" />

Src/Newtonsoft.Json/Newtonsoft.Json.Silverlight.csproj

@@ -211,6 +211,7 @@
     <Compile Include="Utilities\ReflectionDelegateFactory.cs" />
     <Compile Include="Utilities\ReflectionUtils.cs" />
     <Compile Include="Utilities\StringBuffer.cs" />
+    <Compile Include="Utilities\StringReference.cs" />
     <Compile Include="Utilities\StringUtils.cs" />
     <Compile Include="Utilities\ThreadSafeStore.cs" />
     <Compile Include="Utilities\ValidationUtils.cs" />

Src/Newtonsoft.Json/Newtonsoft.Json.WindowsPhone.csproj

@@ -181,6 +181,7 @@
     <Compile Include="Utilities\ReflectionDelegateFactory.cs" />
     <Compile Include="Utilities\ReflectionUtils.cs" />
     <Compile Include="Utilities\StringBuffer.cs" />
+    <Compile Include="Utilities\StringReference.cs" />
     <Compile Include="Utilities\StringUtils.cs" />
     <Compile Include="Utilities\ThreadSafeStore.cs" />
     <Compile Include="Utilities\ValidationUtils.cs" />

Src/Newtonsoft.Json/Newtonsoft.Json.csproj

@@ -209,6 +209,7 @@
     <Compile Include="Utilities\ReflectionDelegateFactory.cs" />
     <Compile Include="Utilities\LateBoundReflectionDelegateFactory.cs" />
     <Compile Include="Utilities\MethodCall.cs" />
+    <Compile Include="Utilities\StringReference.cs" />
     <Compile Include="Utilities\ThreadSafeStore.cs" />
     <Compile Include="TypeNameHandling.cs" />
     <Compile Include="Utilities\BidirectionalDictionary.cs" />

Src/Newtonsoft.Json/Properties/AssemblyInfo.cs

@@ -83,9 +83,9 @@ using System.Security;
 //
 // You can specify all the values or you can default the Revision and Build Numbers 
 // by using the '*' as shown below:
-[assembly: AssemblyVersion("4.0.3.0")]
+[assembly: AssemblyVersion("4.0.5.0")]
 #if !PocketPC
-[assembly: AssemblyFileVersion("4.0.3.14319")]
+[assembly: AssemblyFileVersion("4.0.5.14516")]
 #endif
 
 [assembly: CLSCompliant(true)]

Src/Newtonsoft.Json/Schema/JsonSchemaGenerator.cs

@@ -291,6 +291,14 @@ namespace Newtonsoft.Json.Schema
           CurrentSchema.Items.Add(GenerateInternal(collectionItemType, (!allowNullItem) ? Required.Always : Required.Default, false));
         }
       }
+      else if (contract is JsonStringContract)
+      {
+        JsonSchemaType schemaType = (!ReflectionUtils.IsNullable(contract.UnderlyingType))
+                                      ? JsonSchemaType.String
+                                      : AddNullType(JsonSchemaType.String, valueRequired);
+
+        CurrentSchema.Type = schemaType;
+      }
       else if (contract is JsonPrimitiveContract)
       {
         CurrentSchema.Type = GetJsonSchemaType(type, valueRequired);
@@ -324,14 +332,6 @@ namespace Newtonsoft.Json.Schema
         GenerateISerializableContract(type, (JsonISerializableContract) contract);
       }
 #endif
-      else if (contract is JsonStringContract)
-      {
-        JsonSchemaType schemaType = (!ReflectionUtils.IsNullable(contract.UnderlyingType))
-                                      ? JsonSchemaType.String
-                                      : AddNullType(JsonSchemaType.String, valueRequired);
-
-        CurrentSchema.Type = schemaType;
-      }
       else if (contract is JsonLinqContract)
       {
         CurrentSchema.Type = JsonSchemaType.Any;

Src/Newtonsoft.Json/Serialization/JsonArrayContract.cs

@@ -38,6 +38,7 @@ namespace Newtonsoft.Json.Serialization
   public class JsonArrayContract : JsonContract
   {
     internal Type CollectionItemType { get; private set; }
+    internal JsonContract CollectionItemContract { get; set; }
 
     private readonly bool _isCollectionItemTypeNullableType;
     private readonly Type _genericCollectionDefinitionType;

Src/Newtonsoft.Json/Serialization/JsonDictionaryContract.cs

@@ -45,6 +45,8 @@ namespace Newtonsoft.Json.Serialization
     internal Type DictionaryKeyType { get; private set; }
     internal Type DictionaryValueType { get; private set; }
 
+    internal JsonContract DictionaryValueContract { get; set; }
+
     private readonly bool _isDictionaryValueTypeNullableType;
     private readonly Type _genericCollectionDefinitionType;
     private Type _genericWrapperType;

Src/Newtonsoft.Json/Serialization/JsonProperty.cs

@@ -32,6 +32,9 @@ namespace Newtonsoft.Json.Serialization
   /// </summary>
   public class JsonProperty
   {
+    // use to cache contract during deserialization
+    internal JsonContract PropertyContract { get; set; }
+    
     /// <summary>
     /// Gets or sets the name of the property.
     /// </summary>

Src/Newtonsoft.Json/Serialization/JsonSerializerInternalReader.cs

@@ -102,23 +102,19 @@ namespace Newtonsoft.Json.Serialization
       return Serializer.ContractResolver.ResolveContract(type);
     }
 
-    private JsonContract GetContractSafe(Type type, object value)
-    {
-      if (value == null)
-        return GetContractSafe(type);
-
-      return Serializer.ContractResolver.ResolveContract(value.GetType());
-    }
-
     public object Deserialize(JsonReader reader, Type objectType)
     {
       if (reader == null)
         throw new ArgumentNullException("reader");
 
-      if (reader.TokenType == JsonToken.None && !ReadForType(reader, objectType, null))
+      JsonContract contract = GetContractSafe(objectType);
+
+      JsonConverter converter = GetConverter(contract, null);
+
+      if (reader.TokenType == JsonToken.None && !ReadForType(reader, contract, converter != null, false))
         return null;
 
-      return CreateValueNonProperty(reader, objectType, GetContractSafe(objectType));
+      return CreateValueNonProperty(reader, objectType, contract, converter);
     }
 
     private JsonSerializerProxy GetInternalSerializer()
@@ -181,12 +177,30 @@ namespace Newtonsoft.Json.Serialization
       return token;
     }
 
-    private object CreateValueProperty(JsonReader reader, JsonProperty property, object target, bool gottenCurrentValue, object currentValue)
+    private object CreateValueProperty(JsonReader reader, JsonProperty property, JsonConverter propertyConverter, object target, bool gottenCurrentValue, object currentValue)
     {
-      JsonContract contract = GetContractSafe(property.PropertyType, currentValue);
-      Type objectType = property.PropertyType;
+      JsonContract contract;
+      JsonConverter converter;
 
-      JsonConverter converter = GetConverter(contract, property.MemberConverter);
+      if (property.PropertyContract == null)
+        property.PropertyContract = GetContractSafe(property.PropertyType);
+
+      if (currentValue == null)
+      {
+        contract = property.PropertyContract;
+        converter = propertyConverter;
+      }
+      else
+      {
+        contract = GetContractSafe(currentValue.GetType());
+
+        if (contract != property.PropertyContract)
+          converter = GetConverter(contract, property.MemberConverter);
+        else
+          converter = propertyConverter;
+      }
+
+      Type objectType = property.PropertyType;
 
       if (converter != null && converter.CanRead)
       {
@@ -199,10 +213,8 @@ namespace Newtonsoft.Json.Serialization
       return CreateValueInternal(reader, objectType, contract, property, currentValue);
     }
 
-    private object CreateValueNonProperty(JsonReader reader, Type objectType, JsonContract contract)
+    private object CreateValueNonProperty(JsonReader reader, Type objectType, JsonContract contract, JsonConverter converter)
     {
-      JsonConverter converter = GetConverter(contract, null);
-
       if (converter != null && converter.CanRead)
         return converter.ReadJson(reader, objectType, null, GetInternalSerializer());
 
@@ -522,7 +534,7 @@ namespace Newtonsoft.Json.Serialization
       return value.ToString();
     }
 
-    private void SetPropertyValue(JsonProperty property, JsonReader reader, object target)
+    private void SetPropertyValue(JsonProperty property, JsonConverter propertyConverter, JsonReader reader, object target)
     {
       if (property.Ignored)
       {
@@ -573,7 +585,7 @@ namespace Newtonsoft.Json.Serialization
       }
 
       object existingValue = (useExistingValue) ? currentValue : null;
-      object value = CreateValueProperty(reader, property, target, gottenCurrentValue, existingValue);
+      object value = CreateValueProperty(reader, property, propertyConverter, target, gottenCurrentValue, existingValue);
 
       // always set the value if useExistingValue is false,
       // otherwise also set it if CreateValue returns a new value compared to the currentValue
@@ -651,10 +663,15 @@ namespace Newtonsoft.Json.Serialization
                 throw new JsonSerializationException("Could not convert string '{0}' to dictionary key type '{1}'. Create a TypeConverter to convert from the string to the key type object.".FormatWith(CultureInfo.InvariantCulture, reader.Value, contract.DictionaryKeyType), ex);
               }
 
-              if (!ReadForType(reader, contract.DictionaryValueType, null))
+              if (contract.DictionaryValueContract == null)
+                contract.DictionaryValueContract = GetContractSafe(contract.DictionaryValueType);
+
+              JsonConverter dictionaryValueConverter = GetConverter(contract.DictionaryValueContract, null);
+
+              if (!ReadForType(reader, contract.DictionaryValueContract, dictionaryValueConverter != null, false))
                 throw new JsonSerializationException("Unexpected end when deserializing object.");
 
-              dictionary[keyValue] = CreateValueNonProperty(reader, contract.DictionaryValueType, GetContractSafe(contract.DictionaryValueType));
+              dictionary[keyValue] = CreateValueNonProperty(reader, contract.DictionaryValueType, contract.DictionaryValueContract, dictionaryValueConverter);
             }
             catch (Exception ex)
             {
@@ -696,23 +713,6 @@ namespace Newtonsoft.Json.Serialization
         });
     }
 
-    private bool ReadForTypeArrayHack(JsonReader reader, Type t)
-    {
-      // this is a nasty hack because calling ReadAsDecimal for example will error when we hit the end of the array
-      // need to think of a better way of doing this
-      try
-      {
-        return ReadForType(reader, t, null);
-      }
-      catch (JsonReaderException)
-      {
-        if (reader.TokenType == JsonToken.EndArray)
-          return true;
-
-        throw;
-      }
-    }
-
     private object PopulateList(IWrappedCollection wrappedList, JsonReader reader, string reference, JsonArrayContract contract)
     {
       object list = wrappedList.UnderlyingCollection;
@@ -731,7 +731,10 @@ namespace Newtonsoft.Json.Serialization
 
       int initialDepth = reader.Depth;
 
-      while (ReadForTypeArrayHack(reader, contract.CollectionItemType))
+      JsonContract collectionItemContract = GetContractSafe(contract.CollectionItemType);
+      JsonConverter collectionItemConverter = GetConverter(collectionItemContract, null);
+
+      while (ReadForType(reader, collectionItemContract, collectionItemConverter != null, true))
       {
         switch (reader.TokenType)
         {
@@ -744,16 +747,20 @@ namespace Newtonsoft.Json.Serialization
           default:
             try
             {
-              object value = CreateValueNonProperty(reader, contract.CollectionItemType, GetContractSafe(contract.CollectionItemType));
+              object value = CreateValueNonProperty(reader, contract.CollectionItemType, collectionItemContract, collectionItemConverter);
 
               wrappedList.Add(value);
             }
             catch (Exception ex)
             {
               if (IsErrorHandled(list, contract, wrappedList.Count, ex))
+              {
                 HandleError(reader, initialDepth);
+              }
               else
+              {
                 throw;
+              }
             }
             break;
         }
@@ -843,15 +850,24 @@ namespace Newtonsoft.Json.Serialization
 
               // first attempt to find a settable property, otherwise fall back to a dynamic set without type
               JsonProperty property = contract.Properties.GetClosestMatchProperty(memberName);
+
               if (property != null && property.Writable && !property.Ignored)
               {
-                SetPropertyValue(property, reader, newObject);
+                if (property.PropertyContract == null)
+                  property.PropertyContract = GetContractSafe(property.PropertyType);
+
+                JsonConverter propertyConverter = GetConverter(property.PropertyContract, property.MemberConverter);
+
+                SetPropertyValue(property, propertyConverter, reader, newObject);
               }
               else
               {
                 Type t = (JsonReader.IsPrimitiveToken(reader.TokenType)) ? reader.ValueType : typeof (IDynamicMetaObjectProvider);
 
-                object value = CreateValueNonProperty(reader, t, GetContractSafe(t, null));
+                JsonContract dynamicMemberContract = GetContractSafe(t);
+                JsonConverter dynamicMemberConverter = GetConverter(dynamicMemberContract, null);
+
+                object value = CreateValueNonProperty(reader, t, dynamicMemberContract, dynamicMemberConverter);
 
                 newObject.TrySetMember(memberName, value);
               }
@@ -1008,11 +1024,16 @@ namespace Newtonsoft.Json.Serialization
 
             if (property != null)
             {
-              if (!ReadForType(reader, property.PropertyType, property.Converter))
+              if (property.PropertyContract == null)
+                property.PropertyContract = GetContractSafe(property.PropertyType);
+
+              JsonConverter propertyConverter = GetConverter(property.PropertyContract, property.MemberConverter);
+
+              if (!ReadForType(reader, property.PropertyContract, propertyConverter != null, false))
                 throw new JsonSerializationException("Unexpected end when setting {0}'s value.".FormatWith(CultureInfo.InvariantCulture, memberName));
 
               if (!property.Ignored)
-                propertyValues[property] = CreateValueProperty(reader, property, null, true, null);
+                propertyValues[property] = CreateValueProperty(reader, property, propertyConverter, null, true, null);
               else
                 reader.Skip();
             }
@@ -1040,29 +1061,44 @@ namespace Newtonsoft.Json.Serialization
       return propertyValues;
     }
 
-    private bool ReadForType(JsonReader reader, Type t, JsonConverter propertyConverter)
+    private void SetSerializeInArray(JsonReader reader, bool inArray, bool enable)
     {
+      if (inArray)
+        reader.SetSerializeInArray(enable);
+    }
+
+    private bool ReadForType(JsonReader reader, JsonContract contract, bool hasConverter, bool inArray)
+    {
+      Type t = (contract != null) ? contract.UnderlyingType : null;
+
       // don't read properties with converters as a specific value
       // the value might be a string which will then get converted which will error if read as date for example
-      bool hasConverter = (GetConverter(GetContractSafe(t), propertyConverter) != null);
-
       if (hasConverter)
         return reader.Read();
 
       if (t == typeof (byte[]))
       {
+        SetSerializeInArray(reader, inArray, true);
         reader.ReadAsBytes();
+        SetSerializeInArray(reader, inArray, false);
+
         return true;
       }
       else if ((t == typeof (decimal) || t == typeof (decimal?)))
       {
+        SetSerializeInArray(reader, inArray, true);
         reader.ReadAsDecimal();
+        SetSerializeInArray(reader, inArray, false);
+
         return true;
       }
 #if !NET20
       else if ((t == typeof (DateTimeOffset) || t == typeof (DateTimeOffset?)))
       {
+        SetSerializeInArray(reader, inArray, true);
         reader.ReadAsDateTimeOffset();
+        SetSerializeInArray(reader, inArray, false);
+
         return true;
       }
 #endif
@@ -1110,12 +1146,17 @@ namespace Newtonsoft.Json.Serialization
                 continue;
               }
 
-              if (!ReadForType(reader, property.PropertyType, property.Converter))
+              if (property.PropertyContract == null)
+                property.PropertyContract = GetContractSafe(property.PropertyType);
+
+              JsonConverter propertyConverter = GetConverter(property.PropertyContract, property.MemberConverter);
+
+              if (!ReadForType(reader, property.PropertyContract, propertyConverter != null, false))
                 throw new JsonSerializationException("Unexpected end when setting {0}'s value.".FormatWith(CultureInfo.InvariantCulture, memberName));
 
               SetPropertyPresence(reader, property, propertiesPresence);
 
-              SetPropertyValue(property, reader, newObject);
+              SetPropertyValue(property, propertyConverter, reader, newObject);
             }
             catch (Exception ex)
             {

Src/Newtonsoft.Json/Serialization/JsonSerializerInternalWriter.cs

@@ -123,14 +123,14 @@ namespace Newtonsoft.Json.Serialization
       {
         SerializeConvertable(writer, converter, value, valueContract);
       }
-      else if (valueContract is JsonPrimitiveContract)
-      {
-        SerializePrimitive(writer, value, (JsonPrimitiveContract)valueContract, member, collectionValueContract);
-      }
       else if (valueContract is JsonStringContract)
       {
         SerializeString(writer, value, (JsonStringContract)valueContract);
       }
+      else if (valueContract is JsonPrimitiveContract)
+      {
+        SerializePrimitive(writer, value, (JsonPrimitiveContract)valueContract, member, collectionValueContract);
+      }
       else if (valueContract is JsonObjectContract)
       {
         SerializeObject(writer, value, (JsonObjectContract)valueContract, member, collectionValueContract);
@@ -333,8 +333,11 @@ namespace Newtonsoft.Json.Serialization
         {
           if (!property.Ignored && property.Readable && ShouldSerialize(property, value) && IsSpecified(property, value))
           {
+            if (property.PropertyContract == null)
+              property.PropertyContract = Serializer.ContractResolver.ResolveContract(property.PropertyType);
+
             object memberValue = property.ValueProvider.GetValue(value);
-            JsonContract memberContract = GetContractSafe(memberValue);
+            JsonContract memberContract = (property.PropertyContract.UnderlyingType.IsSealed) ? property.PropertyContract : GetContractSafe(memberValue);
 
             WriteMemberInfoProperty(writer, memberValue, property, memberContract);
           }
@@ -419,7 +422,10 @@ namespace Newtonsoft.Json.Serialization
         writer.WritePropertyName(JsonTypeReflector.ArrayValuesPropertyName);
       }
 
-      JsonContract childValuesContract = Serializer.ContractResolver.ResolveContract(contract.CollectionItemType ?? typeof(object));
+      if (contract.CollectionItemContract == null)
+        contract.CollectionItemContract = Serializer.ContractResolver.ResolveContract(contract.CollectionItemType ?? typeof(object));
+
+      JsonContract collectionItemValueContract = (contract.CollectionItemContract.UnderlyingType.IsSealed) ? contract.CollectionItemContract : null;
 
       writer.WriteStartArray();
 
@@ -431,7 +437,7 @@ namespace Newtonsoft.Json.Serialization
       {
         try
         {
-          JsonContract valueContract = GetContractSafe(value);
+          JsonContract valueContract = collectionItemValueContract ?? GetContractSafe(value);
 
           if (ShouldWriteReference(value, null, valueContract))
           {
@@ -441,7 +447,7 @@ namespace Newtonsoft.Json.Serialization
           {
             if (CheckForCircularReference(value, null, contract))
             {
-              SerializeValue(writer, value, valueContract, null, childValuesContract);
+              SerializeValue(writer, value, valueContract, null, contract.CollectionItemContract);
             }
           }
         }
@@ -582,7 +588,10 @@ namespace Newtonsoft.Json.Serialization
         WriteTypeProperty(writer, values.UnderlyingDictionary.GetType());
       }
 
-      JsonContract childValuesContract = Serializer.ContractResolver.ResolveContract(contract.DictionaryValueType ?? typeof(object));
+      if (contract.DictionaryValueContract == null)
+        contract.DictionaryValueContract = Serializer.ContractResolver.ResolveContract(contract.DictionaryValueType ?? typeof(object));
+
+      JsonContract dictionaryValueContract = (contract.DictionaryValueContract.UnderlyingType.IsSealed) ? contract.DictionaryValueContract : null;
 
       int initialDepth = writer.Top;
 
@@ -600,7 +609,7 @@ namespace Newtonsoft.Json.Serialization
         try
         {
           object value = entry.Value;
-          JsonContract valueContract = GetContractSafe(value);
+          JsonContract valueContract = dictionaryValueContract ?? GetContractSafe(value);
 
           if (ShouldWriteReference(value, null, valueContract))
           {
@@ -614,7 +623,7 @@ namespace Newtonsoft.Json.Serialization
 
             writer.WritePropertyName(propertyName);
 
-            SerializeValue(writer, value, valueContract, null, childValuesContract);
+            SerializeValue(writer, value, valueContract, null, contract.DictionaryValueContract);
           }
         }
         catch (Exception ex)

Src/Newtonsoft.Json/Serialization/JsonStringContract.cs

@@ -33,7 +33,7 @@ namespace Newtonsoft.Json.Serialization
   /// <summary>
   /// Contract details for a <see cref="Type"/> used by the <see cref="JsonSerializer"/>.
   /// </summary>
-  public class JsonStringContract : JsonContract
+  public class JsonStringContract : JsonPrimitiveContract
   {
     /// <summary>
     /// Initializes a new instance of the <see cref="JsonStringContract"/> class.

Src/Newtonsoft.Json/Utilities/StringBuffer.cs

@@ -57,14 +57,22 @@ namespace Newtonsoft.Json.Utilities
     {
       // test if the buffer array is large enough to take the value
       if (_position == _buffer.Length)
-      {
         EnsureSize(1);
-      }
 
       // set value and increment poisition
       _buffer[_position++] = value;
     }
 
+    public void Append(char[] buffer, int startIndex, int count)
+    {
+      if (_position + count >= _buffer.Length)
+        EnsureSize(count);
+
+      Array.Copy(buffer, startIndex, _buffer, _position, count);
+
+      _position += count;
+    }
+
     public void Clear()
     {
       _buffer = _emptyBuffer;

Src/Newtonsoft.Json/Utilities/StringReference.cs

@@ -0,0 +1,41 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Newtonsoft.Json.Utilities
+{
+  internal struct StringReference
+  {
+    private readonly char[] _chars;
+    private readonly int _startIndex;
+    private readonly int _length;
+
+    public char[] Chars
+    {
+      get { return _chars; }
+    }
+
+    public int StartIndex
+    {
+      get { return _startIndex; }
+    }
+
+    public int Length
+    {
+      get { return _length; }
+    }
+
+    public StringReference(char[] chars, int startIndex, int length)
+    {
+      _chars = chars;
+      _startIndex = startIndex;
+      _length = length;
+    }
+
+    public override string ToString()
+    {
+      return new string(_chars, _startIndex, _length);
+    }
+  }
+}