-Refactor

-Tests

Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ExceptionHandlingAsyncTests.cs

@@ -401,7 +401,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[2]);
+            reader.CharBuffer = new char[2];
 #endif
 
             await reader.ReadAsync();

Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ExceptionHandlingTests.cs

@@ -424,7 +424,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[2]);
+            reader.CharBuffer = new char[2];
 #endif
 
             reader.Read();

Src/Newtonsoft.Json.Tests/JsonTextReaderTests/MiscAsyncTests.cs

@@ -460,7 +460,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[129]);
+            reader.CharBuffer = new char[129];
 #endif
 
             for (int i = 0; i < 15; i++)
@@ -492,7 +492,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[129]);
+            reader.CharBuffer = new char[129];
 #endif
 
             for (int i = 0; i < 14; i++)
@@ -665,7 +665,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[2]);
+            reader.CharBuffer = new char[2];
 #endif
 
             await reader.ReadAsync();

Src/Newtonsoft.Json.Tests/JsonTextReaderTests/MiscTests.cs

@@ -494,7 +494,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[129]);
+            reader.CharBuffer = new char[129];
 #endif
 
             for (int i = 0; i < 15; i++)
@@ -526,7 +526,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[129]);
+            reader.CharBuffer = new char[129];
 #endif
 
             for (int i = 0; i < 14; i++)
@@ -699,7 +699,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[2]);
+            reader.CharBuffer = new char[2];
 #endif
 
             reader.Read();

Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ParseAsyncTests.cs

@@ -398,7 +398,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
             string json = "new Date\0()";
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[7]);
+            reader.CharBuffer = new char[7];
 #endif
 
             Assert.IsTrue(await reader.ReadAsync());

Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ParseTests.cs

@@ -418,7 +418,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
             string json = "new Date\0()";
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[7]);
+            reader.CharBuffer = new char[7];
 #endif
 
             Assert.IsTrue(reader.Read());

Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ReadAsyncTests.cs

@@ -114,7 +114,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[10]);
+            reader.CharBuffer = new char[10];
 #endif
 
             Assert.IsTrue(await reader.ReadAsync());
@@ -227,7 +227,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[5]);
+            reader.CharBuffer = new char[5];
 #endif
 
             Assert.IsTrue(await reader.ReadAsync());
@@ -268,6 +268,65 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
             Assert.AreEqual(JsonToken.EndObject, reader.TokenType);
         }
 
+#if DEBUG
+        [Test]
+        public async Task ReadLargeObjectsAsync()
+        {
+            const int nrItems = 2;
+            const int length = 1200;
+            const int largeBufferLength = 2048;
+
+            byte apostrophe = Encoding.ASCII.GetBytes(@"""").First();
+            byte openingBracket = Encoding.ASCII.GetBytes(@"[").First();
+            byte comma = Encoding.ASCII.GetBytes(@",").First();
+            byte closingBracket = Encoding.ASCII.GetBytes(@"]").First();
+
+            using (MemoryStream ms = new MemoryStream())
+            {
+                ms.WriteByte(openingBracket);
+                for (int i = 0; i < nrItems; i++)
+                {
+                    ms.WriteByte(apostrophe);
+
+                    for (int j = 0; j <= length; j++)
+                    {
+                        byte current = Convert.ToByte((j % 10) + 48);
+                        ms.WriteByte(current);
+                    }
+
+                    ms.WriteByte(apostrophe);
+                    if (i < nrItems - 1)
+                    {
+                        ms.WriteByte(comma);
+                    }
+                }
+
+                ms.WriteByte(closingBracket);
+                ms.Seek(0, SeekOrigin.Begin);
+
+                JsonTextReader reader = new JsonTextReader(new StreamReader(ms));
+                reader.LargeBufferLength = largeBufferLength;
+
+                Assert.IsTrue(await reader.ReadAsync());
+                Assert.AreEqual(JsonToken.StartArray, reader.TokenType);
+
+                Assert.IsTrue(await reader.ReadAsync());
+                Assert.AreEqual(JsonToken.String, reader.TokenType);
+                Assert.AreEqual(largeBufferLength, reader.CharBuffer.Length);
+
+                Assert.IsTrue(await reader.ReadAsync());
+                Assert.AreEqual(JsonToken.String, reader.TokenType);
+                // buffer has been shifted before reading the second string
+                Assert.AreEqual(largeBufferLength, reader.CharBuffer.Length);
+
+                Assert.IsTrue(await reader.ReadAsync());
+                Assert.AreEqual(JsonToken.EndArray, reader.TokenType);
+
+                Assert.IsFalse(await reader.ReadAsync());
+            }
+        }
+#endif
+
         [Test]
         public async Task ReadSingleBytesAsync()
         {
@@ -675,7 +734,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[5]);
+            reader.CharBuffer = new char[5];
 #endif
 
             for (int i = 0; i < 13; i++)
@@ -703,7 +762,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[5]);
+            reader.CharBuffer = new char[5];
 #endif
 
             for (int i = 0; i < 26; i++)
@@ -1248,7 +1307,7 @@ third line", jsonTextReader.Value);
             using (JsonTextReader jsonReader = new JsonTextReader(sr))
             {
 #if DEBUG
-                jsonReader.SetCharBuffer(new char[5]);
+                jsonReader.CharBuffer = new char[5];
 #endif
 
                 Assert.AreEqual(jsonReader.TokenType, JsonToken.None);

Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ReadTests.cs

@@ -119,7 +119,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[10]);
+            reader.CharBuffer = new char[10];
 #endif
 
             Assert.IsTrue(reader.Read());
@@ -232,7 +232,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[5]);
+            reader.CharBuffer = new char[5];
 #endif
 
             Assert.IsTrue(reader.Read());
@@ -273,11 +273,13 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
             Assert.AreEqual(JsonToken.EndObject, reader.TokenType);
         }
 
+#if !(NET20 || NET35) && DEBUG
         [Test]
         public void ReadLargeObjects()
         {
             const int nrItems = 2;
-            const int length = int.MaxValue / 3;
+            const int length = 1200;
+            const int largeBufferLength = 2048;
 
             byte apostrophe = Encoding.ASCII.GetBytes(@"""").First();
             byte openingBracket = Encoding.ASCII.GetBytes(@"[").First();
@@ -292,26 +294,43 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
                     ms.WriteByte(apostrophe);
 
                     for (int j = 0; j <= length; j++)
-                        ms.WriteByte(7);
+                    {
+                        byte current = Convert.ToByte((j % 10) + 48);
+                        ms.WriteByte(current);
+                    }
 
                     ms.WriteByte(apostrophe);
                     if (i < nrItems - 1)
+                    {
                         ms.WriteByte(comma);
+                    }
                 }
+
                 ms.WriteByte(closingBracket);
                 ms.Seek(0, SeekOrigin.Begin);
 
-                var reader = new JsonTextReader(new StreamReader(ms));
+                JsonTextReader reader = new JsonTextReader(new StreamReader(ms));
+                reader.LargeBufferLength = largeBufferLength;
 
                 Assert.IsTrue(reader.Read());
-
+                Assert.AreEqual(JsonToken.StartArray, reader.TokenType);
-                for (int i = 0; i < nrItems; i++)
-                    Assert.IsTrue(reader.Read());
 
                 Assert.IsTrue(reader.Read());
+                Assert.AreEqual(JsonToken.String, reader.TokenType);
+                Assert.AreEqual(largeBufferLength, reader.CharBuffer.Length);
+
+                Assert.IsTrue(reader.Read());
+                Assert.AreEqual(JsonToken.String, reader.TokenType);
+                // buffer has been shifted before reading the second string
+                Assert.AreEqual(largeBufferLength, reader.CharBuffer.Length);
+
+                Assert.IsTrue(reader.Read());
+                Assert.AreEqual(JsonToken.EndArray, reader.TokenType);
+
                 Assert.IsFalse(reader.Read());
             }
         }
+#endif
 
         [Test]
         public void ReadSingleBytes()
@@ -732,7 +751,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[5]);
+            reader.CharBuffer = new char[5];
 #endif
 
             for (int i = 0; i < 13; i++)
@@ -760,7 +779,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 
             JsonTextReader reader = new JsonTextReader(new StringReader(json));
 #if DEBUG
-            reader.SetCharBuffer(new char[5]);
+            reader.CharBuffer = new char[5];
 #endif
 
             for (int i = 0; i < 26; i++)
@@ -1308,7 +1327,7 @@ third line", jsonTextReader.Value);
             using (JsonTextReader jsonReader = new JsonTextReader(sr))
             {
 #if DEBUG
-                jsonReader.SetCharBuffer(new char[5]);
+                jsonReader.CharBuffer = new char[5];
 #endif
 
                 Assert.AreEqual(jsonReader.TokenType, JsonToken.None);

Src/Newtonsoft.Json/JsonTextReader.cs

@@ -58,6 +58,12 @@ namespace Newtonsoft.Json
         private const char UnicodeReplacementChar = '\uFFFD';
         private const int MaximumJavascriptIntegerCharacterLength = 380;
 
+#if DEBUG
+        internal int LargeBufferLength { get; set; } = int.MaxValue / 2;
+#else
+        private const int LargeBufferLength = int.MaxValue / 2;
+#endif
+
         private readonly TextReader _reader;
         private char[] _chars;
         private int _charsUsed;
@@ -90,9 +96,15 @@ namespace Newtonsoft.Json
         }
 
 #if DEBUG
-        internal void SetCharBuffer(char[] chars)
+        internal char[] CharBuffer
         {
-            _chars = chars;
+            get => _chars;
+            set => _chars = value;
+        }
+
+        internal int CharPos
+        {
+            get => _charPos;
         }
 #endif
 
@@ -235,11 +247,11 @@ namespace Newtonsoft.Json
 
         private void ShiftBufferIfNeeded()
         {
-            // once in the last 10% of the buffer shift the remaining content to the start to avoid
+            // once in the last 10% of the buffer, or buffer is already very large then
-            // unnecessarily increasing the buffer size when reading numbers/strings
+            // shift the remaining content to the start to avoid unnecessarily increasing
-            //romasz: do it also if we are reading large objects that can cause the buffer's new length to exceed int.MaxValue
+            // the buffer size when reading numbers/strings
             int length = _chars.Length;
-            if (length - _charPos <= length * 0.1 || length > int.MaxValue / 2 - 1)
+            if (length - _charPos <= length * 0.1 || length >= LargeBufferLength)
             {
                 int count = _charsUsed - _charPos;
                 if (count > 0)
@@ -266,8 +278,12 @@ namespace Newtonsoft.Json
             {
                 if (append)
                 {
+                    int doubledArrayLength = _chars.Length * 2;
+
                     // copy to new array either double the size of the current or big enough to fit required content
-                    int newArrayLength = Math.Max(_chars.Length * 2, _charsUsed + charsRequired + 1);
+                    int newArrayLength = Math.Max(
+                        doubledArrayLength < 0 ? int.MaxValue : doubledArrayLength, // handle overflow
+                        _charsUsed + charsRequired + 1);
 
                     // increase the size of the buffer
                     char[] dst = BufferUtils.RentBuffer(_arrayPool, newArrayLength);
@@ -1948,8 +1964,7 @@ namespace Newtonsoft.Json
                 }
                 else
                 {
-                    double value;
+                    if (!double.TryParse(number, NumberStyles.Float, CultureInfo.InvariantCulture, out _))
-                    if (!double.TryParse(number, NumberStyles.Float, CultureInfo.InvariantCulture, out value))
                     {
                         throw ThrowReaderError("Input string '{0}' is not a valid number.".FormatWith(CultureInfo.InvariantCulture, _stringReference.ToString()));
                     }