diff --git a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ExceptionHandlingAsyncTests.cs b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ExceptionHandlingAsyncTests.cs index 4ff011dc..fa2d9a50 100644 --- a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ExceptionHandlingAsyncTests.cs +++ b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ExceptionHandlingAsyncTests.cs @@ -401,7 +401,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[2]); + reader.CharBuffer = new char[2]; #endif await reader.ReadAsync(); diff --git a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ExceptionHandlingTests.cs b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ExceptionHandlingTests.cs index 59081357..830deb55 100644 --- a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ExceptionHandlingTests.cs +++ b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ExceptionHandlingTests.cs @@ -424,7 +424,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[2]); + reader.CharBuffer = new char[2]; #endif reader.Read(); diff --git a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/MiscAsyncTests.cs b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/MiscAsyncTests.cs index 2c36271f..df798e39 100644 --- a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/MiscAsyncTests.cs +++ b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/MiscAsyncTests.cs @@ -460,7 +460,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[129]); + reader.CharBuffer = new char[129]; #endif for (int i = 0; i < 15; i++) @@ -492,7 +492,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[129]); + reader.CharBuffer = new char[129]; #endif for (int i = 0; i < 14; i++) @@ -665,7 +665,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[2]); + reader.CharBuffer = new char[2]; #endif await reader.ReadAsync(); diff --git a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/MiscTests.cs b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/MiscTests.cs index 1793d58f..f25590e7 100644 --- a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/MiscTests.cs +++ b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/MiscTests.cs @@ -494,7 +494,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[129]); + reader.CharBuffer = new char[129]; #endif for (int i = 0; i < 15; i++) @@ -526,7 +526,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[129]); + reader.CharBuffer = new char[129]; #endif for (int i = 0; i < 14; i++) @@ -699,7 +699,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[2]); + reader.CharBuffer = new char[2]; #endif reader.Read(); diff --git a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ParseAsyncTests.cs b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ParseAsyncTests.cs index 5225eaaa..e1a6c69c 100644 --- a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ParseAsyncTests.cs +++ b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ParseAsyncTests.cs @@ -398,7 +398,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests string json = "new Date\0()"; JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[7]); + reader.CharBuffer = new char[7]; #endif Assert.IsTrue(await reader.ReadAsync()); diff --git a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ParseTests.cs b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ParseTests.cs index bb2344a4..5b18955a 100644 --- a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ParseTests.cs +++ b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ParseTests.cs @@ -418,7 +418,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests string json = "new Date\0()"; JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[7]); + reader.CharBuffer = new char[7]; #endif Assert.IsTrue(reader.Read()); diff --git a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ReadAsyncTests.cs b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ReadAsyncTests.cs index c6578785..d5844266 100644 --- a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ReadAsyncTests.cs +++ b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ReadAsyncTests.cs @@ -114,7 +114,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[10]); + reader.CharBuffer = new char[10]; #endif Assert.IsTrue(await reader.ReadAsync()); @@ -227,7 +227,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[5]); + reader.CharBuffer = new char[5]; #endif Assert.IsTrue(await reader.ReadAsync()); @@ -268,6 +268,65 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests Assert.AreEqual(JsonToken.EndObject, reader.TokenType); } +#if DEBUG + [Test] + public async Task ReadLargeObjectsAsync() + { + const int nrItems = 2; + const int length = 1200; + const int largeBufferLength = 2048; + + byte apostrophe = Encoding.ASCII.GetBytes(@"""").First(); + byte openingBracket = Encoding.ASCII.GetBytes(@"[").First(); + byte comma = Encoding.ASCII.GetBytes(@",").First(); + byte closingBracket = Encoding.ASCII.GetBytes(@"]").First(); + + using (MemoryStream ms = new MemoryStream()) + { + ms.WriteByte(openingBracket); + for (int i = 0; i < nrItems; i++) + { + ms.WriteByte(apostrophe); + + for (int j = 0; j <= length; j++) + { + byte current = Convert.ToByte((j % 10) + 48); + ms.WriteByte(current); + } + + ms.WriteByte(apostrophe); + if (i < nrItems - 1) + { + ms.WriteByte(comma); + } + } + + ms.WriteByte(closingBracket); + ms.Seek(0, SeekOrigin.Begin); + + JsonTextReader reader = new JsonTextReader(new StreamReader(ms)); + reader.LargeBufferLength = largeBufferLength; + + Assert.IsTrue(await reader.ReadAsync()); + Assert.AreEqual(JsonToken.StartArray, reader.TokenType); + + Assert.IsTrue(await reader.ReadAsync()); + Assert.AreEqual(JsonToken.String, reader.TokenType); + Assert.AreEqual(largeBufferLength, reader.CharBuffer.Length); + + Assert.IsTrue(await reader.ReadAsync()); + Assert.AreEqual(JsonToken.String, reader.TokenType); + // buffer has been shifted before reading the second string + Assert.AreEqual(largeBufferLength, reader.CharBuffer.Length); + + Assert.IsTrue(await reader.ReadAsync()); + Assert.AreEqual(JsonToken.EndArray, reader.TokenType); + + Assert.IsFalse(await reader.ReadAsync()); + } + } +#endif + [Test] public async Task ReadSingleBytesAsync() { @@ -675,7 +734,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[5]); + reader.CharBuffer = new char[5]; #endif for (int i = 0; i < 13; i++) @@ -703,7 +762,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[5]); + reader.CharBuffer = new char[5]; #endif for (int i = 0; i < 26; i++) @@ -1248,7 +1307,7 @@ third line", jsonTextReader.Value); using (JsonTextReader jsonReader = new JsonTextReader(sr)) { #if DEBUG - jsonReader.SetCharBuffer(new char[5]); + jsonReader.CharBuffer = new char[5]; #endif Assert.AreEqual(jsonReader.TokenType, JsonToken.None); diff --git a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ReadTests.cs b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ReadTests.cs index c895bc11..b389fa20 100644 --- a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ReadTests.cs +++ b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ReadTests.cs @@ -119,7 +119,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[10]); + reader.CharBuffer = new char[10]; #endif Assert.IsTrue(reader.Read()); @@ -232,7 +232,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[5]); + reader.CharBuffer = new char[5]; #endif Assert.IsTrue(reader.Read()); @@ -273,11 +273,13 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests Assert.AreEqual(JsonToken.EndObject, reader.TokenType); } +#if !(NET20 || NET35) && DEBUG [Test] public void ReadLargeObjects() { const int nrItems = 2; - const int length = int.MaxValue / 3; + const int length = 1200; + const int largeBufferLength = 2048; byte apostrophe = Encoding.ASCII.GetBytes(@"""").First(); byte openingBracket = Encoding.ASCII.GetBytes(@"[").First(); @@ -292,26 +294,43 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests ms.WriteByte(apostrophe); for (int j = 0; j <= length; j++) - ms.WriteByte(7); + { + byte current = Convert.ToByte((j % 10) + 48); + ms.WriteByte(current); + } ms.WriteByte(apostrophe); if (i < nrItems - 1) + { ms.WriteByte(comma); + } } + ms.WriteByte(closingBracket); ms.Seek(0, SeekOrigin.Begin); - var reader = new JsonTextReader(new StreamReader(ms)); + JsonTextReader reader = new JsonTextReader(new StreamReader(ms)); + reader.LargeBufferLength = largeBufferLength; Assert.IsTrue(reader.Read()); - - for (int i = 0; i < nrItems; i++) - Assert.IsTrue(reader.Read()); + Assert.AreEqual(JsonToken.StartArray, reader.TokenType); Assert.IsTrue(reader.Read()); + Assert.AreEqual(JsonToken.String, reader.TokenType); + Assert.AreEqual(largeBufferLength, reader.CharBuffer.Length); + + Assert.IsTrue(reader.Read()); + Assert.AreEqual(JsonToken.String, reader.TokenType); + // buffer has been shifted before reading the second string + Assert.AreEqual(largeBufferLength, reader.CharBuffer.Length); + + Assert.IsTrue(reader.Read()); + Assert.AreEqual(JsonToken.EndArray, reader.TokenType); + Assert.IsFalse(reader.Read()); } } +#endif [Test] public void ReadSingleBytes() @@ -732,7 +751,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[5]); + reader.CharBuffer = new char[5]; #endif for (int i = 0; i < 13; i++) @@ -760,7 +779,7 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests JsonTextReader reader = new JsonTextReader(new StringReader(json)); #if DEBUG - reader.SetCharBuffer(new char[5]); + reader.CharBuffer = new char[5]; #endif for (int i = 0; i < 26; i++) @@ -1308,7 +1327,7 @@ third line", jsonTextReader.Value); using (JsonTextReader jsonReader = new JsonTextReader(sr)) { #if DEBUG - jsonReader.SetCharBuffer(new char[5]); + jsonReader.CharBuffer = new char[5]; #endif Assert.AreEqual(jsonReader.TokenType, JsonToken.None); diff --git a/Src/Newtonsoft.Json/JsonTextReader.cs b/Src/Newtonsoft.Json/JsonTextReader.cs index 7a282c0c..47a0e126 100644 --- a/Src/Newtonsoft.Json/JsonTextReader.cs +++ b/Src/Newtonsoft.Json/JsonTextReader.cs @@ -58,6 +58,12 @@ namespace Newtonsoft.Json private const char UnicodeReplacementChar = '\uFFFD'; private const int MaximumJavascriptIntegerCharacterLength = 380; +#if DEBUG + internal int LargeBufferLength { get; set; } = int.MaxValue / 2; +#else + private const int LargeBufferLength = int.MaxValue / 2; +#endif + private readonly TextReader _reader; private char[] _chars; private int _charsUsed; @@ -90,9 +96,15 @@ namespace Newtonsoft.Json } #if DEBUG - internal void SetCharBuffer(char[] chars) + internal char[] CharBuffer { - _chars = chars; + get => _chars; + set => _chars = value; + } + + internal int CharPos + { + get => _charPos; } #endif @@ -235,11 +247,11 @@ namespace Newtonsoft.Json private void ShiftBufferIfNeeded() { - // once in the last 10% of the buffer shift the remaining content to the start to avoid - // unnecessarily increasing the buffer size when reading numbers/strings - //romasz: do it also if we are reading large objects that can cause the buffer's new length to exceed int.MaxValue + // once in the last 10% of the buffer, or buffer is already very large then + // shift the remaining content to the start to avoid unnecessarily increasing + // the buffer size when reading numbers/strings int length = _chars.Length; - if (length - _charPos <= length * 0.1 || length > int.MaxValue / 2 - 1) + if (length - _charPos <= length * 0.1 || length >= LargeBufferLength) { int count = _charsUsed - _charPos; if (count > 0) @@ -266,8 +278,12 @@ namespace Newtonsoft.Json { if (append) { + int doubledArrayLength = _chars.Length * 2; + // copy to new array either double the size of the current or big enough to fit required content - int newArrayLength = Math.Max(_chars.Length * 2, _charsUsed + charsRequired + 1); + int newArrayLength = Math.Max( + doubledArrayLength < 0 ? int.MaxValue : doubledArrayLength, // handle overflow + _charsUsed + charsRequired + 1); // increase the size of the buffer char[] dst = BufferUtils.RentBuffer(_arrayPool, newArrayLength); @@ -1948,8 +1964,7 @@ namespace Newtonsoft.Json } else { - double value; - if (!double.TryParse(number, NumberStyles.Float, CultureInfo.InvariantCulture, out value)) + if (!double.TryParse(number, NumberStyles.Float, CultureInfo.InvariantCulture, out _)) { throw ThrowReaderError("Input string '{0}' is not a valid number.".FormatWith(CultureInfo.InvariantCulture, _stringReference.ToString())); }