From 17f055e9441747c7c8952cf2207dbafe0697cf0b Mon Sep 17 00:00:00 2001 From: Tomasz Romaszkiewicz Date: Fri, 6 Oct 2017 19:47:30 +0200 Subject: [PATCH 1/2] Update JsonTextReader.cs We are doubling the _chars.Length when calculation newArrayLength, thus it's worth to check if we are not out of int.MaxValue. This fix allows to work on bigger files. --- Src/Newtonsoft.Json/JsonTextReader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/Newtonsoft.Json/JsonTextReader.cs b/Src/Newtonsoft.Json/JsonTextReader.cs index e04aa0f8..8d568dbd 100644 --- a/Src/Newtonsoft.Json/JsonTextReader.cs +++ b/Src/Newtonsoft.Json/JsonTextReader.cs @@ -240,7 +240,7 @@ namespace Newtonsoft.Json // once in the last 10% of the buffer shift the remaining content to the start to avoid // unnecessarily increasing the buffer size when reading numbers/strings int length = _chars.Length; - if (length - _charPos <= length * 0.1) + if (length - _charPos <= length * 0.1 || length > int.MaxValue / 2 - 1) { int count = _charsUsed - _charPos; if (count > 0) From 6330ea5d1446890c206864d6f5698d0e4e8a1146 Mon Sep 17 00:00:00 2001 From: Tomasz Romaszkiewicz Date: Sat, 7 Oct 2017 14:01:04 +0200 Subject: [PATCH 2/2] Added Unit test and allowed GC large objects. --- Src/Newtonsoft.Json.TestConsole/App.config | 3 ++ .../JsonTextReaderTests/ReadTests.cs | 40 +++++++++++++++++++ Src/Newtonsoft.Json/JsonTextReader.cs | 1 + 3 files changed, 44 insertions(+) diff --git a/Src/Newtonsoft.Json.TestConsole/App.config b/Src/Newtonsoft.Json.TestConsole/App.config index 5c73fc53..1893e583 100644 --- a/Src/Newtonsoft.Json.TestConsole/App.config +++ b/Src/Newtonsoft.Json.TestConsole/App.config @@ -4,4 +4,7 @@ + + + \ No newline at end of file diff --git a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ReadTests.cs b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ReadTests.cs index 30fa4e00..c895bc11 100644 --- a/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ReadTests.cs +++ b/Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ReadTests.cs @@ -273,6 +273,46 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests Assert.AreEqual(JsonToken.EndObject, reader.TokenType); } + [Test] + public void ReadLargeObjects() + { + const int nrItems = 2; + const int length = int.MaxValue / 3; + + byte apostrophe = Encoding.ASCII.GetBytes(@"""").First(); + byte openingBracket = Encoding.ASCII.GetBytes(@"[").First(); + byte comma = Encoding.ASCII.GetBytes(@",").First(); + byte closingBracket = Encoding.ASCII.GetBytes(@"]").First(); + + using (MemoryStream ms = new MemoryStream()) + { + ms.WriteByte(openingBracket); + for (int i = 0; i < nrItems; i++) + { + ms.WriteByte(apostrophe); + + for (int j = 0; j <= length; j++) + ms.WriteByte(7); + + ms.WriteByte(apostrophe); + if (i < nrItems - 1) + ms.WriteByte(comma); + } + ms.WriteByte(closingBracket); + ms.Seek(0, SeekOrigin.Begin); + + var reader = new JsonTextReader(new StreamReader(ms)); + + Assert.IsTrue(reader.Read()); + + for (int i = 0; i < nrItems; i++) + Assert.IsTrue(reader.Read()); + + Assert.IsTrue(reader.Read()); + Assert.IsFalse(reader.Read()); + } + } + [Test] public void ReadSingleBytes() { diff --git a/Src/Newtonsoft.Json/JsonTextReader.cs b/Src/Newtonsoft.Json/JsonTextReader.cs index 8d568dbd..81a80b38 100644 --- a/Src/Newtonsoft.Json/JsonTextReader.cs +++ b/Src/Newtonsoft.Json/JsonTextReader.cs @@ -239,6 +239,7 @@ namespace Newtonsoft.Json { // once in the last 10% of the buffer shift the remaining content to the start to avoid // unnecessarily increasing the buffer size when reading numbers/strings + //romasz: do it also if we are reading large objects that can cause the buffer's new length to exceed int.MaxValue int length = _chars.Length; if (length - _charPos <= length * 0.1 || length > int.MaxValue / 2 - 1) {