From 7f290b5a857d301f93b4ea37bb4aa227fa66f227 Mon Sep 17 00:00:00 2001 From: James Newton-King Date: Sat, 26 Dec 2015 13:14:31 +1300 Subject: [PATCH] -JsonReader ReadAsBytes refactor --- .../JsonTextReaderTest.cs | 93 ++++++++++++- .../Serialization/JsonSerializerTest.cs | 18 +-- .../SerializationErrorHandlingTests.cs | 30 +++-- Src/Newtonsoft.Json/JsonReader.cs | 81 ++++++------ Src/Newtonsoft.Json/JsonTextReader.cs | 125 ++++++++++++++++-- 5 files changed, 272 insertions(+), 75 deletions(-) diff --git a/Src/Newtonsoft.Json.Tests/JsonTextReaderTest.cs b/Src/Newtonsoft.Json.Tests/JsonTextReaderTest.cs index 49fbc218..b382ab7f 100644 --- a/Src/Newtonsoft.Json.Tests/JsonTextReaderTest.cs +++ b/Src/Newtonsoft.Json.Tests/JsonTextReaderTest.cs @@ -1310,7 +1310,15 @@ third line", jsonTextReader.Value); { JsonReader reader = new JsonTextReader(new StringReader(@"true")); - ExceptionAssert.Throws(() => { reader.ReadAsBytes(); }, "Error reading bytes. Unexpected token: Boolean. Path '', line 1, position 4."); + ExceptionAssert.Throws(() => { reader.ReadAsBytes(); }, "Unexpected character encountered while parsing value: t. Path '', line 1, position 1."); + } + + [Test] + public void ReadInt32WithBadCharacter() + { + JsonReader reader = new JsonTextReader(new StringReader(@"true")); + + ExceptionAssert.Throws(() => { reader.ReadAsInt32(); }, "Unexpected character encountered while parsing value: t. Path '', line 1, position 1."); } [Test] @@ -2068,7 +2076,7 @@ third line", jsonTextReader.Value); { JsonTextReader reader = new JsonTextReader(new StringReader(@"{")); - ExceptionAssert.Throws(() => { reader.ReadAsBytes(); }, "Unexpected end when reading bytes. Path '', line 1, position 1."); + ExceptionAssert.Throws(() => { reader.ReadAsBytes(); }, "Unexpected end when reading JSON. Path '', line 1, position 1."); } #if !NET20 @@ -2104,7 +2112,7 @@ new Date()")); { JsonTextReader reader = new JsonTextReader(new StringReader(@"new Date()")); - ExceptionAssert.Throws(() => { reader.ReadAsBytes(); }, "Error reading bytes. Unexpected token: StartConstructor. Path '', line 1, position 9."); + ExceptionAssert.Throws(() => { reader.ReadAsBytes(); }, "Unexpected character encountered while parsing value: e. Path '', line 1, position 2."); } #if !NET20 @@ -2823,6 +2831,21 @@ new Date()")); Assert.IsTrue(reader.Read()); } + [Test] + public void ReadAsBytes_CommaErrors() + { + JsonTextReader reader = new JsonTextReader(new StringReader("[,'']")); + reader.Read(); + + ExceptionAssert.Throws(() => + { + reader.ReadAsBytes(); + }, "Unexpected character encountered while parsing value: ,. Path '[0]', line 1, position 2."); + + CollectionAssert.AreEquivalent(new byte[0], reader.ReadAsBytes()); + Assert.IsTrue(reader.Read()); + } + [Test] public void ReadNumberValue_InvalidEndArray() { @@ -2845,6 +2868,17 @@ new Date()")); }, "Unexpected character encountered while parsing value: ]. Path '', line 1, position 1."); } + [Test] + public void ReadAsBytes_InvalidEndArray() + { + JsonTextReader reader = new JsonTextReader(new StringReader("]")); + + ExceptionAssert.Throws(() => + { + reader.ReadAsBytes(); + }, "Unexpected character encountered while parsing value: ]. Path '', line 1, position 1."); + } + [Test] public void ReadNumberValue_CommaErrors_Multiple() { @@ -2861,6 +2895,22 @@ new Date()")); Assert.IsTrue(reader.Read()); } + [Test] + public void ReadAsBytes_CommaErrors_Multiple() + { + JsonTextReader reader = new JsonTextReader(new StringReader("['',,'']")); + reader.Read(); + CollectionAssert.AreEquivalent(new byte[0], reader.ReadAsBytes()); + + ExceptionAssert.Throws(() => + { + reader.ReadAsBytes(); + }, "Unexpected character encountered while parsing value: ,. Path '[1]', line 1, position 4."); + + CollectionAssert.AreEquivalent(new byte[0], reader.ReadAsBytes()); + Assert.IsTrue(reader.Read()); + } + [Test] public void ReadStringValue_CommaErrors() { @@ -2892,10 +2942,35 @@ new Date()")); Assert.IsTrue(reader.Read()); } + [Test] + public void ReadStringValue_Numbers_NotString() + { + JsonTextReader reader = new JsonTextReader(new StringReader("[56,56]")); + reader.Read(); + + ExceptionAssert.Throws(() => + { + reader.ReadAsDateTime(); + }, "Unexpected character encountered while parsing value: 5. Path '', line 1, position 2."); + + ExceptionAssert.Throws(() => + { + reader.ReadAsDateTime(); + }, "Unexpected character encountered while parsing value: 6. Path '', line 1, position 3."); + + ExceptionAssert.Throws(() => + { + reader.ReadAsDateTime(); + }, "Unexpected character encountered while parsing value: ,. Path '[0]', line 1, position 4."); + + Assert.AreEqual(56, reader.ReadAsInt32()); + Assert.IsTrue(reader.Read()); + } + [Test] public void ReadValue_EmptyString_Position() { - string json = @"['','','','','']"; + string json = @"['','','','','','']"; JsonTextReader reader = new JsonTextReader(new StringReader(json)); @@ -2910,18 +2985,23 @@ new Date()")); Assert.AreEqual("[3]", reader.Path); reader.ReadAsString(); Assert.AreEqual("[4]", reader.Path); + reader.ReadAsBytes(); + Assert.AreEqual("[5]", reader.Path); Assert.IsNull(reader.ReadAsString()); Assert.AreEqual(JsonToken.EndArray, reader.TokenType); Assert.IsNull(reader.ReadAsString()); Assert.AreEqual(JsonToken.None, reader.TokenType); + + Assert.IsNull(reader.ReadAsBytes()); + Assert.AreEqual(JsonToken.None, reader.TokenType); } [Test] public void ReadValueComments() { - string json = @"/*comment*/[/*comment*/1/*comment*/,2,/*comment*//*comment*/""three""/*comment*/,""four""/*comment*/,null,/*comment*/null,3.99,1.1/*comment*/]/*comment*/"; + string json = @"/*comment*/[/*comment*/1/*comment*/,2,/*comment*//*comment*/""three""/*comment*/,""four""/*comment*/,null,/*comment*/null,3.99,1.1/*comment*/,''/*comment*/]/*comment*/"; JsonTextReader reader = new JsonTextReader(new StreamReader(new SlowStream(json, new UTF8Encoding(false), 1))); @@ -2955,6 +3035,9 @@ new Date()")); Assert.AreEqual(1.1m, reader.ReadAsDecimal()); Assert.AreEqual(JsonToken.Float, reader.TokenType); + CollectionAssert.AreEquivalent(new byte[0], reader.ReadAsBytes()); + Assert.AreEqual(JsonToken.Bytes, reader.TokenType); + Assert.IsTrue(reader.Read()); Assert.AreEqual(JsonToken.Comment, reader.TokenType); diff --git a/Src/Newtonsoft.Json.Tests/Serialization/JsonSerializerTest.cs b/Src/Newtonsoft.Json.Tests/Serialization/JsonSerializerTest.cs index 427e2565..e10cfef8 100644 --- a/Src/Newtonsoft.Json.Tests/Serialization/JsonSerializerTest.cs +++ b/Src/Newtonsoft.Json.Tests/Serialization/JsonSerializerTest.cs @@ -6692,20 +6692,20 @@ Path '', line 1, position 1."); serializer.TypeNameHandling = TypeNameHandling.All; byte[] objectBytes; - using (MemoryStream bsonStream = new MemoryStream()) - using (JsonWriter bsonWriter = new JsonTextWriter(new StreamWriter(bsonStream))) + using (MemoryStream stream = new MemoryStream()) + using (JsonWriter jsonWriter = new JsonTextWriter(new StreamWriter(stream))) { - serializer.Serialize(bsonWriter, test); - bsonWriter.Flush(); + serializer.Serialize(jsonWriter, test); + jsonWriter.Flush(); - objectBytes = bsonStream.ToArray(); + objectBytes = stream.ToArray(); } - using (MemoryStream bsonStream = new MemoryStream(objectBytes)) - using (JsonReader bsonReader = new JsonTextReader(new StreamReader(bsonStream))) + using (MemoryStream stream = new MemoryStream(objectBytes)) + using (JsonReader jsonReader = new JsonTextReader(new StreamReader(stream))) { // Get exception here - TestObject newObject = (TestObject)serializer.Deserialize(bsonReader); + TestObject newObject = (TestObject)serializer.Deserialize(jsonReader); Assert.AreEqual("Test", newObject.Name); CollectionAssert.AreEquivalent(new byte[] { 72, 63, 62, 71, 92, 55 }, newObject.Data); @@ -6964,7 +6964,7 @@ Path '', line 1, position 1."); }"; JsonConvert.DeserializeObject(json); - }, "Unexpected character encountered while parsing value: t. Path 'PreProperty', line 2, position 17."); + }, "Unexpected character encountered while parsing value: t. Path 'PreProperty', line 2, position 18."); } [Test] diff --git a/Src/Newtonsoft.Json.Tests/Serialization/SerializationErrorHandlingTests.cs b/Src/Newtonsoft.Json.Tests/Serialization/SerializationErrorHandlingTests.cs index 24e61fd0..c5104398 100644 --- a/Src/Newtonsoft.Json.Tests/Serialization/SerializationErrorHandlingTests.cs +++ b/Src/Newtonsoft.Json.Tests/Serialization/SerializationErrorHandlingTests.cs @@ -522,7 +522,7 @@ namespace Newtonsoft.Json.Tests.Serialization } [Test] - public void InfiniteLoopArrayHandling() + public void ArrayHandling() { IList errors = new List(); @@ -538,16 +538,17 @@ namespace Newtonsoft.Json.Tests.Serialization } }); - Assert.IsNull(o); + Assert.IsNotNull(o); - Assert.AreEqual(3, errors.Count); - Assert.AreEqual("Unexpected character encountered while parsing value: x. Path '[0]', line 1, position 3.", errors[0]); - Assert.AreEqual("Unexpected character encountered while parsing value: x. Path '[0]', line 1, position 3.", errors[1]); - Assert.AreEqual("Infinite loop detected from error handling. Path '[0]', line 1, position 3.", errors[2]); + Assert.AreEqual(1, errors.Count); + Assert.AreEqual("Unexpected character encountered while parsing value: x. Path '[0]', line 1, position 4.", errors[0]); + + Assert.AreEqual(1, ((int[])o).Length); + Assert.AreEqual(0, ((int[])o)[0]); } [Test] - public void InfiniteLoopArrayHandlingInObject() + public void ArrayHandlingInObject() { IList errors = new List(); @@ -563,13 +564,16 @@ namespace Newtonsoft.Json.Tests.Serialization } }); - Assert.IsNull(o); + Assert.IsNotNull(o); - Assert.AreEqual(4, errors.Count); - Assert.AreEqual("Unexpected character encountered while parsing value: x. Path 'badarray[0]', line 1, position 15.", errors[0]); - Assert.AreEqual("Unexpected character encountered while parsing value: x. Path 'badarray[0]', line 1, position 15.", errors[1]); - Assert.AreEqual("Infinite loop detected from error handling. Path 'badarray[0]', line 1, position 15.", errors[2]); - Assert.AreEqual("Unexpected character encountered while parsing value: x. Path 'badarray[0]', line 1, position 15.", errors[3]); + Assert.AreEqual(2, errors.Count); + Assert.AreEqual("Unexpected character encountered while parsing value: x. Path 'badarray[0]', line 1, position 16.", errors[0]); + Assert.AreEqual("Unexpected character encountered while parsing value: ,. Path 'badarray[1]', line 1, position 17.", errors[1]); + + Assert.AreEqual(2, o.Count); + Assert.AreEqual(2, o["badarray"].Length); + Assert.AreEqual(0, o["badarray"][0]); + Assert.AreEqual(2, o["badarray"][1]); } [Test] diff --git a/Src/Newtonsoft.Json/JsonReader.cs b/Src/Newtonsoft.Json/JsonReader.cs index 5fe2c3bc..761ec563 100644 --- a/Src/Newtonsoft.Json/JsonReader.cs +++ b/Src/Newtonsoft.Json/JsonReader.cs @@ -510,8 +510,10 @@ namespace Newtonsoft.Json return null; } - if (IsWrappedInTypeObject()) + if (TokenType == JsonToken.StartObject) { + ReadIntoWrappedTypeObject(); + byte[] data = ReadAsBytes(); Read(); SetToken(JsonToken.Bytes, data, false); @@ -558,30 +560,36 @@ namespace Newtonsoft.Json return (byte[])Value; case JsonToken.StartArray: - List buffer = new List(); - - while (true) - { - switch (t = GetContentToken()) - { - case JsonToken.None: - throw JsonReaderException.Create(this, "Unexpected end when reading bytes."); - case JsonToken.Integer: - buffer.Add(Convert.ToByte(Value, CultureInfo.InvariantCulture)); - break; - case JsonToken.EndArray: - byte[] d = buffer.ToArray(); - SetToken(JsonToken.Bytes, d, false); - return d; - default: - throw JsonReaderException.Create(this, "Unexpected token when reading bytes: {0}.".FormatWith(CultureInfo.InvariantCulture, t)); - } - } + return ReadArrayIntoByteArray(); } throw JsonReaderException.Create(this, "Error reading bytes. Unexpected token: {0}.".FormatWith(CultureInfo.InvariantCulture, t)); } + internal byte[] ReadArrayIntoByteArray() + { + List buffer = new List(); + + while (true) + { + JsonToken t = GetContentToken(); + switch (t) + { + case JsonToken.None: + throw JsonReaderException.Create(this, "Unexpected end when reading bytes."); + case JsonToken.Integer: + buffer.Add(Convert.ToByte(Value, CultureInfo.InvariantCulture)); + break; + case JsonToken.EndArray: + byte[] d = buffer.ToArray(); + SetToken(JsonToken.Bytes, d, false); + return d; + default: + throw JsonReaderException.Create(this, "Unexpected token when reading bytes: {0}.".FormatWith(CultureInfo.InvariantCulture, t)); + } + } + } + /// /// Reads the next JSON token from the stream as a . /// @@ -736,32 +744,31 @@ namespace Newtonsoft.Json } #endif - private bool IsWrappedInTypeObject() + internal void ReaderReadAndAssert() { - if (TokenType == JsonToken.StartObject) + if (!Read()) { - if (!Read()) - { - throw JsonReaderException.Create(this, "Unexpected end when reading bytes."); - } + throw JsonReaderException.Create(this, "Unexpected end when reading JSON."); + } + } - if (Value.ToString() == JsonTypeReflector.TypePropertyName) + internal void ReadIntoWrappedTypeObject() + { + ReaderReadAndAssert(); + if (Value.ToString() == JsonTypeReflector.TypePropertyName) + { + ReaderReadAndAssert(); + if (Value != null && Value.ToString().StartsWith("System.Byte[]", StringComparison.Ordinal)) { - Read(); - if (Value != null && Value.ToString().StartsWith("System.Byte[]", StringComparison.Ordinal)) + ReaderReadAndAssert(); + if (Value.ToString() == JsonTypeReflector.ValuePropertyName) { - Read(); - if (Value.ToString() == JsonTypeReflector.ValuePropertyName) - { - return true; - } + return; } } - - throw JsonReaderException.Create(this, "Error reading bytes. Unexpected token: {0}.".FormatWith(CultureInfo.InvariantCulture, JsonToken.StartObject)); } - return false; + throw JsonReaderException.Create(this, "Error reading bytes. Unexpected token: {0}.".FormatWith(CultureInfo.InvariantCulture, JsonToken.StartObject)); } /// diff --git a/Src/Newtonsoft.Json/JsonTextReader.cs b/Src/Newtonsoft.Json/JsonTextReader.cs index ebe9f1e4..f64c8d85 100644 --- a/Src/Newtonsoft.Json/JsonTextReader.cs +++ b/Src/Newtonsoft.Json/JsonTextReader.cs @@ -437,6 +437,106 @@ namespace Newtonsoft.Json return (string)ReadStringValue(ReadType.ReadAsString); } + /// + /// Reads the next JSON token from the stream as a []. + /// + /// A [] or a null reference if the next JSON token is null. This method will return null at the end of an array. + public override byte[] ReadAsBytes() + { + EnsureBuffer(); + + switch (_currentState) + { + case State.Start: + case State.Property: + case State.Array: + case State.ArrayStart: + case State.Constructor: + case State.ConstructorStart: + case State.PostValue: + while (true) + { + char currentChar = _chars[_charPos]; + + switch (currentChar) + { + case '\0': + if (ReadNullChar()) + { + SetToken(JsonToken.None, null, false); + return null; + } + break; + case '"': + case '\'': + ParseString(currentChar, ReadType.ReadAsBytes); + return (byte[])Value; + case '{': + _charPos++; + SetToken(JsonToken.StartObject); + ReadIntoWrappedTypeObject(); + break; + case '[': + _charPos++; + SetToken(JsonToken.StartArray); + return ReadArrayIntoByteArray(); + case 'n': + HandleNull(); + return null; + case '/': + ParseComment(false); + break; + case ',': + _charPos++; + + if (_currentState != State.PostValue) + { + SetToken(JsonToken.Undefined); + throw JsonReaderException.Create(this, "Unexpected character encountered while parsing value: {0}.".FormatWith(CultureInfo.InvariantCulture, currentChar)); + } + + SetStateBasedOnCurrent(); + break; + case ']': + _charPos++; + if (_currentState == State.Array || _currentState == State.ArrayStart || _currentState == State.PostValue) + { + SetToken(JsonToken.EndArray); + return null; + } + throw JsonReaderException.Create(this, "Unexpected character encountered while parsing value: {0}.".FormatWith(CultureInfo.InvariantCulture, currentChar)); + case StringUtils.CarriageReturn: + ProcessCarriageReturn(false); + break; + case StringUtils.LineFeed: + ProcessLineFeed(); + break; + case ' ': + case StringUtils.Tab: + // eat + _charPos++; + break; + default: + _charPos++; + + if (!char.IsWhiteSpace(currentChar)) + { + throw JsonReaderException.Create(this, "Unexpected character encountered while parsing value: {0}.".FormatWith(CultureInfo.InvariantCulture, currentChar)); + } + + // eat + break; + } + } + case State.Finished: + ReadFinished(); + SetToken(JsonToken.None); + return null; + default: + throw JsonReaderException.Create(this, "Unexpected state: {0}.".FormatWith(CultureInfo.InvariantCulture, CurrentState)); + } + } + private object ReadStringValue(ReadType readType) { EnsureBuffer(); @@ -503,6 +603,7 @@ namespace Newtonsoft.Json case '9': if (readType != ReadType.ReadAsString) { + _charPos++; throw JsonReaderException.Create(this, "Unexpected character encountered while parsing value: {0}.".FormatWith(CultureInfo.InvariantCulture, currentChar)); } ParseNumber(ReadType.Read); @@ -546,14 +647,15 @@ namespace Newtonsoft.Json _charPos++; break; default: - if (char.IsWhiteSpace(currentChar)) + _charPos++; + + if (!char.IsWhiteSpace(currentChar)) { - // eat - _charPos++; - break; + throw JsonReaderException.Create(this, "Unexpected character encountered while parsing value: {0}.".FormatWith(CultureInfo.InvariantCulture, currentChar)); } - throw JsonReaderException.Create(this, "Unexpected character encountered while parsing value: {0}.".FormatWith(CultureInfo.InvariantCulture, currentChar)); + // eat + break; } } case State.Finished: @@ -654,16 +756,17 @@ namespace Newtonsoft.Json _charPos++; break; default: - if (char.IsWhiteSpace(currentChar)) + _charPos++; + + if (!char.IsWhiteSpace(currentChar)) { - // eat - _charPos++; - break; + throw JsonReaderException.Create(this, "Unexpected character encountered while parsing value: {0}.".FormatWith(CultureInfo.InvariantCulture, currentChar)); } - throw JsonReaderException.Create(this, "Unexpected character encountered while parsing value: {0}.".FormatWith(CultureInfo.InvariantCulture, currentChar)); - } + // eat + break; } + } case State.Finished: ReadFinished(); SetToken(JsonToken.None);