diff --git a/Src/Newtonsoft.Json.Tests/JsonTextReaderTest.cs b/Src/Newtonsoft.Json.Tests/JsonTextReaderTest.cs index bfb7967f..79a99806 100644 --- a/Src/Newtonsoft.Json.Tests/JsonTextReaderTest.cs +++ b/Src/Newtonsoft.Json.Tests/JsonTextReaderTest.cs @@ -58,11 +58,59 @@ namespace Newtonsoft.Json.Tests Assert.AreEqual(JsonToken.String, reader.TokenType); string s = reader.Value.ToString(); + Assert.AreEqual(2, s.Length); StringInfo stringInfo = new StringInfo(s); Assert.AreEqual(1, stringInfo.LengthInTextElements); } + [Test] + public void SurrogatePairReplacement() + { + // existing good surrogate pair + Assert.AreEqual("ABC \ud800\udc00 DEF", ReadString("ABC \\ud800\\udc00 DEF")); + + // invalid surrogates (two high back-to-back) + Assert.AreEqual("ABC \ufffd\ufffd DEF", ReadString("ABC \\ud800\\ud800 DEF")); + + // invalid surrogates (two high back-to-back) + Assert.AreEqual("ABC \ufffd\ufffd\u1234 DEF", ReadString("ABC \\ud800\\ud800\\u1234 DEF")); + + // invalid surrogates (three high back-to-back) + Assert.AreEqual("ABC \ufffd\ufffd\ufffd DEF", ReadString("ABC \\ud800\\ud800\\ud800 DEF")); + + // invalid surrogates (high followed by a good surrogate pair) + Assert.AreEqual("ABC \ufffd\ud800\udc00 DEF", ReadString("ABC \\ud800\\ud800\\udc00 DEF")); + + // invalid high surrogate at end of string + Assert.AreEqual("ABC \ufffd", ReadString("ABC \\ud800")); + + // high surrogate not followed by low surrogate + Assert.AreEqual("ABC \ufffd DEF", ReadString("ABC \\ud800 DEF")); + + // low surrogate not preceded by high surrogate + Assert.AreEqual("ABC \ufffd\ufffd DEF", ReadString("ABC \\udc00\\ud800 DEF")); + + // make sure unencoded invalid surrogate characters don't make it through + Assert.AreEqual("\ufffd\ufffd\ufffd", ReadString("\udc00\ud800\ud800")); + + Assert.AreEqual("ABC \ufffd\b", ReadString("ABC \\ud800\\b")); + Assert.AreEqual("ABC \ufffd ", ReadString("ABC \\ud800 ")); + Assert.AreEqual("ABC \b\ufffd", ReadString("ABC \\b\\ud800")); + } + + private string ReadString(string input) + { + MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(@"""" + input + @"""")); + + JsonTextReader reader = new JsonTextReader(new StreamReader(ms)); + reader.Read(); + + string s = (string)reader.Value; + + return s; + } + [Test] public void CloseInput() { @@ -2227,7 +2275,7 @@ bye", reader.Value); Assert.IsTrue(reader.Read()); ExceptionAssert.Throws( - "This reader's MaxDepth of 1 has been exceeded.", + "The reader's MaxDepth of 1 has been exceeded. Line 1, position 2.", () => { Assert.IsTrue(reader.Read()); diff --git a/Src/Newtonsoft.Json.Tests/Linq/JObjectTests.cs b/Src/Newtonsoft.Json.Tests/Linq/JObjectTests.cs index 4892fdf8..4e626d4a 100644 --- a/Src/Newtonsoft.Json.Tests/Linq/JObjectTests.cs +++ b/Src/Newtonsoft.Json.Tests/Linq/JObjectTests.cs @@ -1580,7 +1580,7 @@ Parameter name: arrayIndex" [Test] public void NumberTooBigForInt64() { - ExceptionAssert.Throws("JSON integer 307953220000517141511 is too large or small for an Int64.", + ExceptionAssert.Throws("JSON integer 307953220000517141511 is too large or small for an Int64. Line 1, position 30.", () => { string json = @"{""code"": 307953220000517141511}"; diff --git a/Src/Newtonsoft.Json.Tests/Serialization/JsonSerializerTest.cs b/Src/Newtonsoft.Json.Tests/Serialization/JsonSerializerTest.cs index d3ef3440..923bdfa7 100644 --- a/Src/Newtonsoft.Json.Tests/Serialization/JsonSerializerTest.cs +++ b/Src/Newtonsoft.Json.Tests/Serialization/JsonSerializerTest.cs @@ -2872,7 +2872,7 @@ To force JSON objects to deserialize add the JsonObjectAttribute to the type. Li { ExceptionAssert.Throws( @"Type 'Newtonsoft.Json.Tests.Serialization.JsonSerializerTest+ISerializableTestObject' implements ISerializable but cannot be deserialized using the ISerializable interface because the current application is not fully trusted and ISerializable can expose secure data. -To fix this error either change the environment to be fully trusted, change the application to not deserialize the type, add to JsonObjectAttribute to the type or change the JsonSerializer setting ContractResolver to use a new DefaultContractResolver with IgnoreSerializableInterface set to true.", +To fix this error either change the environment to be fully trusted, change the application to not deserialize the type, add to JsonObjectAttribute to the type or change the JsonSerializer setting ContractResolver to use a new DefaultContractResolver with IgnoreSerializableInterface set to true. Line 1, position 14.", () => { JsonTypeReflector.SetFullyTrusted(false); @@ -5679,7 +5679,7 @@ To fix this error either change the environment to be fully trusted, change the [Test] [ExpectedException(typeof (JsonSerializationException) #if !NETFX_CORE - , ExpectedMessage = "No JSON content found and type 'System.Double' is not nullable." + , ExpectedMessage = "No JSON content found and type 'System.Double' is not nullable. Line 0, position 0." #endif )] public void DeserializeDoubleFromEmptyString() @@ -5690,7 +5690,7 @@ To fix this error either change the environment to be fully trusted, change the [Test] [ExpectedException(typeof (JsonSerializationException) #if !NETFX_CORE - , ExpectedMessage = "No JSON content found and type 'System.StringComparison' is not nullable." + , ExpectedMessage = "No JSON content found and type 'System.StringComparison' is not nullable. Line 0, position 0." #endif )] public void DeserializeEnumFromEmptyString() @@ -5701,7 +5701,7 @@ To fix this error either change the environment to be fully trusted, change the [Test] [ExpectedException(typeof (JsonSerializationException) #if !NETFX_CORE - , ExpectedMessage = "No JSON content found and type 'System.Int32' is not nullable." + , ExpectedMessage = "No JSON content found and type 'System.Int32' is not nullable. Line 0, position 0." #endif )] public void DeserializeInt32FromEmptyString() diff --git a/Src/Newtonsoft.Json/JsonReader.cs b/Src/Newtonsoft.Json/JsonReader.cs index 24a36782..5254eb4a 100644 --- a/Src/Newtonsoft.Json/JsonReader.cs +++ b/Src/Newtonsoft.Json/JsonReader.cs @@ -256,7 +256,7 @@ namespace Newtonsoft.Json // this is a little hacky because Depth increases when first property/value is written but only testing here is faster/simpler if (_maxDepth != null && Depth + 1 > _maxDepth) - throw new JsonReaderException("This reader's MaxDepth of {0} has been exceeded.".FormatWith(CultureInfo.InvariantCulture, _maxDepth)); + throw CreateReaderException(this, "The reader's MaxDepth of {0} has been exceeded.".FormatWith(CultureInfo.InvariantCulture, _maxDepth)); } } @@ -750,7 +750,7 @@ namespace Newtonsoft.Json JsonContainerType currentObject = Pop(); if (GetTypeForCloseToken(endToken) != currentObject) - throw new JsonReaderException("JsonToken {0} is not valid for closing JsonType {1}.".FormatWith(CultureInfo.InvariantCulture, endToken, currentObject)); + throw CreateReaderException(this, "JsonToken {0} is not valid for closing JsonType {1}.".FormatWith(CultureInfo.InvariantCulture, endToken, currentObject)); _currentState = (Peek() != JsonContainerType.None) ? State.PostValue : State.Finished; } @@ -777,7 +777,7 @@ namespace Newtonsoft.Json _currentState = State.Finished; break; default: - throw new JsonReaderException("While setting the reader state back to current object an unexpected JsonType was encountered: {0}".FormatWith(CultureInfo.InvariantCulture, currentObject)); + throw CreateReaderException(this, "While setting the reader state back to current object an unexpected JsonType was encountered: {0}".FormatWith(CultureInfo.InvariantCulture, currentObject)); } } @@ -823,7 +823,7 @@ namespace Newtonsoft.Json case JsonToken.EndConstructor: return JsonContainerType.Constructor; default: - throw new JsonReaderException("Not a valid close JsonToken: {0}".FormatWith(CultureInfo.InvariantCulture, token)); + throw CreateReaderException(this, "Not a valid close JsonToken: {0}".FormatWith(CultureInfo.InvariantCulture, token)); } } diff --git a/Src/Newtonsoft.Json/JsonTextReader.cs b/Src/Newtonsoft.Json/JsonTextReader.cs index 00d93f92..001a0372 100644 --- a/Src/Newtonsoft.Json/JsonTextReader.cs +++ b/Src/Newtonsoft.Json/JsonTextReader.cs @@ -52,6 +52,8 @@ namespace Newtonsoft.Json /// public class JsonTextReader : JsonReader, IJsonLineInfo { + private const char UnicodeReplacementChar = '\uFFFD'; + private readonly TextReader _reader; private char[] _chars; @@ -583,19 +585,62 @@ namespace Newtonsoft.Json case 'u': charPos++; _charPos = charPos; - if (EnsureChars(4, true)) + writeChar = ParseUnicode(); + + if (StringUtils.IsLowSurrogate(writeChar)) { - string hexValues = new string(_chars, charPos, 4); - char hexChar = Convert.ToChar(int.Parse(hexValues, NumberStyles.HexNumber, NumberFormatInfo.InvariantInfo)); - writeChar = hexChar; + // low surrogate with no preceding high surrogate; this char is replaced + writeChar = UnicodeReplacementChar; + } + else if (StringUtils.IsHighSurrogate(writeChar)) + { + bool anotherHighSurrogate; - charPos += 4; - } - else - { - _charPos = charPos; - throw CreateReaderException(this, "Unexpected end while parsing unicode character."); + // loop for handling situations where there are multiple consecutive high surrogates + do + { + anotherHighSurrogate = false; + + // potential start of a surrogate pair + if (EnsureChars(2, true) && _chars[_charPos] == '\\' && _chars[_charPos + 1] == 'u') + { + char highSurrogate = writeChar; + + _charPos += 2; + writeChar = ParseUnicode(); + + if (StringUtils.IsLowSurrogate(writeChar)) + { + // a valid surrogate pair! + } + else if (StringUtils.IsHighSurrogate(writeChar)) + { + // another high surrogate; replace current and start check over + highSurrogate = UnicodeReplacementChar; + anotherHighSurrogate = true; + } + else + { + // high surrogate not followed by low surrogate; original char is replaced + highSurrogate = UnicodeReplacementChar; + } + + if (buffer == null) + buffer = GetBuffer(); + + WriteCharToBuffer(buffer, highSurrogate, lastWritePosition, escapeStartPos); + lastWritePosition = _charPos; + } + else + { + // there are not enough remaining chars for the low surrogate or is not follow by unicode sequence + // replace high surrogate and continue on as usual + writeChar = UnicodeReplacementChar; + } + } while (anotherHighSurrogate); } + + charPos = _charPos; break; default: charPos++; @@ -606,12 +651,7 @@ namespace Newtonsoft.Json if (buffer == null) buffer = GetBuffer(); - if (escapeStartPos > lastWritePosition) - { - buffer.Append(_chars, lastWritePosition, escapeStartPos - lastWritePosition); - } - - buffer.Append(writeChar); + WriteCharToBuffer(buffer, writeChar, lastWritePosition, escapeStartPos); lastWritePosition = charPos; break; @@ -655,6 +695,34 @@ namespace Newtonsoft.Json } } + private void WriteCharToBuffer(StringBuffer buffer, char writeChar, int lastWritePosition, int writeToPosition) + { + if (writeToPosition > lastWritePosition) + { + buffer.Append(_chars, lastWritePosition, writeToPosition - lastWritePosition); + } + + buffer.Append(writeChar); + } + + private char ParseUnicode() + { + char writeChar; + if (EnsureChars(4, true)) + { + string hexValues = new string(_chars, _charPos, 4); + char hexChar = Convert.ToChar(int.Parse(hexValues, NumberStyles.HexNumber, NumberFormatInfo.InvariantInfo)); + writeChar = hexChar; + + _charPos += 4; + } + else + { + throw CreateReaderException(this, "Unexpected end while parsing unicode character."); + } + return writeChar; + } + private void ReadNumberIntoBuffer() { int charPos = _charPos; @@ -1279,7 +1347,7 @@ namespace Newtonsoft.Json } catch (OverflowException ex) { - throw new JsonReaderException("JSON integer {0} is too large or small for an Int64.".FormatWith(CultureInfo.InvariantCulture, number), ex); + throw CreateReaderException((JsonReader)this, "JSON integer {0} is too large or small for an Int64.".FormatWith(CultureInfo.InvariantCulture, number), ex); } numberType = JsonToken.Integer; diff --git a/Src/Newtonsoft.Json/JsonWriter.cs b/Src/Newtonsoft.Json/JsonWriter.cs index 282ecef2..30e87f47 100644 --- a/Src/Newtonsoft.Json/JsonWriter.cs +++ b/Src/Newtonsoft.Json/JsonWriter.cs @@ -482,17 +482,17 @@ namespace Newtonsoft.Json private void WriteConstructorDate(JsonReader reader) { if (!reader.Read()) - throw new Exception("Unexpected end when reading date constructor."); + throw new JsonWriterException("Unexpected end when reading date constructor."); if (reader.TokenType != JsonToken.Integer) - throw new Exception("Unexpected token when reading date constructor. Expected Integer, got " + reader.TokenType); + throw new JsonWriterException("Unexpected token when reading date constructor. Expected Integer, got " + reader.TokenType); long ticks = (long)reader.Value; DateTime date = JsonConvert.ConvertJavaScriptTicksToDateTime(ticks); if (!reader.Read()) - throw new Exception("Unexpected end when reading date constructor."); + throw new JsonWriterException("Unexpected end when reading date constructor."); if (reader.TokenType != JsonToken.EndConstructor) - throw new Exception("Unexpected token when reading date constructor. Expected EndConstructor, got " + reader.TokenType); + throw new JsonWriterException("Unexpected token when reading date constructor. Expected EndConstructor, got " + reader.TokenType); WriteValue(date); } diff --git a/Src/Newtonsoft.Json/Serialization/JsonSerializerInternalReader.cs b/Src/Newtonsoft.Json/Serialization/JsonSerializerInternalReader.cs index 95af3992..aa6ec8a9 100644 --- a/Src/Newtonsoft.Json/Serialization/JsonSerializerInternalReader.cs +++ b/Src/Newtonsoft.Json/Serialization/JsonSerializerInternalReader.cs @@ -118,7 +118,7 @@ namespace Newtonsoft.Json.Serialization if (reader.TokenType == JsonToken.None && !ReadForType(reader, contract, converter != null, false)) { if (contract != null && !contract.IsNullable) - throw new JsonSerializationException("No JSON content found and type '{0}' is not nullable.".FormatWith(CultureInfo.InvariantCulture, contract.UnderlyingType)); + throw CreateSerializationException(reader, "No JSON content found and type '{0}' is not nullable.".FormatWith(CultureInfo.InvariantCulture, contract.UnderlyingType)); return null; } @@ -831,7 +831,7 @@ To force JSON arrays to deserialize add the JsonArrayAttribute to the type.".For if (!JsonTypeReflector.FullyTrusted) { - throw new JsonSerializationException(@"Type '{0}' implements ISerializable but cannot be deserialized using the ISerializable interface because the current application is not fully trusted and ISerializable can expose secure data. + throw CreateSerializationException(reader, @"Type '{0}' implements ISerializable but cannot be deserialized using the ISerializable interface because the current application is not fully trusted and ISerializable can expose secure data. To fix this error either change the environment to be fully trusted, change the application to not deserialize the type, add to JsonObjectAttribute to the type or change the JsonSerializer setting ContractResolver to use a new DefaultContractResolver with IgnoreSerializableInterface set to true.".FormatWith(CultureInfo.InvariantCulture, objectType)); } diff --git a/Src/Newtonsoft.Json/Utilities/StringUtils.cs b/Src/Newtonsoft.Json/Utilities/StringUtils.cs index abedcd46..e744b46c 100644 --- a/Src/Newtonsoft.Json/Utilities/StringUtils.cs +++ b/Src/Newtonsoft.Json/Utilities/StringUtils.cs @@ -152,5 +152,23 @@ namespace Newtonsoft.Json.Utilities return camelCase; } + + public static bool IsHighSurrogate(char c) + { +#if !SILVERLIGHT + return char.IsHighSurrogate(c); +#else + return (c >= 55296 && c <= 56319); +#endif + } + + public static bool IsLowSurrogate(char c) + { +#if !SILVERLIGHT + return char.IsLowSurrogate(c); +#else + return (c >= 56320 && c <= 57343); +#endif + } } } \ No newline at end of file