-Fixed reading invalid unicode surrogate pairs

Src/Newtonsoft.Json.Tests/JsonTextReaderTest.cs

@@ -58,11 +58,59 @@ namespace Newtonsoft.Json.Tests
       Assert.AreEqual(JsonToken.String, reader.TokenType);
 
       string s = reader.Value.ToString();
+      Assert.AreEqual(2, s.Length);
 
       StringInfo stringInfo = new StringInfo(s);
       Assert.AreEqual(1, stringInfo.LengthInTextElements);
     }
 
+    [Test]
+    public void SurrogatePairReplacement()
+    {
+      // existing good surrogate pair
+      Assert.AreEqual("ABC \ud800\udc00 DEF", ReadString("ABC \\ud800\\udc00 DEF"));
+
+      // invalid surrogates (two high back-to-back)
+      Assert.AreEqual("ABC \ufffd\ufffd DEF", ReadString("ABC \\ud800\\ud800 DEF"));
+
+      // invalid surrogates (two high back-to-back)
+      Assert.AreEqual("ABC \ufffd\ufffd\u1234 DEF", ReadString("ABC \\ud800\\ud800\\u1234 DEF"));
+
+      // invalid surrogates (three high back-to-back)
+      Assert.AreEqual("ABC \ufffd\ufffd\ufffd DEF", ReadString("ABC \\ud800\\ud800\\ud800 DEF"));
+
+      // invalid surrogates (high followed by a good surrogate pair)
+      Assert.AreEqual("ABC \ufffd\ud800\udc00 DEF", ReadString("ABC \\ud800\\ud800\\udc00 DEF"));
+
+      // invalid high surrogate at end of string
+      Assert.AreEqual("ABC \ufffd", ReadString("ABC \\ud800"));
+
+      // high surrogate not followed by low surrogate
+      Assert.AreEqual("ABC \ufffd DEF", ReadString("ABC \\ud800 DEF"));
+
+      // low surrogate not preceded by high surrogate
+      Assert.AreEqual("ABC \ufffd\ufffd DEF", ReadString("ABC \\udc00\\ud800 DEF"));
+
+      // make sure unencoded invalid surrogate characters don't make it through
+      Assert.AreEqual("\ufffd\ufffd\ufffd", ReadString("\udc00\ud800\ud800"));
+
+      Assert.AreEqual("ABC \ufffd\b", ReadString("ABC \\ud800\\b"));
+      Assert.AreEqual("ABC \ufffd ", ReadString("ABC \\ud800 "));
+      Assert.AreEqual("ABC \b\ufffd", ReadString("ABC \\b\\ud800"));
+    }
+
+    private string ReadString(string input)
+    {
+      MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(@"""" + input + @""""));
+
+      JsonTextReader reader = new JsonTextReader(new StreamReader(ms));
+      reader.Read();
+
+      string s = (string)reader.Value;
+
+      return s;
+    }
+
     [Test]
     public void CloseInput()
     {
@@ -2227,7 +2275,7 @@ bye", reader.Value);
       Assert.IsTrue(reader.Read());
 
       ExceptionAssert.Throws<JsonReaderException>(
-        "This reader's MaxDepth of 1 has been exceeded.",
+        "The reader's MaxDepth of 1 has been exceeded. Line 1, position 2.",
         () =>
           {
             Assert.IsTrue(reader.Read());

Src/Newtonsoft.Json.Tests/Linq/JObjectTests.cs

@@ -1580,7 +1580,7 @@ Parameter name: arrayIndex"
     [Test]
     public void NumberTooBigForInt64()
     {
-      ExceptionAssert.Throws<JsonReaderException>("JSON integer 307953220000517141511 is too large or small for an Int64.",
+      ExceptionAssert.Throws<JsonReaderException>("JSON integer 307953220000517141511 is too large or small for an Int64. Line 1, position 30.",
         () =>
         {
           string json = @"{""code"": 307953220000517141511}";

Src/Newtonsoft.Json.Tests/Serialization/JsonSerializerTest.cs

@@ -2872,7 +2872,7 @@ To force JSON objects to deserialize add the JsonObjectAttribute to the type. Li
       {
         ExceptionAssert.Throws<JsonSerializationException>(
           @"Type 'Newtonsoft.Json.Tests.Serialization.JsonSerializerTest+ISerializableTestObject' implements ISerializable but cannot be deserialized using the ISerializable interface because the current application is not fully trusted and ISerializable can expose secure data.
-To fix this error either change the environment to be fully trusted, change the application to not deserialize the type, add to JsonObjectAttribute to the type or change the JsonSerializer setting ContractResolver to use a new DefaultContractResolver with IgnoreSerializableInterface set to true.",
+To fix this error either change the environment to be fully trusted, change the application to not deserialize the type, add to JsonObjectAttribute to the type or change the JsonSerializer setting ContractResolver to use a new DefaultContractResolver with IgnoreSerializableInterface set to true. Line 1, position 14.",
           () =>
             {
               JsonTypeReflector.SetFullyTrusted(false);
@@ -5679,7 +5679,7 @@ To fix this error either change the environment to be fully trusted, change the
     [Test]
     [ExpectedException(typeof (JsonSerializationException)
 #if !NETFX_CORE
-      , ExpectedMessage = "No JSON content found and type 'System.Double' is not nullable."
+      , ExpectedMessage = "No JSON content found and type 'System.Double' is not nullable. Line 0, position 0."
 #endif
       )]
     public void DeserializeDoubleFromEmptyString()
@@ -5690,7 +5690,7 @@ To fix this error either change the environment to be fully trusted, change the
     [Test]
     [ExpectedException(typeof (JsonSerializationException)
 #if !NETFX_CORE
-      , ExpectedMessage = "No JSON content found and type 'System.StringComparison' is not nullable."
+      , ExpectedMessage = "No JSON content found and type 'System.StringComparison' is not nullable. Line 0, position 0."
 #endif
       )]
     public void DeserializeEnumFromEmptyString()
@@ -5701,7 +5701,7 @@ To fix this error either change the environment to be fully trusted, change the
     [Test]
     [ExpectedException(typeof (JsonSerializationException)
 #if !NETFX_CORE
-      , ExpectedMessage = "No JSON content found and type 'System.Int32' is not nullable."
+      , ExpectedMessage = "No JSON content found and type 'System.Int32' is not nullable. Line 0, position 0."
 #endif
       )]
     public void DeserializeInt32FromEmptyString()

Src/Newtonsoft.Json/JsonReader.cs

@@ -256,7 +256,7 @@ namespace Newtonsoft.Json
 
         // this is a little hacky because Depth increases when first property/value is written but only testing here is faster/simpler
         if (_maxDepth != null && Depth + 1 > _maxDepth)
-          throw new JsonReaderException("This reader's MaxDepth of {0} has been exceeded.".FormatWith(CultureInfo.InvariantCulture, _maxDepth));
+          throw CreateReaderException(this, "The reader's MaxDepth of {0} has been exceeded.".FormatWith(CultureInfo.InvariantCulture, _maxDepth));
       }
     }
 
@@ -750,7 +750,7 @@ namespace Newtonsoft.Json
       JsonContainerType currentObject = Pop();
 
       if (GetTypeForCloseToken(endToken) != currentObject)
-        throw new JsonReaderException("JsonToken {0} is not valid for closing JsonType {1}.".FormatWith(CultureInfo.InvariantCulture, endToken, currentObject));
+        throw CreateReaderException(this, "JsonToken {0} is not valid for closing JsonType {1}.".FormatWith(CultureInfo.InvariantCulture, endToken, currentObject));
 
       _currentState = (Peek() != JsonContainerType.None) ? State.PostValue : State.Finished;
     }
@@ -777,7 +777,7 @@ namespace Newtonsoft.Json
           _currentState = State.Finished;
           break;
         default:
-          throw new JsonReaderException("While setting the reader state back to current object an unexpected JsonType was encountered: {0}".FormatWith(CultureInfo.InvariantCulture, currentObject));
+          throw CreateReaderException(this, "While setting the reader state back to current object an unexpected JsonType was encountered: {0}".FormatWith(CultureInfo.InvariantCulture, currentObject));
       }
     }
 
@@ -823,7 +823,7 @@ namespace Newtonsoft.Json
         case JsonToken.EndConstructor:
           return JsonContainerType.Constructor;
         default:
-          throw new JsonReaderException("Not a valid close JsonToken: {0}".FormatWith(CultureInfo.InvariantCulture, token));
+          throw CreateReaderException(this, "Not a valid close JsonToken: {0}".FormatWith(CultureInfo.InvariantCulture, token));
       }
     }
 

Src/Newtonsoft.Json/JsonTextReader.cs

@@ -52,6 +52,8 @@ namespace Newtonsoft.Json
   /// </summary>
   public class JsonTextReader : JsonReader, IJsonLineInfo
   {
+    private const char UnicodeReplacementChar = '\uFFFD';
+
     private readonly TextReader _reader;
 
     private char[] _chars;
@@ -583,19 +585,62 @@ namespace Newtonsoft.Json
               case 'u':
                 charPos++;
                 _charPos = charPos;
-                if (EnsureChars(4, true))
+                writeChar = ParseUnicode();
+                
+                if (StringUtils.IsLowSurrogate(writeChar))
                 {
-                  string hexValues = new string(_chars, charPos, 4);
-                  char hexChar = Convert.ToChar(int.Parse(hexValues, NumberStyles.HexNumber, NumberFormatInfo.InvariantInfo));
-                  writeChar = hexChar;
+                  // low surrogate with no preceding high surrogate; this char is replaced
+                  writeChar = UnicodeReplacementChar;
+                }
+                else if (StringUtils.IsHighSurrogate(writeChar))
+                {
+                  bool anotherHighSurrogate;
 
-                  charPos += 4;
-                }
-                else
-                {
-                  _charPos = charPos;
-                  throw CreateReaderException(this, "Unexpected end while parsing unicode character.");
+                  // loop for handling situations where there are multiple consecutive high surrogates
+                  do
+                  {
+                    anotherHighSurrogate = false;
+
+                    // potential start of a surrogate pair
+                    if (EnsureChars(2, true) && _chars[_charPos] == '\\' && _chars[_charPos + 1] == 'u')
+                    {
+                      char highSurrogate = writeChar;
+
+                      _charPos += 2;
+                      writeChar = ParseUnicode();
+
+                      if (StringUtils.IsLowSurrogate(writeChar))
+                      {
+                        // a valid surrogate pair!
+                      }
+                      else if (StringUtils.IsHighSurrogate(writeChar))
+                      {
+                        // another high surrogate; replace current and start check over
+                        highSurrogate = UnicodeReplacementChar;
+                        anotherHighSurrogate = true;
+                      }
+                      else
+                      {
+                        // high surrogate not followed by low surrogate; original char is replaced
+                        highSurrogate = UnicodeReplacementChar;
+                      }
+
+                      if (buffer == null)
+                        buffer = GetBuffer();
+
+                      WriteCharToBuffer(buffer, highSurrogate, lastWritePosition, escapeStartPos);
+                      lastWritePosition = _charPos;
+                    }
+                    else
+                    {
+                      // there are not enough remaining chars for the low surrogate or is not follow by unicode sequence
+                      // replace high surrogate and continue on as usual
+                      writeChar = UnicodeReplacementChar;
+                    }
+                  } while (anotherHighSurrogate);
                 }
+
+                charPos = _charPos;
                 break;
               default:
                 charPos++;
@@ -606,12 +651,7 @@ namespace Newtonsoft.Json
             if (buffer == null)
               buffer = GetBuffer();
 
-            if (escapeStartPos > lastWritePosition)
-            {
-              buffer.Append(_chars, lastWritePosition, escapeStartPos - lastWritePosition);
-            }
-
-            buffer.Append(writeChar);
+            WriteCharToBuffer(buffer, writeChar, lastWritePosition, escapeStartPos);
 
             lastWritePosition = charPos;
             break;
@@ -655,6 +695,34 @@ namespace Newtonsoft.Json
       }
     }
 
+    private void WriteCharToBuffer(StringBuffer buffer, char writeChar, int lastWritePosition, int writeToPosition)
+    {
+      if (writeToPosition > lastWritePosition)
+      {
+        buffer.Append(_chars, lastWritePosition, writeToPosition - lastWritePosition);
+      }
+
+      buffer.Append(writeChar);
+    }
+
+    private char ParseUnicode()
+    {
+      char writeChar;
+      if (EnsureChars(4, true))
+      {
+        string hexValues = new string(_chars, _charPos, 4);
+        char hexChar = Convert.ToChar(int.Parse(hexValues, NumberStyles.HexNumber, NumberFormatInfo.InvariantInfo));
+        writeChar = hexChar;
+
+        _charPos += 4;
+      }
+      else
+      {
+        throw CreateReaderException(this, "Unexpected end while parsing unicode character.");
+      }
+      return writeChar;
+    }
+
     private void ReadNumberIntoBuffer()
     {
       int charPos = _charPos;
@@ -1279,7 +1347,7 @@ namespace Newtonsoft.Json
             }
             catch (OverflowException ex)
             {
-              throw new JsonReaderException("JSON integer {0} is too large or small for an Int64.".FormatWith(CultureInfo.InvariantCulture, number), ex);
+              throw CreateReaderException((JsonReader)this, "JSON integer {0} is too large or small for an Int64.".FormatWith(CultureInfo.InvariantCulture, number), ex);
             }
 
             numberType = JsonToken.Integer;

Src/Newtonsoft.Json/JsonWriter.cs

@@ -482,17 +482,17 @@ namespace Newtonsoft.Json
     private void WriteConstructorDate(JsonReader reader)
     {
       if (!reader.Read())
-        throw new Exception("Unexpected end when reading date constructor.");
+        throw new JsonWriterException("Unexpected end when reading date constructor.");
       if (reader.TokenType != JsonToken.Integer)
-        throw new Exception("Unexpected token when reading date constructor. Expected Integer, got " + reader.TokenType);
+        throw new JsonWriterException("Unexpected token when reading date constructor. Expected Integer, got " + reader.TokenType);
 
       long ticks = (long)reader.Value;
       DateTime date = JsonConvert.ConvertJavaScriptTicksToDateTime(ticks);
 
       if (!reader.Read())
-        throw new Exception("Unexpected end when reading date constructor.");
+        throw new JsonWriterException("Unexpected end when reading date constructor.");
       if (reader.TokenType != JsonToken.EndConstructor)
-        throw new Exception("Unexpected token when reading date constructor. Expected EndConstructor, got " + reader.TokenType);
+        throw new JsonWriterException("Unexpected token when reading date constructor. Expected EndConstructor, got " + reader.TokenType);
 
       WriteValue(date);
     }

Src/Newtonsoft.Json/Serialization/JsonSerializerInternalReader.cs

@@ -118,7 +118,7 @@ namespace Newtonsoft.Json.Serialization
       if (reader.TokenType == JsonToken.None && !ReadForType(reader, contract, converter != null, false))
       {
         if (contract != null && !contract.IsNullable)
-          throw new JsonSerializationException("No JSON content found and type '{0}' is not nullable.".FormatWith(CultureInfo.InvariantCulture, contract.UnderlyingType));
+          throw CreateSerializationException(reader, "No JSON content found and type '{0}' is not nullable.".FormatWith(CultureInfo.InvariantCulture, contract.UnderlyingType));
 
         return null;
       }
@@ -831,7 +831,7 @@ To force JSON arrays to deserialize add the JsonArrayAttribute to the type.".For
 
       if (!JsonTypeReflector.FullyTrusted)
       {
-        throw new JsonSerializationException(@"Type '{0}' implements ISerializable but cannot be deserialized using the ISerializable interface because the current application is not fully trusted and ISerializable can expose secure data.
+        throw CreateSerializationException(reader, @"Type '{0}' implements ISerializable but cannot be deserialized using the ISerializable interface because the current application is not fully trusted and ISerializable can expose secure data.
 To fix this error either change the environment to be fully trusted, change the application to not deserialize the type, add to JsonObjectAttribute to the type or change the JsonSerializer setting ContractResolver to use a new DefaultContractResolver with IgnoreSerializableInterface set to true.".FormatWith(CultureInfo.InvariantCulture, objectType));
       }
 

Src/Newtonsoft.Json/Utilities/StringUtils.cs

@@ -152,5 +152,23 @@ namespace Newtonsoft.Json.Utilities
 
       return camelCase;
     }
+
+    public static bool IsHighSurrogate(char c)
+    {
+#if !SILVERLIGHT
+      return char.IsHighSurrogate(c);
+#else
+      return (c >= 55296 && c <= 56319);
+#endif
+    }
+
+    public static bool IsLowSurrogate(char c)
+    {
+#if !SILVERLIGHT
+      return char.IsLowSurrogate(c);
+#else
+      return (c >= 56320 && c <= 57343);
+#endif
+    }
   }
 }