-Fixed JsonWriter.ReadAsXXX methods not erroring on missing commas

Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ExceptionHandlingAsyncTests.cs

@@ -50,6 +50,106 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 #endif
     public class ExceptionHandlingAsyncTests : TestFixtureBase
     {
+        [Test]
+        public async Task ReadAsBytes_MissingCommaAsync()
+        {
+            byte[] data = Encoding.UTF8.GetBytes("Hello world");
+
+            string json = @"['" + Convert.ToBase64String(data) + "' '" + Convert.ToBase64String(data) + @"']";
+            JsonTextReader reader = new JsonTextReader(new StringReader(json));
+
+            Assert.IsTrue(await reader.ReadAsync());
+            CollectionAssert.AreEquivalent(data, await reader.ReadAsBytesAsync());
+
+            await ExceptionAssert.ThrowsAsync<JsonReaderException>(
+                () => reader.ReadAsBytesAsync(),
+                "After parsing a value an unexpected character was encountered: '. Path '[0]', line 1, position 20.");
+        }
+
+        [Test]
+        public async Task ReadAsInt32_MissingCommaAsync()
+        {
+            string json = "[0 1 2]";
+            JsonTextReader reader = new JsonTextReader(new StringReader(json));
+
+            Assert.IsTrue(await reader.ReadAsync());
+            Assert.AreEqual(0, (int)await reader.ReadAsInt32Async());
+
+            await ExceptionAssert.ThrowsAsync<JsonReaderException>(
+                () => reader.ReadAsInt32Async(),
+                "After parsing a value an unexpected character was encountered: 1. Path '[0]', line 1, position 3.");
+        }
+
+        [Test]
+        public async Task ReadAsBoolean_MissingCommaAsync()
+        {
+            string json = "[true false true]";
+            JsonTextReader reader = new JsonTextReader(new StringReader(json));
+
+            Assert.IsTrue(await reader.ReadAsync());
+            Assert.AreEqual(true, (bool)await reader.ReadAsBooleanAsync());
+
+            await ExceptionAssert.ThrowsAsync<JsonReaderException>(
+                () => reader.ReadAsBooleanAsync(),
+                "After parsing a value an unexpected character was encountered: f. Path '[0]', line 1, position 6.");
+        }
+
+        [Test]
+        public async Task ReadAsDateTime_MissingCommaAsync()
+        {
+            string json = "['2017-02-04T00:00:00Z' '2018-02-04T00:00:00Z' '2019-02-04T00:00:00Z']";
+            JsonTextReader reader = new JsonTextReader(new StringReader(json));
+
+            Assert.IsTrue(await reader.ReadAsync());
+            Assert.AreEqual(new DateTime(2017, 2, 4, 0, 0, 0, DateTimeKind.Utc), (DateTime)await reader.ReadAsDateTimeAsync());
+
+            await ExceptionAssert.ThrowsAsync<JsonReaderException>(
+                () => reader.ReadAsDateTimeAsync(),
+                "After parsing a value an unexpected character was encountered: '. Path '[0]', line 1, position 24.");
+        }
+
+        [Test]
+        public async Task ReadAsDateTimeOffset_MissingCommaAsync()
+        {
+            string json = "['2017-02-04T00:00:00Z' '2018-02-04T00:00:00Z' '2019-02-04T00:00:00Z']";
+            JsonTextReader reader = new JsonTextReader(new StringReader(json));
+
+            Assert.IsTrue(await reader.ReadAsync());
+            Assert.AreEqual(new DateTimeOffset(2017, 2, 4, 0, 0, 0, TimeSpan.Zero), (DateTimeOffset)await reader.ReadAsDateTimeOffsetAsync());
+
+            await ExceptionAssert.ThrowsAsync<JsonReaderException>(
+                () => reader.ReadAsDateTimeOffsetAsync(),
+                "After parsing a value an unexpected character was encountered: '. Path '[0]', line 1, position 24.");
+        }
+
+        [Test]
+        public async Task ReadAsString_MissingCommaAsync()
+        {
+            string json = "['2017-02-04T00:00:00Z' '2018-02-04T00:00:00Z' '2019-02-04T00:00:00Z']";
+            JsonTextReader reader = new JsonTextReader(new StringReader(json));
+
+            Assert.IsTrue(await reader.ReadAsync());
+            Assert.AreEqual("2017-02-04T00:00:00Z", await reader.ReadAsStringAsync());
+
+            await ExceptionAssert.ThrowsAsync<JsonReaderException>(
+                () => reader.ReadAsStringAsync(),
+                "After parsing a value an unexpected character was encountered: '. Path '[0]', line 1, position 24.");
+        }
+
+        [Test]
+        public async Task Read_MissingCommaAsync()
+        {
+            string json = "[0 1 2]";
+            JsonTextReader reader = new JsonTextReader(new StringReader(json));
+
+            Assert.IsTrue(await reader.ReadAsync());
+            Assert.IsTrue(await reader.ReadAsync());
+
+            await ExceptionAssert.ThrowsAsync<JsonReaderException>(
+                () => reader.ReadAsync(),
+                "After parsing a value an unexpected character was encountered: 1. Path '[0]', line 1, position 3.");
+        }
+
         [Test]
         public async Task UnexpectedEndAfterReadingNAsync()
         {

Src/Newtonsoft.Json.Tests/JsonTextReaderTests/ExceptionHandlingTests.cs

@@ -61,6 +61,106 @@ namespace Newtonsoft.Json.Tests.JsonTextReaderTests
 #endif
     public class ExceptionHandlingTests : TestFixtureBase
     {
+        [Test]
+        public void ReadAsBytes_MissingComma()
+        {
+            byte[] data = Encoding.UTF8.GetBytes("Hello world");
+
+            string json = @"['" + Convert.ToBase64String(data) + "' '" + Convert.ToBase64String(data) + @"']";
+            JsonTextReader reader = new JsonTextReader(new StringReader(json));
+
+            Assert.IsTrue(reader.Read());
+            CollectionAssert.AreEquivalent(data, reader.ReadAsBytes());
+
+            ExceptionAssert.Throws<JsonReaderException>(
+                () => reader.ReadAsBytes(),
+                "After parsing a value an unexpected character was encountered: '. Path '[0]', line 1, position 20.");
+        }
+
+        [Test]
+        public void ReadAsInt32_MissingComma()
+        {
+            string json = "[0 1 2]";
+            JsonTextReader reader = new JsonTextReader(new StringReader(json));
+
+            Assert.IsTrue(reader.Read());
+            Assert.AreEqual(0, (int)reader.ReadAsInt32());
+
+            ExceptionAssert.Throws<JsonReaderException>(
+                () => reader.ReadAsInt32(),
+                "After parsing a value an unexpected character was encountered: 1. Path '[0]', line 1, position 3.");
+        }
+
+        [Test]
+        public void ReadAsBoolean_MissingComma()
+        {
+            string json = "[true false true]";
+            JsonTextReader reader = new JsonTextReader(new StringReader(json));
+
+            Assert.IsTrue(reader.Read());
+            Assert.AreEqual(true, (bool)reader.ReadAsBoolean());
+
+            ExceptionAssert.Throws<JsonReaderException>(
+                () => reader.ReadAsBoolean(),
+                "After parsing a value an unexpected character was encountered: f. Path '[0]', line 1, position 6.");
+        }
+
+        [Test]
+        public void ReadAsDateTime_MissingComma()
+        {
+            string json = "['2017-02-04T00:00:00Z' '2018-02-04T00:00:00Z' '2019-02-04T00:00:00Z']";
+            JsonTextReader reader = new JsonTextReader(new StringReader(json));
+
+            Assert.IsTrue(reader.Read());
+            Assert.AreEqual(new DateTime(2017, 2, 4, 0, 0, 0, DateTimeKind.Utc), (DateTime)reader.ReadAsDateTime());
+
+            ExceptionAssert.Throws<JsonReaderException>(
+                () => reader.ReadAsDateTime(),
+                "After parsing a value an unexpected character was encountered: '. Path '[0]', line 1, position 24.");
+        }
+
+        [Test]
+        public void ReadAsDateTimeOffset_MissingComma()
+        {
+            string json = "['2017-02-04T00:00:00Z' '2018-02-04T00:00:00Z' '2019-02-04T00:00:00Z']";
+            JsonTextReader reader = new JsonTextReader(new StringReader(json));
+
+            Assert.IsTrue(reader.Read());
+            Assert.AreEqual(new DateTimeOffset(2017, 2, 4, 0, 0, 0, TimeSpan.Zero), (DateTimeOffset)reader.ReadAsDateTimeOffset());
+
+            ExceptionAssert.Throws<JsonReaderException>(
+                () => reader.ReadAsDateTimeOffset(),
+                "After parsing a value an unexpected character was encountered: '. Path '[0]', line 1, position 24.");
+        }
+
+        [Test]
+        public void ReadAsString_MissingComma()
+        {
+            string json = "['2017-02-04T00:00:00Z' '2018-02-04T00:00:00Z' '2019-02-04T00:00:00Z']";
+            JsonTextReader reader = new JsonTextReader(new StringReader(json));
+
+            Assert.IsTrue(reader.Read());
+            Assert.AreEqual("2017-02-04T00:00:00Z", reader.ReadAsString());
+
+            ExceptionAssert.Throws<JsonReaderException>(
+                () => reader.ReadAsString(),
+                "After parsing a value an unexpected character was encountered: '. Path '[0]', line 1, position 24.");
+        }
+
+        [Test]
+        public void Read_MissingComma()
+        {
+            string json = "[0 1 2]";
+            JsonTextReader reader = new JsonTextReader(new StringReader(json));
+
+            Assert.IsTrue(reader.Read());
+            Assert.IsTrue(reader.Read());
+
+            ExceptionAssert.Throws<JsonReaderException>(
+                () => reader.Read(),
+                "After parsing a value an unexpected character was encountered: 1. Path '[0]', line 1, position 3.");
+        }
+
         [Test]
         public void UnexpectedEndAfterReadingN()
         {

Src/Newtonsoft.Json/JsonTextReader.Async.cs

@@ -61,30 +61,56 @@ namespace Newtonsoft.Json
         internal Task<bool> DoReadAsync(CancellationToken cancellationToken)
         {
             EnsureBuffer();
-            switch (_currentState)
+
+            while (true)
             {
-                case State.Start:
+                switch (_currentState)
-                case State.Property:
+                {
-                case State.Array:
+                    case State.Start:
-                case State.ArrayStart:
+                    case State.Property:
-                case State.Constructor:
+                    case State.Array:
-                case State.ConstructorStart:
+                    case State.ArrayStart:
-                    return ParseValueAsync(cancellationToken);
+                    case State.Constructor:
-                case State.Object:
+                    case State.ConstructorStart:
-                case State.ObjectStart:
+                        return ParseValueAsync(cancellationToken);
-                    return ParseObjectAsync(cancellationToken);
+                    case State.Object:
-                case State.PostValue:
+                    case State.ObjectStart:
-                    return LoopReadAsync(cancellationToken);
+                        return ParseObjectAsync(cancellationToken);
-                case State.Finished:
+                    case State.PostValue:
-                    return ReadFromFinishedAsync(cancellationToken);
+                        Task<bool> task = ParsePostValueAsync(false, cancellationToken);
-                default:
+                        if (task.Status == TaskStatus.RanToCompletion)
-                    throw JsonReaderException.Create(this, "Unexpected state: {0}.".FormatWith(CultureInfo.InvariantCulture, CurrentState));
+                        {
+                            if (task.Result)
+                            {
+                                return AsyncUtils.True;
+                            }
+                        }
+                        else
+                        {
+                            return DoReadAsync(task, cancellationToken);
+                        }
+                        break;
+                    case State.Finished:
+                        return ReadFromFinishedAsync(cancellationToken);
+                    default:
+                        throw JsonReaderException.Create(this, "Unexpected state: {0}.".FormatWith(CultureInfo.InvariantCulture, CurrentState));
+                }
             }
         }
 
-        private async Task<bool> LoopReadAsync(CancellationToken cancellationToken)
+        private async Task<bool> DoReadAsync(Task<bool> task, CancellationToken cancellationToken)
         {
-            while (_currentState == State.PostValue)
+            bool result = await task.ConfigureAwait(false);
+            if (result)
+            {
+                return true;
+            }
+            return await DoReadAsync(cancellationToken).ConfigureAwait(false);
+        }
+
+        private async Task<bool> ParsePostValueAsync(bool ignoreComments, CancellationToken cancellationToken)
+        {
+            while (true)
             {
                 char currentChar = _chars[_charPos];
 
@@ -96,6 +122,7 @@ namespace Newtonsoft.Json
                             if (await ReadDataAsync(false, cancellationToken).ConfigureAwait(false) == 0)
                             {
                                 _currentState = State.Finished;
+                                return false;
                             }
                         }
                         else
@@ -117,14 +144,18 @@ namespace Newtonsoft.Json
                         SetToken(JsonToken.EndConstructor);
                         return true;
                     case '/':
-                        await ParseCommentAsync(true, cancellationToken).ConfigureAwait(false);
+                        await ParseCommentAsync(!ignoreComments, cancellationToken).ConfigureAwait(false);
-                        return true;
+                        if (!ignoreComments)
+                        {
+                            return true;
+                        }
+                        break;
                     case ',':
                         _charPos++;
 
                         // finished parsing
                         SetStateBasedOnCurrent();
-                        break;
+                        return false;
                     case ' ':
                     case StringUtils.Tab:
 
@@ -151,8 +182,6 @@ namespace Newtonsoft.Json
                         break;
                 }
             }
-
-            return await DoReadAsync(cancellationToken).ConfigureAwait(false);
         }
 
         private async Task<bool> ReadFromFinishedAsync(CancellationToken cancellationToken)
@@ -1101,13 +1130,18 @@ namespace Newtonsoft.Json
 
             switch (_currentState)
             {
+                case State.PostValue:
+                    if (await ParsePostValueAsync(true, cancellationToken))
+                    {
+                        return null;
+                    }
+                    goto case State.Start;
                 case State.Start:
                 case State.Property:
                 case State.Array:
                 case State.ArrayStart:
                 case State.Constructor:
                 case State.ConstructorStart:
-                case State.PostValue:
                     while (true)
                     {
                         char currentChar = _chars[_charPos];
@@ -1231,13 +1265,18 @@ namespace Newtonsoft.Json
 
             switch (_currentState)
             {
+                case State.PostValue:
+                    if (await ParsePostValueAsync(true, cancellationToken))
+                    {
+                        return null;
+                    }
+                    goto case State.Start;
                 case State.Start:
                 case State.Property:
                 case State.Array:
                 case State.ArrayStart:
                 case State.Constructor:
                 case State.ConstructorStart:
-                case State.PostValue:
                     while (true)
                     {
                         char currentChar = _chars[_charPos];
@@ -1352,13 +1391,18 @@ namespace Newtonsoft.Json
 
             switch (_currentState)
             {
+                case State.PostValue:
+                    if (await ParsePostValueAsync(true, cancellationToken))
+                    {
+                        return null;
+                    }
+                    goto case State.Start;
                 case State.Start:
                 case State.Property:
                 case State.Array:
                 case State.ArrayStart:
                 case State.Constructor:
                 case State.ConstructorStart:
-                case State.PostValue:
                     while (true)
                     {
                         char currentChar = _chars[_charPos];
@@ -1483,13 +1527,18 @@ namespace Newtonsoft.Json
 
             switch (_currentState)
             {
+                case State.PostValue:
+                    if (await ParsePostValueAsync(true, cancellationToken))
+                    {
+                        return null;
+                    }
+                    goto case State.Start;
                 case State.Start:
                 case State.Property:
                 case State.Array:
                 case State.ArrayStart:
                 case State.Constructor:
                 case State.ConstructorStart:
-                case State.PostValue:
                     while (true)
                     {
                         char currentChar = _chars[_charPos];

Src/Newtonsoft.Json/JsonTextReader.cs

@@ -406,7 +406,7 @@ namespace Newtonsoft.Json
                     case State.PostValue:
                         // returns true if it hits
                         // end of object or array
-                        if (ParsePostValue())
+                        if (ParsePostValue(false))
                         {
                             return true;
                         }
@@ -474,13 +474,18 @@ namespace Newtonsoft.Json
 
             switch (_currentState)
             {
+                case State.PostValue:
+                    if (ParsePostValue(true))
+                    {
+                        return null;
+                    }
+                    goto case State.Start;
                 case State.Start:
                 case State.Property:
                 case State.Array:
                 case State.ArrayStart:
                 case State.Constructor:
                 case State.ConstructorStart:
-                case State.PostValue:
                     while (true)
                     {
                         char currentChar = _chars[_charPos];
@@ -572,13 +577,18 @@ namespace Newtonsoft.Json
 
             switch (_currentState)
             {
+                case State.PostValue:
+                    if (ParsePostValue(true))
+                    {
+                        return null;
+                    }
+                    goto case State.Start;
                 case State.Start:
                 case State.Property:
                 case State.Array:
                 case State.ArrayStart:
                 case State.Constructor:
                 case State.ConstructorStart:
-                case State.PostValue:
                     while (true)
                     {
                         char currentChar = _chars[_charPos];
@@ -733,13 +743,18 @@ namespace Newtonsoft.Json
 
             switch (_currentState)
             {
+                case State.PostValue:
+                    if (ParsePostValue(true))
+                    {
+                        return null;
+                    }
+                    goto case State.Start;
                 case State.Start:
                 case State.Property:
                 case State.Array:
                 case State.ArrayStart:
                 case State.Constructor:
                 case State.ConstructorStart:
-                case State.PostValue:
                     while (true)
                     {
                         char currentChar = _chars[_charPos];
@@ -849,7 +864,11 @@ namespace Newtonsoft.Json
             if (_currentState != State.PostValue)
             {
                 SetToken(JsonToken.Undefined);
-                throw CreateUnexpectedCharacterException(',');
+                JsonReaderException ex = CreateUnexpectedCharacterException(',');
+                // so the comma will be parsed again
+                _charPos--;
+
+                throw ex;
             }
 
             SetStateBasedOnCurrent();
@@ -861,13 +880,18 @@ namespace Newtonsoft.Json
 
             switch (_currentState)
             {
+                case State.PostValue:
+                    if (ParsePostValue(true))
+                    {
+                        return null;
+                    }
+                    goto case State.Start;
                 case State.Start:
                 case State.Property:
                 case State.Array:
                 case State.ArrayStart:
                 case State.Constructor:
                 case State.ConstructorStart:
-                case State.PostValue:
                     while (true)
                     {
                         char currentChar = _chars[_charPos];
@@ -1368,7 +1392,7 @@ namespace Newtonsoft.Json
             _stringReference = new StringReference();
         }
 
-        private bool ParsePostValue()
+        private bool ParsePostValue(bool ignoreComments)
         {
             while (true)
             {
@@ -1403,8 +1427,12 @@ namespace Newtonsoft.Json
                         SetToken(JsonToken.EndConstructor);
                         return true;
                     case '/':
-                        ParseComment(true);
+                        ParseComment(!ignoreComments);
-                        return true;
+                        if (!ignoreComments)
+                        {
+                            return true;
+                        }
+                        break;
                     case ',':
                         _charPos++;