From 5dd9f0a90435c8362d9802a87d553fd380938623 Mon Sep 17 00:00:00 2001 From: James Newton-King Date: Sun, 11 Sep 2016 16:09:08 +1200 Subject: [PATCH] -Change double parsing validation to match JSON spec --- .../Utilities/ConvertUtilsTests.cs | 21 +++- Src/Newtonsoft.Json/Utilities/ConvertUtils.cs | 112 +++++++++++++----- 2 files changed, 100 insertions(+), 33 deletions(-) diff --git a/Src/Newtonsoft.Json.Tests/Utilities/ConvertUtilsTests.cs b/Src/Newtonsoft.Json.Tests/Utilities/ConvertUtilsTests.cs index b069ed0a..531960fb 100644 --- a/Src/Newtonsoft.Json.Tests/Utilities/ConvertUtilsTests.cs +++ b/Src/Newtonsoft.Json.Tests/Utilities/ConvertUtilsTests.cs @@ -50,7 +50,12 @@ namespace Newtonsoft.Json.Tests.Utilities ParseResult result = ConvertUtils.DoubleTryParse(c, 0, c.Length, out d); double d2; - bool result2 = double.TryParse(s, NumberStyles.Float, CultureInfo.InvariantCulture, out d2); + bool result2 = double.TryParse(s, NumberStyles.Float, CultureInfo.InvariantCulture, out d2) + && !s.StartsWith(".") + && !s.EndsWith(".") + && !(s.StartsWith("0") && s.Length > 1 && !s.StartsWith("0.")) + && !(s.StartsWith("-0") && s.Length > 2 && !s.StartsWith("-0.")) + && s.IndexOf(".e", StringComparison.OrdinalIgnoreCase) == -1; Assert.AreEqual(expectedResult, result); Assert.AreEqual(expectedResult == ParseResult.Success, result2); @@ -68,13 +73,15 @@ namespace Newtonsoft.Json.Tests.Utilities [Test] public void DoubleTryParse() { + AssertDoubleTryParse("0.1", ParseResult.Success, 0.1); + AssertDoubleTryParse("567.89", ParseResult.Success, 567.89); + AssertDoubleTryParse("0.25e+5", ParseResult.Success, 25000d); AssertDoubleTryParse("0.25e-5", ParseResult.Success, 0.0000025d); AssertDoubleTryParse("-123", ParseResult.Success, -123); AssertDoubleTryParse("0", ParseResult.Success, 0); AssertDoubleTryParse("123", ParseResult.Success, 123); - AssertDoubleTryParse("567.89", ParseResult.Success, 567.89); AssertDoubleTryParse("-567.89", ParseResult.Success, -567.89); AssertDoubleTryParse("1E23", ParseResult.Success, 1E23); AssertDoubleTryParse("1.1E23", ParseResult.Success, 1.1E23); @@ -98,13 +105,16 @@ namespace Newtonsoft.Json.Tests.Utilities AssertDoubleTryParse("1E-100", ParseResult.Success, 1E-100); AssertDoubleTryParse("1E-300", ParseResult.Success, 1E-300); - AssertDoubleTryParse(".1E23", ParseResult.Success, 1E22); - AssertDoubleTryParse("1.E23", ParseResult.Success, 1E23); - AssertDoubleTryParse("1E+309", ParseResult.Overflow, null); AssertDoubleTryParse("-1E+5000", ParseResult.Overflow, null); + AssertDoubleTryParse("01E308", ParseResult.Invalid, null); + AssertDoubleTryParse("-01E308", ParseResult.Invalid, null); + AssertDoubleTryParse("1.", ParseResult.Invalid, null); + AssertDoubleTryParse("0.", ParseResult.Invalid, null); + AssertDoubleTryParse(".1E23", ParseResult.Invalid, null); AssertDoubleTryParse("1..1E23", ParseResult.Invalid, null); + AssertDoubleTryParse("1.E23", ParseResult.Invalid, null); AssertDoubleTryParse("1E2.3", ParseResult.Invalid, null); AssertDoubleTryParse("1EE-10", ParseResult.Invalid, null); AssertDoubleTryParse("1E-1-0", ParseResult.Invalid, null); @@ -130,7 +140,6 @@ namespace Newtonsoft.Json.Tests.Utilities AssertDoubleTryParse("1E4294967297", ParseResult.Overflow, null); AssertDoubleTryParse("1E4294967297B", ParseResult.Invalid, null); AssertDoubleTryParse("1E-4294967297", ParseResult.Success, 0); - AssertDoubleTryParse("00000000000000000000000000000001E308", ParseResult.Success, 1E308); } [Test] diff --git a/Src/Newtonsoft.Json/Utilities/ConvertUtils.cs b/Src/Newtonsoft.Json/Utilities/ConvertUtils.cs index ec92f2fb..1e5f05f5 100644 --- a/Src/Newtonsoft.Json/Utilities/ConvertUtils.cs +++ b/Src/Newtonsoft.Json/Utilities/ConvertUtils.cs @@ -929,14 +929,13 @@ namespace Newtonsoft.Json.Utilities return ParseResult.Success; } - + private static class IEEE754 { - /// /// Exponents for both powers of 10 and 0.1 /// - private static readonly int[] MultExp64Power10 = new int[] + private static readonly int[] MultExp64Power10 = new int[] { 4, 7, 10, 14, 17, 20, 24, 27, 30, 34, 37, 40, 44, 47, 50 }; @@ -944,7 +943,7 @@ namespace Newtonsoft.Json.Utilities /// /// Normalized powers of 10 /// - private static readonly ulong[] MultVal64Power10 = new ulong[] + private static readonly ulong[] MultVal64Power10 = new ulong[] { 0xa000000000000000, 0xc800000000000000, 0xfa00000000000000, 0x9c40000000000000, 0xc350000000000000, 0xf424000000000000, @@ -956,7 +955,7 @@ namespace Newtonsoft.Json.Utilities /// /// Normalized powers of 0.1 /// - private static readonly ulong[] MultVal64Power10Inv = new ulong[] + private static readonly ulong[] MultVal64Power10Inv = new ulong[] { 0xcccccccccccccccd, 0xa3d70a3d70a3d70b, 0x83126e978d4fdf3c, 0xd1b71758e219652e, 0xa7c5ac471b478425, 0x8637bd05af6c69b7, @@ -968,7 +967,7 @@ namespace Newtonsoft.Json.Utilities /// /// Exponents for both powers of 10^16 and 0.1^16 /// - private static readonly int[] MultExp64Power10By16 = new int[] + private static readonly int[] MultExp64Power10By16 = new int[] { 54, 107, 160, 213, 266, 319, 373, 426, 479, 532, 585, 638, 691, 745, 798, 851, 904, 957, 1010, 1064, 1117, @@ -977,7 +976,7 @@ namespace Newtonsoft.Json.Utilities /// /// Normalized powers of 10^16 /// - private static readonly ulong[] MultVal64Power10By16 = new ulong[] + private static readonly ulong[] MultVal64Power10By16 = new ulong[] { 0x8e1bc9bf04000000, 0x9dc5ada82b70b59e, 0xaf298d050e4395d6, 0xc2781f49ffcfa6d4, 0xd7e77a8f87daf7fa, 0xefb3ab16c59b14a0, @@ -991,7 +990,7 @@ namespace Newtonsoft.Json.Utilities /// /// Normalized powers of 0.1^16 /// - private static readonly ulong[] MultVal64Power10By16Inv = new ulong[] + private static readonly ulong[] MultVal64Power10By16Inv = new ulong[] { 0xe69594bec44de160, 0xcfb11ead453994c3, 0xbb127c53b17ec165, 0xa87fea27a539e9b3, 0x97c560ba6b0919b5, 0x88b402f7fd7553ab, @@ -1002,7 +1001,6 @@ namespace Newtonsoft.Json.Utilities 0x8c71dcd9ba0b495c, 0xfd00b89747823938, 0xe3e27a444d8d991a, }; - /// /// Packs *10^ as 64-bit floating point value according to IEEE 754 standard /// @@ -1016,29 +1014,50 @@ namespace Newtonsoft.Json.Utilities public static double PackDouble(bool negative, ulong val, int scale) { // handle zero value - if (val == 0) { return negative ? -0.0 : 0.0; } // normalize the mantissa - int exp = 64; - if ((val & 0xFFFFFFFF00000000) == 0) { val <<= 32; exp -= 32; } - if ((val & 0xFFFF000000000000) == 0) { val <<= 16; exp -= 16; } - if ((val & 0xFF00000000000000) == 0) { val <<= 8; exp -= 8; } - if ((val & 0xF000000000000000) == 0) { val <<= 4; exp -= 4; } - if ((val & 0xC000000000000000) == 0) { val <<= 2; exp -= 2; } - if ((val & 0x8000000000000000) == 0) { val <<= 1; exp -= 1; } + if ((val & 0xFFFFFFFF00000000) == 0) + { + val <<= 32; + exp -= 32; + } + if ((val & 0xFFFF000000000000) == 0) + { + val <<= 16; + exp -= 16; + } + if ((val & 0xFF00000000000000) == 0) + { + val <<= 8; + exp -= 8; + } + if ((val & 0xF000000000000000) == 0) + { + val <<= 4; + exp -= 4; + } + if ((val & 0xC000000000000000) == 0) + { + val <<= 2; + exp -= 2; + } + if ((val & 0x8000000000000000) == 0) + { + val <<= 1; + exp -= 1; + } if (scale < 0) { scale = -scale; // check scale bounds - if (scale >= 22 * 16) { // underflow @@ -1046,7 +1065,6 @@ namespace Newtonsoft.Json.Utilities } // perform scaling - var index = scale & 15; if (index != 0) { @@ -1064,7 +1082,6 @@ namespace Newtonsoft.Json.Utilities else { // check scale bounds - if (scale >= 22 * 16) { // overflow @@ -1072,7 +1089,6 @@ namespace Newtonsoft.Json.Utilities } // perform scaling - var index = scale & 15; if (index != 0) { @@ -1150,8 +1166,10 @@ namespace Newtonsoft.Json.Utilities private static ulong Mul64Lossy(ulong a, ulong b, ref int exp) { - ulong a_hi = (a >> 32); uint a_lo = (uint)a; - ulong b_hi = (b >> 32); uint b_lo = (uint)b; + ulong a_hi = (a >> 32); + uint a_lo = (uint)a; + ulong b_hi = (b >> 32); + uint b_lo = (uint)b; var result = a_hi * b_hi; @@ -1159,17 +1177,24 @@ namespace Newtonsoft.Json.Utilities // (hi-parts will be always big enough, since a and b are normalized) if ((b_lo & 0xFFFF0000) != 0) + { result += (a_hi * b_lo) >> 32; + } if ((a_lo & 0xFFFF0000) != 0) + { result += (a_lo * b_hi) >> 32; + } // normalize - if ((result & 0x8000000000000000) == 0) { result <<= 1; exp--; } + if ((result & 0x8000000000000000) == 0) + { + result <<= 1; + exp--; + } return result; } - } public static ParseResult DoubleTryParse(char[] chars, int start, int length, out double value) @@ -1208,6 +1233,15 @@ namespace Newtonsoft.Json.Utilities switch (c) { case '.': + if (i == start) + { + return ParseResult.Invalid; + } + if (i + 1 == end) + { + return ParseResult.Invalid; + } + if (numDecimalStart != end) { // multiple decimal points @@ -1222,6 +1256,11 @@ namespace Newtonsoft.Json.Utilities { return ParseResult.Invalid; } + if (i == numDecimalStart) + { + // E follows decimal point + return ParseResult.Invalid; + } i++; if (i == end) { @@ -1254,7 +1293,7 @@ namespace Newtonsoft.Json.Utilities { return ParseResult.Invalid; } - + var newExponent = (10 * exponent) + (c - '0'); // stops updating exponent when overflowing if (exponent < newExponent) @@ -1274,6 +1313,25 @@ namespace Newtonsoft.Json.Utilities return ParseResult.Invalid; } + if (i == start && c == '0') + { + i++; + if (i != end) + { + if (chars[i] != '.') + { + return ParseResult.Invalid; + } + if (i + 1 == end) + { + return ParseResult.Invalid; + } + + numDecimalStart = i + 1; + continue; + } + } + if (mantissaDigits < 19) { mantissa = (10 * mantissa) + (ulong)(c - '0'); @@ -1294,7 +1352,7 @@ namespace Newtonsoft.Json.Utilities // correct the decimal point exponent -= (numDecimalEnd - numDecimalStart); - + value = IEEE754.PackDouble(isNegative, mantissa, exponent); return double.IsInfinity(value) ? ParseResult.Overflow : ParseResult.Success; }