diff --git a/Src/Newtonsoft.Json.Tests/Bson/BsonReaderTests.cs b/Src/Newtonsoft.Json.Tests/Bson/BsonReaderTests.cs index ffb53252..ca55c877 100644 --- a/Src/Newtonsoft.Json.Tests/Bson/BsonReaderTests.cs +++ b/Src/Newtonsoft.Json.Tests/Bson/BsonReaderTests.cs @@ -25,6 +25,7 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Text; using NUnit.Framework; @@ -536,5 +537,40 @@ namespace Newtonsoft.Json.Tests.Bson BsonReader reader = new BsonReader(new MemoryStream()); Assert.IsFalse(reader.Read()); } + + [Test] + public void ReadLargeStrings() + { + string bson = + "4E-02-00-00-02-30-2D-31-2D-32-2D-33-2D-34-2D-35-2D-36-2D-37-2D-38-2D-39-2D-31-30-2D-31-31-2D-31-32-2D-31-33-2D-31-34-2D-31-35-2D-31-36-2D-31-37-2D-31-38-2D-31-39-2D-32-30-2D-32-31-2D-32-32-2D-32-33-2D-32-34-2D-32-35-2D-32-36-2D-32-37-2D-32-38-2D-32-39-2D-33-30-2D-33-31-2D-33-32-2D-33-33-2D-33-34-2D-33-35-2D-33-36-2D-33-37-2D-33-38-2D-33-39-2D-34-30-2D-34-31-2D-34-32-2D-34-33-2D-34-34-2D-34-35-2D-34-36-2D-34-37-2D-34-38-2D-34-39-2D-35-30-2D-35-31-2D-35-32-2D-35-33-2D-35-34-2D-35-35-2D-35-36-2D-35-37-2D-35-38-2D-35-39-2D-36-30-2D-36-31-2D-36-32-2D-36-33-2D-36-34-2D-36-35-2D-36-36-2D-36-37-2D-36-38-2D-36-39-2D-37-30-2D-37-31-2D-37-32-2D-37-33-2D-37-34-2D-37-35-2D-37-36-2D-37-37-2D-37-38-2D-37-39-2D-38-30-2D-38-31-2D-38-32-2D-38-33-2D-38-34-2D-38-35-2D-38-36-2D-38-37-2D-38-38-2D-38-39-2D-39-30-2D-39-31-2D-39-32-2D-39-33-2D-39-34-2D-39-35-2D-39-36-2D-39-37-2D-39-38-2D-39-39-00-22-01-00-00-30-2D-31-2D-32-2D-33-2D-34-2D-35-2D-36-2D-37-2D-38-2D-39-2D-31-30-2D-31-31-2D-31-32-2D-31-33-2D-31-34-2D-31-35-2D-31-36-2D-31-37-2D-31-38-2D-31-39-2D-32-30-2D-32-31-2D-32-32-2D-32-33-2D-32-34-2D-32-35-2D-32-36-2D-32-37-2D-32-38-2D-32-39-2D-33-30-2D-33-31-2D-33-32-2D-33-33-2D-33-34-2D-33-35-2D-33-36-2D-33-37-2D-33-38-2D-33-39-2D-34-30-2D-34-31-2D-34-32-2D-34-33-2D-34-34-2D-34-35-2D-34-36-2D-34-37-2D-34-38-2D-34-39-2D-35-30-2D-35-31-2D-35-32-2D-35-33-2D-35-34-2D-35-35-2D-35-36-2D-35-37-2D-35-38-2D-35-39-2D-36-30-2D-36-31-2D-36-32-2D-36-33-2D-36-34-2D-36-35-2D-36-36-2D-36-37-2D-36-38-2D-36-39-2D-37-30-2D-37-31-2D-37-32-2D-37-33-2D-37-34-2D-37-35-2D-37-36-2D-37-37-2D-37-38-2D-37-39-2D-38-30-2D-38-31-2D-38-32-2D-38-33-2D-38-34-2D-38-35-2D-38-36-2D-38-37-2D-38-38-2D-38-39-2D-39-30-2D-39-31-2D-39-32-2D-39-33-2D-39-34-2D-39-35-2D-39-36-2D-39-37-2D-39-38-2D-39-39-00-00"; + + BsonReader reader = new BsonReader(new MemoryStream(MiscellaneousUtils.HexToBytes(bson))); + + StringBuilder largeStringBuilder = new StringBuilder(); + for (int i = 0; i < 100; i++) + { + if (i > 0) + largeStringBuilder.Append("-"); + + largeStringBuilder.Append(i.ToString(CultureInfo.InvariantCulture)); + } + string largeString = largeStringBuilder.ToString(); + + Assert.IsTrue(reader.Read()); + Assert.AreEqual(JsonToken.StartObject, reader.TokenType); + + Assert.IsTrue(reader.Read()); + Assert.AreEqual(JsonToken.PropertyName, reader.TokenType); + Assert.AreEqual(largeString, reader.Value); + + Assert.IsTrue(reader.Read()); + Assert.AreEqual(JsonToken.String, reader.TokenType); + Assert.AreEqual(largeString, reader.Value); + + Assert.IsTrue(reader.Read()); + Assert.AreEqual(JsonToken.EndObject, reader.TokenType); + + Assert.IsFalse(reader.Read()); + } } } \ No newline at end of file diff --git a/Src/Newtonsoft.Json.Tests/Bson/BsonWriterTests.cs b/Src/Newtonsoft.Json.Tests/Bson/BsonWriterTests.cs index 9699a2e1..c9209566 100644 --- a/Src/Newtonsoft.Json.Tests/Bson/BsonWriterTests.cs +++ b/Src/Newtonsoft.Json.Tests/Bson/BsonWriterTests.cs @@ -32,6 +32,7 @@ using Newtonsoft.Json.Bson; using System.IO; using Newtonsoft.Json.Utilities; using Newtonsoft.Json.Tests.TestObjects; +using System.Globalization; namespace Newtonsoft.Json.Tests.Bson { @@ -190,5 +191,30 @@ namespace Newtonsoft.Json.Tests.Bson Console.WriteLine(l1); Console.WriteLine(l2); } + + [Test] + public void WriteLargeStrings() + { + MemoryStream ms = new MemoryStream(); + BsonWriter writer = new BsonWriter(ms); + + StringBuilder largeStringBuilder = new StringBuilder(); + for (int i = 0; i < 100; i++) + { + if (i > 0) + largeStringBuilder.Append("-"); + + largeStringBuilder.Append(i.ToString(CultureInfo.InvariantCulture)); + } + string largeString = largeStringBuilder.ToString(); + + writer.WriteStartObject(); + writer.WritePropertyName(largeString); + writer.WriteValue(largeString); + writer.WriteEndObject(); + + string bson = MiscellaneousUtils.BytesToHex(ms.ToArray()); + Assert.AreEqual("4E-02-00-00-02-30-2D-31-2D-32-2D-33-2D-34-2D-35-2D-36-2D-37-2D-38-2D-39-2D-31-30-2D-31-31-2D-31-32-2D-31-33-2D-31-34-2D-31-35-2D-31-36-2D-31-37-2D-31-38-2D-31-39-2D-32-30-2D-32-31-2D-32-32-2D-32-33-2D-32-34-2D-32-35-2D-32-36-2D-32-37-2D-32-38-2D-32-39-2D-33-30-2D-33-31-2D-33-32-2D-33-33-2D-33-34-2D-33-35-2D-33-36-2D-33-37-2D-33-38-2D-33-39-2D-34-30-2D-34-31-2D-34-32-2D-34-33-2D-34-34-2D-34-35-2D-34-36-2D-34-37-2D-34-38-2D-34-39-2D-35-30-2D-35-31-2D-35-32-2D-35-33-2D-35-34-2D-35-35-2D-35-36-2D-35-37-2D-35-38-2D-35-39-2D-36-30-2D-36-31-2D-36-32-2D-36-33-2D-36-34-2D-36-35-2D-36-36-2D-36-37-2D-36-38-2D-36-39-2D-37-30-2D-37-31-2D-37-32-2D-37-33-2D-37-34-2D-37-35-2D-37-36-2D-37-37-2D-37-38-2D-37-39-2D-38-30-2D-38-31-2D-38-32-2D-38-33-2D-38-34-2D-38-35-2D-38-36-2D-38-37-2D-38-38-2D-38-39-2D-39-30-2D-39-31-2D-39-32-2D-39-33-2D-39-34-2D-39-35-2D-39-36-2D-39-37-2D-39-38-2D-39-39-00-22-01-00-00-30-2D-31-2D-32-2D-33-2D-34-2D-35-2D-36-2D-37-2D-38-2D-39-2D-31-30-2D-31-31-2D-31-32-2D-31-33-2D-31-34-2D-31-35-2D-31-36-2D-31-37-2D-31-38-2D-31-39-2D-32-30-2D-32-31-2D-32-32-2D-32-33-2D-32-34-2D-32-35-2D-32-36-2D-32-37-2D-32-38-2D-32-39-2D-33-30-2D-33-31-2D-33-32-2D-33-33-2D-33-34-2D-33-35-2D-33-36-2D-33-37-2D-33-38-2D-33-39-2D-34-30-2D-34-31-2D-34-32-2D-34-33-2D-34-34-2D-34-35-2D-34-36-2D-34-37-2D-34-38-2D-34-39-2D-35-30-2D-35-31-2D-35-32-2D-35-33-2D-35-34-2D-35-35-2D-35-36-2D-35-37-2D-35-38-2D-35-39-2D-36-30-2D-36-31-2D-36-32-2D-36-33-2D-36-34-2D-36-35-2D-36-36-2D-36-37-2D-36-38-2D-36-39-2D-37-30-2D-37-31-2D-37-32-2D-37-33-2D-37-34-2D-37-35-2D-37-36-2D-37-37-2D-37-38-2D-37-39-2D-38-30-2D-38-31-2D-38-32-2D-38-33-2D-38-34-2D-38-35-2D-38-36-2D-38-37-2D-38-38-2D-38-39-2D-39-30-2D-39-31-2D-39-32-2D-39-33-2D-39-34-2D-39-35-2D-39-36-2D-39-37-2D-39-38-2D-39-39-00-00", bson); + } } } \ No newline at end of file diff --git a/Src/Newtonsoft.Json.Tests/PerformanceTests.cs b/Src/Newtonsoft.Json.Tests/PerformanceTests.cs index fc6f87b3..208efadd 100644 --- a/Src/Newtonsoft.Json.Tests/PerformanceTests.cs +++ b/Src/Newtonsoft.Json.Tests/PerformanceTests.cs @@ -8,10 +8,11 @@ using Newtonsoft.Json; using System.IO; using System.Web.Script.Serialization; - +using Newtonsoft.Json.Utilities; using NUnit.Framework; using System.Runtime.Serialization.Json; using System.Text; +using Newtonsoft.Json.Bson; namespace Newtonsoft.Json.Tests { @@ -23,9 +24,47 @@ namespace Newtonsoft.Json.Tests private int Iterations = 100; //private int Iterations = 5000; + private const string BsonHex = + @"B4-01-00-00-04-73-74-72-69-6E-67-73-00-44-00-00-00-02-30-00-05-00-00-00-52-69-63-6B-00-02-31-00-17-00-00-00-4D-61-72-6B-75-73-20-65-67-67-65-72-20-5D-5B-2C-20-28-32-6E-64-29-00-02-32-00-0E-00-00-00-4B-65-76-69-6E-20-4D-63-4E-65-69-73-68-00-00-03-64-69-63-74-69-6F-6E-61-72-79-00-27-00-00-00-10-56-61-6C-20-61-73-64-31-00-01-00-00-00-10-56-61-6C-32-00-03-00-00-00-10-56-61-6C-33-00-04-00-00-00-00-02-4E-61-6D-65-00-05-00-00-00-52-69-63-6B-00-09-4E-6F-77-00-A0-80-DB-70-25-01-00-00-01-42-69-67-4E-75-6D-62-65-72-00-E7-7B-CC-26-96-C7-1F-42-03-41-64-64-72-65-73-73-31-00-48-00-00-00-02-41-64-64-72-65-73-73-00-0B-00-00-00-66-66-66-20-53-74-72-65-65-74-00-02-50-68-6F-6E-65-00-0F-00-00-00-28-35-30-33-29-20-38-31-34-2D-36-33-33-35-00-09-45-6E-74-65-72-65-64-00-20-C2-A3-D7-25-01-00-00-00-04-41-64-64-72-65-73-73-65-73-00-A3-00-00-00-03-30-00-4B-00-00-00-02-41-64-64-72-65-73-73-00-0E-00-00-00-61-72-72-61-79-20-61-64-64-72-65-73-73-00-02-50-68-6F-6E-65-00-0F-00-00-00-28-35-30-33-29-20-38-31-34-2D-36-33-33-35-00-09-45-6E-74-65-72-65-64-00-20-36-7E-6B-25-01-00-00-00-03-31-00-4D-00-00-00-02-41-64-64-72-65-73-73-00-10-00-00-00-61-72-72-61-79-20-32-20-61-64-64-72-65-73-73-00-02-50-68-6F-6E-65-00-0F-00-00-00-28-35-30-33-29-20-38-31-34-2D-36-33-33-35-00-09-45-6E-74-65-72-65-64-00-20-DA-57-66-25-01-00-00-00-00-00"; + + private const string JsonText = + @"{ + ""strings"": [ + ""Rick"", + ""Markus egger ][, (2nd)"", + ""Kevin McNeish"" + ], + ""dictionary"": { + ""Val asd1"": 1, + ""Val2"": 3, + ""Val3"": 4 + }, + ""Name"": ""Rick"", + ""Now"": ""\/Date(1220867547892+1200)\/"", + ""BigNumber"": 34123123123.121, + ""Address1"": { + ""Address"": ""fff Street"", + ""Phone"": ""(503) 814-6335"", + ""Entered"": ""\/Date(1222588347892+1300)\/"" + }, + ""Addresses"": [ + { + ""Address"": ""array address"", + ""Phone"": ""(503) 814-6335"", + ""Entered"": ""\/Date(1220777547892+1200)\/"" + }, + { + ""Address"": ""array 2 address"", + ""Phone"": ""(503) 814-6335"", + ""Entered"": ""\/Date(1220691147893+1200)\/"" + } + ] +}"; + public enum SerializeMethod { JsonNet, + JsonNetBinary, JavaScriptSerializer, DataContractJsonSerializer } @@ -74,6 +113,21 @@ namespace Newtonsoft.Json.Tests return serializer.Deserialize(new StringReader(json), type); } + public object DeserializeJsonNetBinary(byte[] bson) + { + Type type = typeof(T); + + JsonSerializer serializer = new JsonSerializer(); + + serializer.NullValueHandling = NullValueHandling.Ignore; + + serializer.ObjectCreationHandling = Newtonsoft.Json.ObjectCreationHandling.Replace; + serializer.MissingMemberHandling = Newtonsoft.Json.MissingMemberHandling.Ignore; + serializer.ReferenceLoopHandling = ReferenceLoopHandling.Ignore; + + return serializer.Deserialize(new BsonReader(new MemoryStream(bson)), type); + } + public string SerializeWebExtensions(object value) { JavaScriptSerializer ser = new JavaScriptSerializer(); @@ -127,6 +181,13 @@ namespace Newtonsoft.Json.Tests case SerializeMethod.JsonNet: json = JsonConvert.SerializeObject(value); break; + case SerializeMethod.JsonNetBinary: + JsonSerializer serializer = new JsonSerializer(); + MemoryStream ms = new MemoryStream(); + BsonWriter writer = new BsonWriter(ms); + serializer.Serialize(writer, value); + json = "{Bytes " + ms.Length + "}"; + break; case SerializeMethod.JavaScriptSerializer: json = SerializeWebExtensions(value); break; @@ -144,7 +205,7 @@ namespace Newtonsoft.Json.Tests Console.WriteLine(); } - public void BenchmarkDeserializeMethod(SerializeMethod method, string json) + public void BenchmarkDeserializeMethod(SerializeMethod method, object json) { Stopwatch timed = new Stopwatch(); timed.Start(); @@ -155,13 +216,16 @@ namespace Newtonsoft.Json.Tests switch (method) { case SerializeMethod.JsonNet: - value = DeserializeJsonNet(json); + value = DeserializeJsonNet((string)json); + break; + case SerializeMethod.JsonNetBinary: + value = DeserializeJsonNetBinary((byte[])json); break; case SerializeMethod.JavaScriptSerializer: - value = DeserializeWebExtensions(json); + value = DeserializeWebExtensions((string)json); break; case SerializeMethod.DataContractJsonSerializer: - value = DeserializeDataContract(json); + value = DeserializeDataContract((string)json); break; } } @@ -198,6 +262,7 @@ namespace Newtonsoft.Json.Tests test.Addresses.Add(address); BenchmarkSerializeMethod(SerializeMethod.JsonNet, test); + BenchmarkSerializeMethod(SerializeMethod.JsonNetBinary, test); BenchmarkSerializeMethod(SerializeMethod.JavaScriptSerializer, test); BenchmarkSerializeMethod(SerializeMethod.DataContractJsonSerializer, test); } @@ -205,42 +270,10 @@ namespace Newtonsoft.Json.Tests [Test] public void Deserialize() { - string json = @"{ - ""strings"": [ - ""Rick"", - ""Markus egger ][, (2nd)"", - ""Kevin McNeish"" - ], - ""dictionary"": { - ""Val asd1"": 1, - ""Val2"": 3, - ""Val3"": 4 - }, - ""Name"": ""Rick"", - ""Now"": ""\/Date(1220867547892+1200)\/"", - ""BigNumber"": 34123123123.121, - ""Address1"": { - ""Address"": ""fff Street"", - ""Phone"": ""(503) 814-6335"", - ""Entered"": ""\/Date(1222588347892+1300)\/"" - }, - ""Addresses"": [ - { - ""Address"": ""array address"", - ""Phone"": ""(503) 814-6335"", - ""Entered"": ""\/Date(1220777547892+1200)\/"" - }, - { - ""Address"": ""array 2 address"", - ""Phone"": ""(503) 814-6335"", - ""Entered"": ""\/Date(1220691147893+1200)\/"" - } - ] -}"; - - BenchmarkDeserializeMethod(SerializeMethod.JsonNet, json); - BenchmarkDeserializeMethod(SerializeMethod.JavaScriptSerializer, json); - BenchmarkDeserializeMethod(SerializeMethod.DataContractJsonSerializer, json); + BenchmarkDeserializeMethod(SerializeMethod.JsonNet, JsonText); + BenchmarkDeserializeMethod(SerializeMethod.JsonNetBinary, MiscellaneousUtils.HexToBytes(BsonHex)); + BenchmarkDeserializeMethod(SerializeMethod.JavaScriptSerializer, JsonText); + BenchmarkDeserializeMethod(SerializeMethod.DataContractJsonSerializer, JsonText); } } diff --git a/Src/Newtonsoft.Json/Bson/BsonReader.cs b/Src/Newtonsoft.Json/Bson/BsonReader.cs index 225d0fa8..b0d6f7c0 100644 --- a/Src/Newtonsoft.Json/Bson/BsonReader.cs +++ b/Src/Newtonsoft.Json/Bson/BsonReader.cs @@ -43,8 +43,11 @@ namespace Newtonsoft.Json.Bson private readonly bool _rootTypeIsArray; private readonly List _stack; + private byte[] _byteBuffer; + private char[] _charBuffer; private BsonType _currentElementType; private BsonReaderState _bsonReaderState; + private const int MaxCharBytesSize = 128; private enum BsonReaderState { @@ -441,17 +444,41 @@ namespace Newtonsoft.Json.Bson private string ReadString() { - List buff = new List(); - byte b = _reader.ReadByte(); - while (b != 0) - { - buff.Add(b); - b = _reader.ReadByte(); - } + EnsureBuffers(); - MovePosition(buff.Count + 1); - string ret = Encoding.UTF8.GetString(buff.ToArray()); - return ret; + StringBuilder builder = null; + + int totalBytesRead = 0; + do + { + int byteCount = 0; + byte b; + while (byteCount < MaxCharBytesSize && (b = _reader.ReadByte()) > 0) + { + _byteBuffer[byteCount++] = b; + } + totalBytesRead += byteCount; + + int length = Encoding.UTF8.GetChars(_byteBuffer, 0, byteCount, _charBuffer, 0); + + if (byteCount < MaxCharBytesSize && builder == null) + { + MovePosition(totalBytesRead + 1); + return new string(_charBuffer, 0, length); + } + + if (builder == null) + builder = new StringBuilder(MaxCharBytesSize * 2); + + builder.Append(_charBuffer, 0, length); + + if (byteCount < MaxCharBytesSize) + { + MovePosition(totalBytesRead + 1); + return builder.ToString(); + } + } + while (true); } private string ReadLengthString() @@ -459,10 +486,56 @@ namespace Newtonsoft.Json.Bson int length = ReadInt32(); MovePosition(length); - byte[] buffer = _reader.ReadBytes(length - 1); + + string s = GetString(length - 1); _reader.ReadByte(); - return Encoding.UTF8.GetString(buffer); + return s; + } + + private string GetString(int length) + { + if (length == 0) + return string.Empty; + + EnsureBuffers(); + + StringBuilder builder = null; + + int totalBytesRead = 0; + do + { + int count = ((length - totalBytesRead) > MaxCharBytesSize) ? MaxCharBytesSize : (length - totalBytesRead); + int byteCount = _reader.BaseStream.Read(_byteBuffer, 0, count); + if (byteCount == 0) + throw new EndOfStreamException("Unable to read beyond the end of the stream."); + + int charCount = Encoding.UTF8.GetChars(_byteBuffer, 0, byteCount, _charBuffer, 0); + if (totalBytesRead == 0 && byteCount == length) + return new string(_charBuffer, 0, charCount); + + if (builder == null) + builder = new StringBuilder(length); + + builder.Append(_charBuffer, 0, charCount); + totalBytesRead += byteCount; + } + while (totalBytesRead < length); + + return builder.ToString(); + } + + private void EnsureBuffers() + { + if (_byteBuffer == null) + { + _byteBuffer = new byte[MaxCharBytesSize]; + } + if (_charBuffer == null) + { + int charBufferSize = Encoding.UTF8.GetMaxCharCount(MaxCharBytesSize); + _charBuffer = new char[charBufferSize]; + } } private double ReadDouble() @@ -510,4 +583,4 @@ namespace Newtonsoft.Json.Bson return _reader.ReadBytes(count); } } -} +} \ No newline at end of file diff --git a/Src/Newtonsoft.Json/Utilities/MiscellaneousUtils.cs b/Src/Newtonsoft.Json/Utilities/MiscellaneousUtils.cs index 394b69c5..d21b6f10 100644 --- a/Src/Newtonsoft.Json/Utilities/MiscellaneousUtils.cs +++ b/Src/Newtonsoft.Json/Utilities/MiscellaneousUtils.cs @@ -91,11 +91,12 @@ namespace Newtonsoft.Json.Utilities return false; for (int i = 0; i < a1.Length; i++) + { if (a1[i] != a2[i]) return false; + } return true; } - } } \ No newline at end of file