-Improve BsonReader string performance

This commit is contained in:
JamesNK
2009-12-09 05:24:06 +00:00
parent 2e54bf69ab
commit b6ff852467
5 changed files with 224 additions and 55 deletions
@@ -25,6 +25,7 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using NUnit.Framework;
@@ -536,5 +537,40 @@ namespace Newtonsoft.Json.Tests.Bson
BsonReader reader = new BsonReader(new MemoryStream());
Assert.IsFalse(reader.Read());
}
[Test]
public void ReadLargeStrings()
{
string bson =
"4E-02-00-00-02-30-2D-31-2D-32-2D-33-2D-34-2D-35-2D-36-2D-37-2D-38-2D-39-2D-31-30-2D-31-31-2D-31-32-2D-31-33-2D-31-34-2D-31-35-2D-31-36-2D-31-37-2D-31-38-2D-31-39-2D-32-30-2D-32-31-2D-32-32-2D-32-33-2D-32-34-2D-32-35-2D-32-36-2D-32-37-2D-32-38-2D-32-39-2D-33-30-2D-33-31-2D-33-32-2D-33-33-2D-33-34-2D-33-35-2D-33-36-2D-33-37-2D-33-38-2D-33-39-2D-34-30-2D-34-31-2D-34-32-2D-34-33-2D-34-34-2D-34-35-2D-34-36-2D-34-37-2D-34-38-2D-34-39-2D-35-30-2D-35-31-2D-35-32-2D-35-33-2D-35-34-2D-35-35-2D-35-36-2D-35-37-2D-35-38-2D-35-39-2D-36-30-2D-36-31-2D-36-32-2D-36-33-2D-36-34-2D-36-35-2D-36-36-2D-36-37-2D-36-38-2D-36-39-2D-37-30-2D-37-31-2D-37-32-2D-37-33-2D-37-34-2D-37-35-2D-37-36-2D-37-37-2D-37-38-2D-37-39-2D-38-30-2D-38-31-2D-38-32-2D-38-33-2D-38-34-2D-38-35-2D-38-36-2D-38-37-2D-38-38-2D-38-39-2D-39-30-2D-39-31-2D-39-32-2D-39-33-2D-39-34-2D-39-35-2D-39-36-2D-39-37-2D-39-38-2D-39-39-00-22-01-00-00-30-2D-31-2D-32-2D-33-2D-34-2D-35-2D-36-2D-37-2D-38-2D-39-2D-31-30-2D-31-31-2D-31-32-2D-31-33-2D-31-34-2D-31-35-2D-31-36-2D-31-37-2D-31-38-2D-31-39-2D-32-30-2D-32-31-2D-32-32-2D-32-33-2D-32-34-2D-32-35-2D-32-36-2D-32-37-2D-32-38-2D-32-39-2D-33-30-2D-33-31-2D-33-32-2D-33-33-2D-33-34-2D-33-35-2D-33-36-2D-33-37-2D-33-38-2D-33-39-2D-34-30-2D-34-31-2D-34-32-2D-34-33-2D-34-34-2D-34-35-2D-34-36-2D-34-37-2D-34-38-2D-34-39-2D-35-30-2D-35-31-2D-35-32-2D-35-33-2D-35-34-2D-35-35-2D-35-36-2D-35-37-2D-35-38-2D-35-39-2D-36-30-2D-36-31-2D-36-32-2D-36-33-2D-36-34-2D-36-35-2D-36-36-2D-36-37-2D-36-38-2D-36-39-2D-37-30-2D-37-31-2D-37-32-2D-37-33-2D-37-34-2D-37-35-2D-37-36-2D-37-37-2D-37-38-2D-37-39-2D-38-30-2D-38-31-2D-38-32-2D-38-33-2D-38-34-2D-38-35-2D-38-36-2D-38-37-2D-38-38-2D-38-39-2D-39-30-2D-39-31-2D-39-32-2D-39-33-2D-39-34-2D-39-35-2D-39-36-2D-39-37-2D-39-38-2D-39-39-00-00";
BsonReader reader = new BsonReader(new MemoryStream(MiscellaneousUtils.HexToBytes(bson)));
StringBuilder largeStringBuilder = new StringBuilder();
for (int i = 0; i < 100; i++)
{
if (i > 0)
largeStringBuilder.Append("-");
largeStringBuilder.Append(i.ToString(CultureInfo.InvariantCulture));
}
string largeString = largeStringBuilder.ToString();
Assert.IsTrue(reader.Read());
Assert.AreEqual(JsonToken.StartObject, reader.TokenType);
Assert.IsTrue(reader.Read());
Assert.AreEqual(JsonToken.PropertyName, reader.TokenType);
Assert.AreEqual(largeString, reader.Value);
Assert.IsTrue(reader.Read());
Assert.AreEqual(JsonToken.String, reader.TokenType);
Assert.AreEqual(largeString, reader.Value);
Assert.IsTrue(reader.Read());
Assert.AreEqual(JsonToken.EndObject, reader.TokenType);
Assert.IsFalse(reader.Read());
}
}
}
@@ -32,6 +32,7 @@ using Newtonsoft.Json.Bson;
using System.IO;
using Newtonsoft.Json.Utilities;
using Newtonsoft.Json.Tests.TestObjects;
using System.Globalization;
namespace Newtonsoft.Json.Tests.Bson
{
@@ -190,5 +191,30 @@ namespace Newtonsoft.Json.Tests.Bson
Console.WriteLine(l1);
Console.WriteLine(l2);
}
[Test]
public void WriteLargeStrings()
{
MemoryStream ms = new MemoryStream();
BsonWriter writer = new BsonWriter(ms);
StringBuilder largeStringBuilder = new StringBuilder();
for (int i = 0; i < 100; i++)
{
if (i > 0)
largeStringBuilder.Append("-");
largeStringBuilder.Append(i.ToString(CultureInfo.InvariantCulture));
}
string largeString = largeStringBuilder.ToString();
writer.WriteStartObject();
writer.WritePropertyName(largeString);
writer.WriteValue(largeString);
writer.WriteEndObject();
string bson = MiscellaneousUtils.BytesToHex(ms.ToArray());
Assert.AreEqual("4E-02-00-00-02-30-2D-31-2D-32-2D-33-2D-34-2D-35-2D-36-2D-37-2D-38-2D-39-2D-31-30-2D-31-31-2D-31-32-2D-31-33-2D-31-34-2D-31-35-2D-31-36-2D-31-37-2D-31-38-2D-31-39-2D-32-30-2D-32-31-2D-32-32-2D-32-33-2D-32-34-2D-32-35-2D-32-36-2D-32-37-2D-32-38-2D-32-39-2D-33-30-2D-33-31-2D-33-32-2D-33-33-2D-33-34-2D-33-35-2D-33-36-2D-33-37-2D-33-38-2D-33-39-2D-34-30-2D-34-31-2D-34-32-2D-34-33-2D-34-34-2D-34-35-2D-34-36-2D-34-37-2D-34-38-2D-34-39-2D-35-30-2D-35-31-2D-35-32-2D-35-33-2D-35-34-2D-35-35-2D-35-36-2D-35-37-2D-35-38-2D-35-39-2D-36-30-2D-36-31-2D-36-32-2D-36-33-2D-36-34-2D-36-35-2D-36-36-2D-36-37-2D-36-38-2D-36-39-2D-37-30-2D-37-31-2D-37-32-2D-37-33-2D-37-34-2D-37-35-2D-37-36-2D-37-37-2D-37-38-2D-37-39-2D-38-30-2D-38-31-2D-38-32-2D-38-33-2D-38-34-2D-38-35-2D-38-36-2D-38-37-2D-38-38-2D-38-39-2D-39-30-2D-39-31-2D-39-32-2D-39-33-2D-39-34-2D-39-35-2D-39-36-2D-39-37-2D-39-38-2D-39-39-00-22-01-00-00-30-2D-31-2D-32-2D-33-2D-34-2D-35-2D-36-2D-37-2D-38-2D-39-2D-31-30-2D-31-31-2D-31-32-2D-31-33-2D-31-34-2D-31-35-2D-31-36-2D-31-37-2D-31-38-2D-31-39-2D-32-30-2D-32-31-2D-32-32-2D-32-33-2D-32-34-2D-32-35-2D-32-36-2D-32-37-2D-32-38-2D-32-39-2D-33-30-2D-33-31-2D-33-32-2D-33-33-2D-33-34-2D-33-35-2D-33-36-2D-33-37-2D-33-38-2D-33-39-2D-34-30-2D-34-31-2D-34-32-2D-34-33-2D-34-34-2D-34-35-2D-34-36-2D-34-37-2D-34-38-2D-34-39-2D-35-30-2D-35-31-2D-35-32-2D-35-33-2D-35-34-2D-35-35-2D-35-36-2D-35-37-2D-35-38-2D-35-39-2D-36-30-2D-36-31-2D-36-32-2D-36-33-2D-36-34-2D-36-35-2D-36-36-2D-36-37-2D-36-38-2D-36-39-2D-37-30-2D-37-31-2D-37-32-2D-37-33-2D-37-34-2D-37-35-2D-37-36-2D-37-37-2D-37-38-2D-37-39-2D-38-30-2D-38-31-2D-38-32-2D-38-33-2D-38-34-2D-38-35-2D-38-36-2D-38-37-2D-38-38-2D-38-39-2D-39-30-2D-39-31-2D-39-32-2D-39-33-2D-39-34-2D-39-35-2D-39-36-2D-39-37-2D-39-38-2D-39-39-00-00", bson);
}
}
}
+74 -41
View File
@@ -8,10 +8,11 @@ using Newtonsoft.Json;
using System.IO;
using System.Web.Script.Serialization;
using Newtonsoft.Json.Utilities;
using NUnit.Framework;
using System.Runtime.Serialization.Json;
using System.Text;
using Newtonsoft.Json.Bson;
namespace Newtonsoft.Json.Tests
{
@@ -23,9 +24,47 @@ namespace Newtonsoft.Json.Tests
private int Iterations = 100;
//private int Iterations = 5000;
private const string BsonHex =
@"B4-01-00-00-04-73-74-72-69-6E-67-73-00-44-00-00-00-02-30-00-05-00-00-00-52-69-63-6B-00-02-31-00-17-00-00-00-4D-61-72-6B-75-73-20-65-67-67-65-72-20-5D-5B-2C-20-28-32-6E-64-29-00-02-32-00-0E-00-00-00-4B-65-76-69-6E-20-4D-63-4E-65-69-73-68-00-00-03-64-69-63-74-69-6F-6E-61-72-79-00-27-00-00-00-10-56-61-6C-20-61-73-64-31-00-01-00-00-00-10-56-61-6C-32-00-03-00-00-00-10-56-61-6C-33-00-04-00-00-00-00-02-4E-61-6D-65-00-05-00-00-00-52-69-63-6B-00-09-4E-6F-77-00-A0-80-DB-70-25-01-00-00-01-42-69-67-4E-75-6D-62-65-72-00-E7-7B-CC-26-96-C7-1F-42-03-41-64-64-72-65-73-73-31-00-48-00-00-00-02-41-64-64-72-65-73-73-00-0B-00-00-00-66-66-66-20-53-74-72-65-65-74-00-02-50-68-6F-6E-65-00-0F-00-00-00-28-35-30-33-29-20-38-31-34-2D-36-33-33-35-00-09-45-6E-74-65-72-65-64-00-20-C2-A3-D7-25-01-00-00-00-04-41-64-64-72-65-73-73-65-73-00-A3-00-00-00-03-30-00-4B-00-00-00-02-41-64-64-72-65-73-73-00-0E-00-00-00-61-72-72-61-79-20-61-64-64-72-65-73-73-00-02-50-68-6F-6E-65-00-0F-00-00-00-28-35-30-33-29-20-38-31-34-2D-36-33-33-35-00-09-45-6E-74-65-72-65-64-00-20-36-7E-6B-25-01-00-00-00-03-31-00-4D-00-00-00-02-41-64-64-72-65-73-73-00-10-00-00-00-61-72-72-61-79-20-32-20-61-64-64-72-65-73-73-00-02-50-68-6F-6E-65-00-0F-00-00-00-28-35-30-33-29-20-38-31-34-2D-36-33-33-35-00-09-45-6E-74-65-72-65-64-00-20-DA-57-66-25-01-00-00-00-00-00";
private const string JsonText =
@"{
""strings"": [
""Rick"",
""Markus egger ][, (2nd)"",
""Kevin McNeish""
],
""dictionary"": {
""Val asd1"": 1,
""Val2"": 3,
""Val3"": 4
},
""Name"": ""Rick"",
""Now"": ""\/Date(1220867547892+1200)\/"",
""BigNumber"": 34123123123.121,
""Address1"": {
""Address"": ""fff Street"",
""Phone"": ""(503) 814-6335"",
""Entered"": ""\/Date(1222588347892+1300)\/""
},
""Addresses"": [
{
""Address"": ""array address"",
""Phone"": ""(503) 814-6335"",
""Entered"": ""\/Date(1220777547892+1200)\/""
},
{
""Address"": ""array 2 address"",
""Phone"": ""(503) 814-6335"",
""Entered"": ""\/Date(1220691147893+1200)\/""
}
]
}";
public enum SerializeMethod
{
JsonNet,
JsonNetBinary,
JavaScriptSerializer,
DataContractJsonSerializer
}
@@ -74,6 +113,21 @@ namespace Newtonsoft.Json.Tests
return serializer.Deserialize(new StringReader(json), type);
}
public object DeserializeJsonNetBinary<T>(byte[] bson)
{
Type type = typeof(T);
JsonSerializer serializer = new JsonSerializer();
serializer.NullValueHandling = NullValueHandling.Ignore;
serializer.ObjectCreationHandling = Newtonsoft.Json.ObjectCreationHandling.Replace;
serializer.MissingMemberHandling = Newtonsoft.Json.MissingMemberHandling.Ignore;
serializer.ReferenceLoopHandling = ReferenceLoopHandling.Ignore;
return serializer.Deserialize(new BsonReader(new MemoryStream(bson)), type);
}
public string SerializeWebExtensions(object value)
{
JavaScriptSerializer ser = new JavaScriptSerializer();
@@ -127,6 +181,13 @@ namespace Newtonsoft.Json.Tests
case SerializeMethod.JsonNet:
json = JsonConvert.SerializeObject(value);
break;
case SerializeMethod.JsonNetBinary:
JsonSerializer serializer = new JsonSerializer();
MemoryStream ms = new MemoryStream();
BsonWriter writer = new BsonWriter(ms);
serializer.Serialize(writer, value);
json = "{Bytes " + ms.Length + "}";
break;
case SerializeMethod.JavaScriptSerializer:
json = SerializeWebExtensions(value);
break;
@@ -144,7 +205,7 @@ namespace Newtonsoft.Json.Tests
Console.WriteLine();
}
public void BenchmarkDeserializeMethod<T>(SerializeMethod method, string json)
public void BenchmarkDeserializeMethod<T>(SerializeMethod method, object json)
{
Stopwatch timed = new Stopwatch();
timed.Start();
@@ -155,13 +216,16 @@ namespace Newtonsoft.Json.Tests
switch (method)
{
case SerializeMethod.JsonNet:
value = DeserializeJsonNet<T>(json);
value = DeserializeJsonNet<T>((string)json);
break;
case SerializeMethod.JsonNetBinary:
value = DeserializeJsonNetBinary<T>((byte[])json);
break;
case SerializeMethod.JavaScriptSerializer:
value = DeserializeWebExtensions<T>(json);
value = DeserializeWebExtensions<T>((string)json);
break;
case SerializeMethod.DataContractJsonSerializer:
value = DeserializeDataContract<T>(json);
value = DeserializeDataContract<T>((string)json);
break;
}
}
@@ -198,6 +262,7 @@ namespace Newtonsoft.Json.Tests
test.Addresses.Add(address);
BenchmarkSerializeMethod(SerializeMethod.JsonNet, test);
BenchmarkSerializeMethod(SerializeMethod.JsonNetBinary, test);
BenchmarkSerializeMethod(SerializeMethod.JavaScriptSerializer, test);
BenchmarkSerializeMethod(SerializeMethod.DataContractJsonSerializer, test);
}
@@ -205,42 +270,10 @@ namespace Newtonsoft.Json.Tests
[Test]
public void Deserialize()
{
string json = @"{
""strings"": [
""Rick"",
""Markus egger ][, (2nd)"",
""Kevin McNeish""
],
""dictionary"": {
""Val asd1"": 1,
""Val2"": 3,
""Val3"": 4
},
""Name"": ""Rick"",
""Now"": ""\/Date(1220867547892+1200)\/"",
""BigNumber"": 34123123123.121,
""Address1"": {
""Address"": ""fff Street"",
""Phone"": ""(503) 814-6335"",
""Entered"": ""\/Date(1222588347892+1300)\/""
},
""Addresses"": [
{
""Address"": ""array address"",
""Phone"": ""(503) 814-6335"",
""Entered"": ""\/Date(1220777547892+1200)\/""
},
{
""Address"": ""array 2 address"",
""Phone"": ""(503) 814-6335"",
""Entered"": ""\/Date(1220691147893+1200)\/""
}
]
}";
BenchmarkDeserializeMethod<TestClass>(SerializeMethod.JsonNet, json);
BenchmarkDeserializeMethod<TestClass>(SerializeMethod.JavaScriptSerializer, json);
BenchmarkDeserializeMethod<TestClass>(SerializeMethod.DataContractJsonSerializer, json);
BenchmarkDeserializeMethod<TestClass>(SerializeMethod.JsonNet, JsonText);
BenchmarkDeserializeMethod<TestClass>(SerializeMethod.JsonNetBinary, MiscellaneousUtils.HexToBytes(BsonHex));
BenchmarkDeserializeMethod<TestClass>(SerializeMethod.JavaScriptSerializer, JsonText);
BenchmarkDeserializeMethod<TestClass>(SerializeMethod.DataContractJsonSerializer, JsonText);
}
}
+86 -13
View File
@@ -43,8 +43,11 @@ namespace Newtonsoft.Json.Bson
private readonly bool _rootTypeIsArray;
private readonly List<ContainerContext> _stack;
private byte[] _byteBuffer;
private char[] _charBuffer;
private BsonType _currentElementType;
private BsonReaderState _bsonReaderState;
private const int MaxCharBytesSize = 128;
private enum BsonReaderState
{
@@ -441,17 +444,41 @@ namespace Newtonsoft.Json.Bson
private string ReadString()
{
List<byte> buff = new List<byte>();
byte b = _reader.ReadByte();
while (b != 0)
{
buff.Add(b);
b = _reader.ReadByte();
}
EnsureBuffers();
MovePosition(buff.Count + 1);
string ret = Encoding.UTF8.GetString(buff.ToArray());
return ret;
StringBuilder builder = null;
int totalBytesRead = 0;
do
{
int byteCount = 0;
byte b;
while (byteCount < MaxCharBytesSize && (b = _reader.ReadByte()) > 0)
{
_byteBuffer[byteCount++] = b;
}
totalBytesRead += byteCount;
int length = Encoding.UTF8.GetChars(_byteBuffer, 0, byteCount, _charBuffer, 0);
if (byteCount < MaxCharBytesSize && builder == null)
{
MovePosition(totalBytesRead + 1);
return new string(_charBuffer, 0, length);
}
if (builder == null)
builder = new StringBuilder(MaxCharBytesSize * 2);
builder.Append(_charBuffer, 0, length);
if (byteCount < MaxCharBytesSize)
{
MovePosition(totalBytesRead + 1);
return builder.ToString();
}
}
while (true);
}
private string ReadLengthString()
@@ -459,10 +486,56 @@ namespace Newtonsoft.Json.Bson
int length = ReadInt32();
MovePosition(length);
byte[] buffer = _reader.ReadBytes(length - 1);
string s = GetString(length - 1);
_reader.ReadByte();
return Encoding.UTF8.GetString(buffer);
return s;
}
private string GetString(int length)
{
if (length == 0)
return string.Empty;
EnsureBuffers();
StringBuilder builder = null;
int totalBytesRead = 0;
do
{
int count = ((length - totalBytesRead) > MaxCharBytesSize) ? MaxCharBytesSize : (length - totalBytesRead);
int byteCount = _reader.BaseStream.Read(_byteBuffer, 0, count);
if (byteCount == 0)
throw new EndOfStreamException("Unable to read beyond the end of the stream.");
int charCount = Encoding.UTF8.GetChars(_byteBuffer, 0, byteCount, _charBuffer, 0);
if (totalBytesRead == 0 && byteCount == length)
return new string(_charBuffer, 0, charCount);
if (builder == null)
builder = new StringBuilder(length);
builder.Append(_charBuffer, 0, charCount);
totalBytesRead += byteCount;
}
while (totalBytesRead < length);
return builder.ToString();
}
private void EnsureBuffers()
{
if (_byteBuffer == null)
{
_byteBuffer = new byte[MaxCharBytesSize];
}
if (_charBuffer == null)
{
int charBufferSize = Encoding.UTF8.GetMaxCharCount(MaxCharBytesSize);
_charBuffer = new char[charBufferSize];
}
}
private double ReadDouble()
@@ -510,4 +583,4 @@ namespace Newtonsoft.Json.Bson
return _reader.ReadBytes(count);
}
}
}
}
@@ -91,11 +91,12 @@ namespace Newtonsoft.Json.Utilities
return false;
for (int i = 0; i < a1.Length; i++)
{
if (a1[i] != a2[i])
return false;
}
return true;
}
}
}