LiRuochen_WorkFlow/Newtonsoft.Json
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

-Fix reading BSON multi byte utf8 characters

Browse Source
This commit is contained in:
JamesNK
2010-03-20 01:29:02 +00:00
parent 0b42b38f34
commit d393430f30
2 changed files with 322 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Src/Newtonsoft.Json.Tests/Bson/BsonReaderTests.cs
+201
View File
@@ -38,6 +38,9 @@ namespace Newtonsoft.Json.Tests.Bson
{
public class BsonReaderTests : TestFixtureBase
{
private char pound = '\u00a3';
private char euro = '\u20ac';
[Test]
public void ReadSingleObject()
{
@@ -689,5 +692,203 @@ namespace Newtonsoft.Json.Tests.Bson
o = (JObject)JToken.ReadFrom(reader);
Assert.AreEqual(DateTime.SpecifyKind(value.ToLocalTime(), DateTimeKind.Unspecified), (DateTime)o["DateTime"]);
}
private string WriteAndReadStringValue(string val)
{
MemoryStream ms = new MemoryStream();
BsonWriter bs = new BsonWriter(ms);
bs.WriteStartObject();
bs.WritePropertyName("StringValue");
bs.WriteValue(val);
bs.WriteEnd();
ms.Seek(0, SeekOrigin.Begin);
BsonReader reader = new BsonReader(ms);
// object
reader.Read();
// property name
reader.Read();
// string
reader.Read();
return (string)reader.Value;
}
private string WriteAndReadStringPropertyName(string val)
{
MemoryStream ms = new MemoryStream();
BsonWriter bs = new BsonWriter(ms);
bs.WriteStartObject();
bs.WritePropertyName(val);
bs.WriteValue("Dummy");
bs.WriteEnd();
ms.Seek(0, SeekOrigin.Begin);
BsonReader reader = new BsonReader(ms);
// object
reader.Read();
// property name
reader.Read();
return (string)reader.Value;
}
[Test]
public void TestReadLenStringValueShortTripleByte()
{
StringBuilder sb = new StringBuilder();
//sb.Append('1',127); //first char of euro at the end of the boundry.
//sb.Append(euro, 5);
//sb.Append('1',128);
sb.Append(euro);
string expected = sb.ToString();
Assert.AreEqual(expected, WriteAndReadStringValue(expected));
}
[Test]
public void TestReadLenStringValueTripleByteCharBufferBoundry0()
{
StringBuilder sb = new StringBuilder();
sb.Append('1', 127); //first char of euro at the end of the boundry.
sb.Append(euro, 5);
sb.Append('1', 128);
sb.Append(euro);
string expected = sb.ToString();
Assert.AreEqual(expected, WriteAndReadStringValue(expected));
}
[Test]
public void TestReadLenStringValueTripleByteCharBufferBoundry1()
{
StringBuilder sb = new StringBuilder();
sb.Append('1', 126);
sb.Append(euro, 5); //middle char of euro at the end of the boundry.
sb.Append('1', 128);
sb.Append(euro);
string expected = sb.ToString();
string result = WriteAndReadStringValue(expected);
Assert.AreEqual(expected, result);
}
[Test]
public void TestReadLenStringValueTripleByteCharOne()
{
StringBuilder sb = new StringBuilder();
sb.Append(euro, 1); //Just one triple byte char in the string.
string expected = sb.ToString();
Assert.AreEqual(expected, WriteAndReadStringValue(expected));
}
[Test]
public void TestReadLenStringValueTripleByteCharBufferBoundry2()
{
StringBuilder sb = new StringBuilder();
sb.Append('1', 125);
sb.Append(euro, 5); //last char of the eruo at the end of the boundry.
sb.Append('1', 128);
sb.Append(euro);
string expected = sb.ToString();
Assert.AreEqual(expected, WriteAndReadStringValue(expected));
}
[Test]
public void TestReadStringValue()
{
string expected = "test";
Assert.AreEqual(expected, WriteAndReadStringValue(expected));
}
[Test]
public void TestReadStringValueLong()
{
StringBuilder sb = new StringBuilder();
sb.Append('t', 150);
string expected = sb.ToString();
Assert.AreEqual(expected, WriteAndReadStringValue(expected));
}
[Test]
public void TestReadStringPropertyNameShortTripleByte()
{
StringBuilder sb = new StringBuilder();
//sb.Append('1',127); //first char of euro at the end of the boundry.
//sb.Append(euro, 5);
//sb.Append('1',128);
sb.Append(euro);
string expected = sb.ToString();
Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected));
}
[Test]
public void TestReadStringPropertyNameTripleByteCharBufferBoundry0()
{
StringBuilder sb = new StringBuilder();
sb.Append('1', 127); //first char of euro at the end of the boundry.
sb.Append(euro, 5);
sb.Append('1', 128);
sb.Append(euro);
string expected = sb.ToString();
string result = WriteAndReadStringPropertyName(expected);
Assert.AreEqual(expected, result);
}
[Test]
public void TestReadStringPropertyNameTripleByteCharBufferBoundry1()
{
StringBuilder sb = new StringBuilder();
sb.Append('1', 126);
sb.Append(euro, 5); //middle char of euro at the end of the boundry.
sb.Append('1', 128);
sb.Append(euro);
string expected = sb.ToString();
Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected));
}
[Test]
public void TestReadStringPropertyNameTripleByteCharOne()
{
StringBuilder sb = new StringBuilder();
sb.Append(euro, 1); //Just one triple byte char in the string.
string expected = sb.ToString();
Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected));
}
[Test]
public void TestReadStringPropertyNameTripleByteCharBufferBoundry2()
{
StringBuilder sb = new StringBuilder();
sb.Append('1', 125);
sb.Append(euro, 5); //last char of the eruo at the end of the boundry.
sb.Append('1', 128);
sb.Append(euro);
string expected = sb.ToString();
Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected));
}
[Test]
public void TestReadStringPropertyName()
{
string expected = "test";
Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected));
}
[Test]
public void TestReadStringPropertyNameLong()
{
StringBuilder sb = new StringBuilder();
sb.Append('t', 150);
string expected = sb.ToString();
Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected));
}
}
}
Src/Newtonsoft.Json/Bson/BsonReader.cs
+121 -22
View File
@@ -39,12 +39,17 @@ namespace Newtonsoft.Json.Bson
public class BsonReader : JsonReader
{
private const int MaxCharBytesSize = 128;
private static readonly byte[] _seqRange1 = new byte[] { 0, 127 }; // range of 1-byte sequence
private static readonly byte[] _seqRange2 = new byte[] { 194, 223 }; // range of 2-byte sequence
private static readonly byte[] _seqRange3 = new byte[] { 224, 239 }; // range of 3-byte sequence
private static readonly byte[] _seqRange4 = new byte[] { 240, 244 }; // range of 4-byte sequence
private readonly BinaryReader _reader;
private readonly List<ContainerContext> _stack;
private byte[] _byteBuffer;
private char[] _charBuffer;
private BsonType _currentElementType;
private BsonReaderState _bsonReaderState;
private ContainerContext _currentContext;
@@ -489,33 +494,57 @@ namespace Newtonsoft.Json.Bson
StringBuilder builder = null;
int totalBytesRead = 0;
// used in case of left over multibyte characters in the buffer
int offset = 0;
do
{
int byteCount = 0;
int count = offset;
byte b;
while (byteCount < MaxCharBytesSize && (b = _reader.ReadByte()) > 0)
while (count < MaxCharBytesSize && (b = _reader.ReadByte()) > 0)
{
_byteBuffer[byteCount++] = b;
_byteBuffer[count++] = b;
}
int byteCount = count - offset;
totalBytesRead += byteCount;
int length = Encoding.UTF8.GetChars(_byteBuffer, 0, byteCount, _charBuffer, 0);
if (byteCount < MaxCharBytesSize && builder == null)
if (count < MaxCharBytesSize && builder == null)
{
// pref optimization to avoid reading into a string builder
// if string is smaller than the buffer then return it directly
int length = Encoding.UTF8.GetChars(_byteBuffer, 0, byteCount, _charBuffer, 0);
MovePosition(totalBytesRead + 1);
return new string(_charBuffer, 0, length);
}
if (builder == null)
builder = new StringBuilder(MaxCharBytesSize * 2);
builder.Append(_charBuffer, 0, length);
if (byteCount < MaxCharBytesSize)
else
{
MovePosition(totalBytesRead + 1);
return builder.ToString();
// calculate the index of the end of the last full character in the buffer
int lastFullCharStop = GetLastFullCharStop(count - 1);
int charCount = Encoding.UTF8.GetChars(_byteBuffer, 0, lastFullCharStop + 1, _charBuffer, 0);
if (builder == null)
builder = new StringBuilder(MaxCharBytesSize * 2);
builder.Append(_charBuffer, 0, charCount);
if (lastFullCharStop < byteCount - 1)
{
offset = byteCount - lastFullCharStop - 1;
// copy left over multi byte characters to beginning of buffer for next iteration
Array.Copy(_byteBuffer, lastFullCharStop + 1, _byteBuffer, 0, offset);
}
else
{
// reached end of string
if (count < MaxCharBytesSize)
{
MovePosition(totalBytesRead + 1);
return builder.ToString();
}
offset = 0;
}
}
}
while (true);
@@ -543,28 +572,98 @@ namespace Newtonsoft.Json.Bson
StringBuilder builder = null;
int totalBytesRead = 0;
int offset = 0;
do
{
int count = ((length - totalBytesRead) > MaxCharBytesSize) ? MaxCharBytesSize : (length - totalBytesRead);
int byteCount = _reader.BaseStream.Read(_byteBuffer, 0, count);
int count = ((length - totalBytesRead) > MaxCharBytesSize - offset)
? MaxCharBytesSize - offset
: length - totalBytesRead;
int byteCount = _reader.BaseStream.Read(_byteBuffer, offset, count);
if (byteCount == 0)
throw new EndOfStreamException("Unable to read beyond the end of the stream.");
int charCount = Encoding.UTF8.GetChars(_byteBuffer, 0, byteCount, _charBuffer, 0);
byteCount += offset;
if (totalBytesRead == 0 && byteCount == length)
{
// pref optimization to avoid reading into a string builder
// first iteration and all bytes read then return string directly
int charCount = Encoding.UTF8.GetChars(_byteBuffer, 0, byteCount, _charBuffer, 0);
return new string(_charBuffer, 0, charCount);
}
else
{
int lastFullCharStop = GetLastFullCharStop(byteCount - 1);
if (builder == null)
builder = new StringBuilder(length);
if (builder == null)
builder = new StringBuilder(length);
builder.Append(_charBuffer, 0, charCount);
totalBytesRead += byteCount;
int charCount = Encoding.UTF8.GetChars(_byteBuffer, 0, lastFullCharStop + 1, _charBuffer, 0);
builder.Append(_charBuffer, 0, charCount);
if (lastFullCharStop < byteCount - 1)
{
offset = byteCount - lastFullCharStop - 1;
// copy left over multi byte characters to beginning of buffer for next iteration
Array.Copy(_byteBuffer, lastFullCharStop + 1, _byteBuffer, 0, offset);
}
else
{
offset = 0;
}
totalBytesRead += (byteCount - offset);
}
}
while (totalBytesRead < length);
return builder.ToString();
}
private int GetLastFullCharStop(int start)
{
int lookbackPos = start;
int bis = 0;
while (lookbackPos >= 0)
{
bis = BytesInSequence(_byteBuffer[lookbackPos]);
if (bis == 0)
{
lookbackPos--;
continue;
}
else if (bis == 1)
{
break;
}
else
{
lookbackPos--;
break;
}
}
if (bis == start - lookbackPos)
{
//Full character.
return start;
}
else
{
return lookbackPos;
}
}
private int BytesInSequence(byte b)
{
if (b <= _seqRange1[1]) return 1;
if (b >= _seqRange2[0] && b <= _seqRange2[1]) return 2;
if (b >= _seqRange3[0] && b <= _seqRange3[1]) return 3;
if (b >= _seqRange4[0] && b <= _seqRange4[1]) return 4;
return 0;
}
private void EnsureBuffers()
{
if (_byteBuffer == null)