-Fix reading BSON multi byte utf8 characters
This commit is contained in:
@@ -38,6 +38,9 @@ namespace Newtonsoft.Json.Tests.Bson
|
|||||||
{
|
{
|
||||||
public class BsonReaderTests : TestFixtureBase
|
public class BsonReaderTests : TestFixtureBase
|
||||||
{
|
{
|
||||||
|
private char pound = '\u00a3';
|
||||||
|
private char euro = '\u20ac';
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
public void ReadSingleObject()
|
public void ReadSingleObject()
|
||||||
{
|
{
|
||||||
@@ -689,5 +692,203 @@ namespace Newtonsoft.Json.Tests.Bson
|
|||||||
o = (JObject)JToken.ReadFrom(reader);
|
o = (JObject)JToken.ReadFrom(reader);
|
||||||
Assert.AreEqual(DateTime.SpecifyKind(value.ToLocalTime(), DateTimeKind.Unspecified), (DateTime)o["DateTime"]);
|
Assert.AreEqual(DateTime.SpecifyKind(value.ToLocalTime(), DateTimeKind.Unspecified), (DateTime)o["DateTime"]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private string WriteAndReadStringValue(string val)
|
||||||
|
{
|
||||||
|
MemoryStream ms = new MemoryStream();
|
||||||
|
BsonWriter bs = new BsonWriter(ms);
|
||||||
|
bs.WriteStartObject();
|
||||||
|
bs.WritePropertyName("StringValue");
|
||||||
|
bs.WriteValue(val);
|
||||||
|
bs.WriteEnd();
|
||||||
|
|
||||||
|
ms.Seek(0, SeekOrigin.Begin);
|
||||||
|
|
||||||
|
BsonReader reader = new BsonReader(ms);
|
||||||
|
// object
|
||||||
|
reader.Read();
|
||||||
|
// property name
|
||||||
|
reader.Read();
|
||||||
|
// string
|
||||||
|
reader.Read();
|
||||||
|
return (string)reader.Value;
|
||||||
|
}
|
||||||
|
|
||||||
|
private string WriteAndReadStringPropertyName(string val)
|
||||||
|
{
|
||||||
|
MemoryStream ms = new MemoryStream();
|
||||||
|
BsonWriter bs = new BsonWriter(ms);
|
||||||
|
bs.WriteStartObject();
|
||||||
|
bs.WritePropertyName(val);
|
||||||
|
bs.WriteValue("Dummy");
|
||||||
|
bs.WriteEnd();
|
||||||
|
|
||||||
|
ms.Seek(0, SeekOrigin.Begin);
|
||||||
|
|
||||||
|
BsonReader reader = new BsonReader(ms);
|
||||||
|
// object
|
||||||
|
reader.Read();
|
||||||
|
// property name
|
||||||
|
reader.Read();
|
||||||
|
return (string)reader.Value;
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TestReadLenStringValueShortTripleByte()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
//sb.Append('1',127); //first char of euro at the end of the boundry.
|
||||||
|
//sb.Append(euro, 5);
|
||||||
|
//sb.Append('1',128);
|
||||||
|
sb.Append(euro);
|
||||||
|
|
||||||
|
string expected = sb.ToString();
|
||||||
|
Assert.AreEqual(expected, WriteAndReadStringValue(expected));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TestReadLenStringValueTripleByteCharBufferBoundry0()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.Append('1', 127); //first char of euro at the end of the boundry.
|
||||||
|
sb.Append(euro, 5);
|
||||||
|
sb.Append('1', 128);
|
||||||
|
sb.Append(euro);
|
||||||
|
|
||||||
|
string expected = sb.ToString();
|
||||||
|
Assert.AreEqual(expected, WriteAndReadStringValue(expected));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TestReadLenStringValueTripleByteCharBufferBoundry1()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.Append('1', 126);
|
||||||
|
sb.Append(euro, 5); //middle char of euro at the end of the boundry.
|
||||||
|
sb.Append('1', 128);
|
||||||
|
sb.Append(euro);
|
||||||
|
|
||||||
|
string expected = sb.ToString();
|
||||||
|
string result = WriteAndReadStringValue(expected);
|
||||||
|
Assert.AreEqual(expected, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TestReadLenStringValueTripleByteCharOne()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.Append(euro, 1); //Just one triple byte char in the string.
|
||||||
|
|
||||||
|
string expected = sb.ToString();
|
||||||
|
Assert.AreEqual(expected, WriteAndReadStringValue(expected));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TestReadLenStringValueTripleByteCharBufferBoundry2()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.Append('1', 125);
|
||||||
|
sb.Append(euro, 5); //last char of the eruo at the end of the boundry.
|
||||||
|
sb.Append('1', 128);
|
||||||
|
sb.Append(euro);
|
||||||
|
|
||||||
|
string expected = sb.ToString();
|
||||||
|
Assert.AreEqual(expected, WriteAndReadStringValue(expected));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TestReadStringValue()
|
||||||
|
{
|
||||||
|
string expected = "test";
|
||||||
|
Assert.AreEqual(expected, WriteAndReadStringValue(expected));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TestReadStringValueLong()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.Append('t', 150);
|
||||||
|
string expected = sb.ToString();
|
||||||
|
Assert.AreEqual(expected, WriteAndReadStringValue(expected));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TestReadStringPropertyNameShortTripleByte()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
//sb.Append('1',127); //first char of euro at the end of the boundry.
|
||||||
|
//sb.Append(euro, 5);
|
||||||
|
//sb.Append('1',128);
|
||||||
|
sb.Append(euro);
|
||||||
|
|
||||||
|
string expected = sb.ToString();
|
||||||
|
Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TestReadStringPropertyNameTripleByteCharBufferBoundry0()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.Append('1', 127); //first char of euro at the end of the boundry.
|
||||||
|
sb.Append(euro, 5);
|
||||||
|
sb.Append('1', 128);
|
||||||
|
sb.Append(euro);
|
||||||
|
|
||||||
|
string expected = sb.ToString();
|
||||||
|
string result = WriteAndReadStringPropertyName(expected);
|
||||||
|
Assert.AreEqual(expected, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TestReadStringPropertyNameTripleByteCharBufferBoundry1()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.Append('1', 126);
|
||||||
|
sb.Append(euro, 5); //middle char of euro at the end of the boundry.
|
||||||
|
sb.Append('1', 128);
|
||||||
|
sb.Append(euro);
|
||||||
|
|
||||||
|
string expected = sb.ToString();
|
||||||
|
Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TestReadStringPropertyNameTripleByteCharOne()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.Append(euro, 1); //Just one triple byte char in the string.
|
||||||
|
|
||||||
|
string expected = sb.ToString();
|
||||||
|
Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TestReadStringPropertyNameTripleByteCharBufferBoundry2()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.Append('1', 125);
|
||||||
|
sb.Append(euro, 5); //last char of the eruo at the end of the boundry.
|
||||||
|
sb.Append('1', 128);
|
||||||
|
sb.Append(euro);
|
||||||
|
|
||||||
|
string expected = sb.ToString();
|
||||||
|
Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TestReadStringPropertyName()
|
||||||
|
{
|
||||||
|
string expected = "test";
|
||||||
|
Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TestReadStringPropertyNameLong()
|
||||||
|
{
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.Append('t', 150);
|
||||||
|
string expected = sb.ToString();
|
||||||
|
Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -39,12 +39,17 @@ namespace Newtonsoft.Json.Bson
|
|||||||
public class BsonReader : JsonReader
|
public class BsonReader : JsonReader
|
||||||
{
|
{
|
||||||
private const int MaxCharBytesSize = 128;
|
private const int MaxCharBytesSize = 128;
|
||||||
|
private static readonly byte[] _seqRange1 = new byte[] { 0, 127 }; // range of 1-byte sequence
|
||||||
|
private static readonly byte[] _seqRange2 = new byte[] { 194, 223 }; // range of 2-byte sequence
|
||||||
|
private static readonly byte[] _seqRange3 = new byte[] { 224, 239 }; // range of 3-byte sequence
|
||||||
|
private static readonly byte[] _seqRange4 = new byte[] { 240, 244 }; // range of 4-byte sequence
|
||||||
|
|
||||||
private readonly BinaryReader _reader;
|
private readonly BinaryReader _reader;
|
||||||
private readonly List<ContainerContext> _stack;
|
private readonly List<ContainerContext> _stack;
|
||||||
|
|
||||||
private byte[] _byteBuffer;
|
private byte[] _byteBuffer;
|
||||||
private char[] _charBuffer;
|
private char[] _charBuffer;
|
||||||
|
|
||||||
private BsonType _currentElementType;
|
private BsonType _currentElementType;
|
||||||
private BsonReaderState _bsonReaderState;
|
private BsonReaderState _bsonReaderState;
|
||||||
private ContainerContext _currentContext;
|
private ContainerContext _currentContext;
|
||||||
@@ -489,33 +494,57 @@ namespace Newtonsoft.Json.Bson
|
|||||||
StringBuilder builder = null;
|
StringBuilder builder = null;
|
||||||
|
|
||||||
int totalBytesRead = 0;
|
int totalBytesRead = 0;
|
||||||
|
// used in case of left over multibyte characters in the buffer
|
||||||
|
int offset = 0;
|
||||||
do
|
do
|
||||||
{
|
{
|
||||||
int byteCount = 0;
|
int count = offset;
|
||||||
byte b;
|
byte b;
|
||||||
while (byteCount < MaxCharBytesSize && (b = _reader.ReadByte()) > 0)
|
while (count < MaxCharBytesSize && (b = _reader.ReadByte()) > 0)
|
||||||
{
|
{
|
||||||
_byteBuffer[byteCount++] = b;
|
_byteBuffer[count++] = b;
|
||||||
}
|
}
|
||||||
|
int byteCount = count - offset;
|
||||||
totalBytesRead += byteCount;
|
totalBytesRead += byteCount;
|
||||||
|
|
||||||
int length = Encoding.UTF8.GetChars(_byteBuffer, 0, byteCount, _charBuffer, 0);
|
if (count < MaxCharBytesSize && builder == null)
|
||||||
|
|
||||||
if (byteCount < MaxCharBytesSize && builder == null)
|
|
||||||
{
|
{
|
||||||
|
// pref optimization to avoid reading into a string builder
|
||||||
|
// if string is smaller than the buffer then return it directly
|
||||||
|
int length = Encoding.UTF8.GetChars(_byteBuffer, 0, byteCount, _charBuffer, 0);
|
||||||
|
|
||||||
MovePosition(totalBytesRead + 1);
|
MovePosition(totalBytesRead + 1);
|
||||||
return new string(_charBuffer, 0, length);
|
return new string(_charBuffer, 0, length);
|
||||||
}
|
}
|
||||||
|
else
|
||||||
if (builder == null)
|
|
||||||
builder = new StringBuilder(MaxCharBytesSize * 2);
|
|
||||||
|
|
||||||
builder.Append(_charBuffer, 0, length);
|
|
||||||
|
|
||||||
if (byteCount < MaxCharBytesSize)
|
|
||||||
{
|
{
|
||||||
MovePosition(totalBytesRead + 1);
|
// calculate the index of the end of the last full character in the buffer
|
||||||
return builder.ToString();
|
int lastFullCharStop = GetLastFullCharStop(count - 1);
|
||||||
|
|
||||||
|
int charCount = Encoding.UTF8.GetChars(_byteBuffer, 0, lastFullCharStop + 1, _charBuffer, 0);
|
||||||
|
|
||||||
|
if (builder == null)
|
||||||
|
builder = new StringBuilder(MaxCharBytesSize * 2);
|
||||||
|
|
||||||
|
builder.Append(_charBuffer, 0, charCount);
|
||||||
|
|
||||||
|
if (lastFullCharStop < byteCount - 1)
|
||||||
|
{
|
||||||
|
offset = byteCount - lastFullCharStop - 1;
|
||||||
|
// copy left over multi byte characters to beginning of buffer for next iteration
|
||||||
|
Array.Copy(_byteBuffer, lastFullCharStop + 1, _byteBuffer, 0, offset);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// reached end of string
|
||||||
|
if (count < MaxCharBytesSize)
|
||||||
|
{
|
||||||
|
MovePosition(totalBytesRead + 1);
|
||||||
|
return builder.ToString();
|
||||||
|
}
|
||||||
|
|
||||||
|
offset = 0;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
while (true);
|
while (true);
|
||||||
@@ -543,28 +572,98 @@ namespace Newtonsoft.Json.Bson
|
|||||||
StringBuilder builder = null;
|
StringBuilder builder = null;
|
||||||
|
|
||||||
int totalBytesRead = 0;
|
int totalBytesRead = 0;
|
||||||
|
int offset = 0;
|
||||||
do
|
do
|
||||||
{
|
{
|
||||||
int count = ((length - totalBytesRead) > MaxCharBytesSize) ? MaxCharBytesSize : (length - totalBytesRead);
|
int count = ((length - totalBytesRead) > MaxCharBytesSize - offset)
|
||||||
int byteCount = _reader.BaseStream.Read(_byteBuffer, 0, count);
|
? MaxCharBytesSize - offset
|
||||||
|
: length - totalBytesRead;
|
||||||
|
|
||||||
|
int byteCount = _reader.BaseStream.Read(_byteBuffer, offset, count);
|
||||||
|
|
||||||
if (byteCount == 0)
|
if (byteCount == 0)
|
||||||
throw new EndOfStreamException("Unable to read beyond the end of the stream.");
|
throw new EndOfStreamException("Unable to read beyond the end of the stream.");
|
||||||
|
|
||||||
int charCount = Encoding.UTF8.GetChars(_byteBuffer, 0, byteCount, _charBuffer, 0);
|
byteCount += offset;
|
||||||
|
|
||||||
if (totalBytesRead == 0 && byteCount == length)
|
if (totalBytesRead == 0 && byteCount == length)
|
||||||
|
{
|
||||||
|
// pref optimization to avoid reading into a string builder
|
||||||
|
// first iteration and all bytes read then return string directly
|
||||||
|
int charCount = Encoding.UTF8.GetChars(_byteBuffer, 0, byteCount, _charBuffer, 0);
|
||||||
return new string(_charBuffer, 0, charCount);
|
return new string(_charBuffer, 0, charCount);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
int lastFullCharStop = GetLastFullCharStop(byteCount - 1);
|
||||||
|
|
||||||
if (builder == null)
|
if (builder == null)
|
||||||
builder = new StringBuilder(length);
|
builder = new StringBuilder(length);
|
||||||
|
|
||||||
builder.Append(_charBuffer, 0, charCount);
|
int charCount = Encoding.UTF8.GetChars(_byteBuffer, 0, lastFullCharStop + 1, _charBuffer, 0);
|
||||||
totalBytesRead += byteCount;
|
builder.Append(_charBuffer, 0, charCount);
|
||||||
|
|
||||||
|
if (lastFullCharStop < byteCount - 1)
|
||||||
|
{
|
||||||
|
offset = byteCount - lastFullCharStop - 1;
|
||||||
|
// copy left over multi byte characters to beginning of buffer for next iteration
|
||||||
|
Array.Copy(_byteBuffer, lastFullCharStop + 1, _byteBuffer, 0, offset);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
offset = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
totalBytesRead += (byteCount - offset);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
while (totalBytesRead < length);
|
while (totalBytesRead < length);
|
||||||
|
|
||||||
return builder.ToString();
|
return builder.ToString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private int GetLastFullCharStop(int start)
|
||||||
|
{
|
||||||
|
int lookbackPos = start;
|
||||||
|
int bis = 0;
|
||||||
|
while (lookbackPos >= 0)
|
||||||
|
{
|
||||||
|
bis = BytesInSequence(_byteBuffer[lookbackPos]);
|
||||||
|
if (bis == 0)
|
||||||
|
{
|
||||||
|
lookbackPos--;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
else if (bis == 1)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
lookbackPos--;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (bis == start - lookbackPos)
|
||||||
|
{
|
||||||
|
//Full character.
|
||||||
|
return start;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
return lookbackPos;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private int BytesInSequence(byte b)
|
||||||
|
{
|
||||||
|
if (b <= _seqRange1[1]) return 1;
|
||||||
|
if (b >= _seqRange2[0] && b <= _seqRange2[1]) return 2;
|
||||||
|
if (b >= _seqRange3[0] && b <= _seqRange3[1]) return 3;
|
||||||
|
if (b >= _seqRange4[0] && b <= _seqRange4[1]) return 4;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
private void EnsureBuffers()
|
private void EnsureBuffers()
|
||||||
{
|
{
|
||||||
if (_byteBuffer == null)
|
if (_byteBuffer == null)
|
||||||
|
|||||||
Reference in New Issue
Block a user