From d393430f30d769613950fcca4e1b2163191b59d4 Mon Sep 17 00:00:00 2001 From: JamesNK Date: Sat, 20 Mar 2010 01:29:02 +0000 Subject: [PATCH] -Fix reading BSON multi byte utf8 characters --- .../Bson/BsonReaderTests.cs | 201 ++++++++++++++++++ Src/Newtonsoft.Json/Bson/BsonReader.cs | 143 +++++++++++-- 2 files changed, 322 insertions(+), 22 deletions(-) diff --git a/Src/Newtonsoft.Json.Tests/Bson/BsonReaderTests.cs b/Src/Newtonsoft.Json.Tests/Bson/BsonReaderTests.cs index f8262781..8c6a0b39 100644 --- a/Src/Newtonsoft.Json.Tests/Bson/BsonReaderTests.cs +++ b/Src/Newtonsoft.Json.Tests/Bson/BsonReaderTests.cs @@ -38,6 +38,9 @@ namespace Newtonsoft.Json.Tests.Bson { public class BsonReaderTests : TestFixtureBase { + private char pound = '\u00a3'; + private char euro = '\u20ac'; + [Test] public void ReadSingleObject() { @@ -689,5 +692,203 @@ namespace Newtonsoft.Json.Tests.Bson o = (JObject)JToken.ReadFrom(reader); Assert.AreEqual(DateTime.SpecifyKind(value.ToLocalTime(), DateTimeKind.Unspecified), (DateTime)o["DateTime"]); } + + private string WriteAndReadStringValue(string val) + { + MemoryStream ms = new MemoryStream(); + BsonWriter bs = new BsonWriter(ms); + bs.WriteStartObject(); + bs.WritePropertyName("StringValue"); + bs.WriteValue(val); + bs.WriteEnd(); + + ms.Seek(0, SeekOrigin.Begin); + + BsonReader reader = new BsonReader(ms); + // object + reader.Read(); + // property name + reader.Read(); + // string + reader.Read(); + return (string)reader.Value; + } + + private string WriteAndReadStringPropertyName(string val) + { + MemoryStream ms = new MemoryStream(); + BsonWriter bs = new BsonWriter(ms); + bs.WriteStartObject(); + bs.WritePropertyName(val); + bs.WriteValue("Dummy"); + bs.WriteEnd(); + + ms.Seek(0, SeekOrigin.Begin); + + BsonReader reader = new BsonReader(ms); + // object + reader.Read(); + // property name + reader.Read(); + return (string)reader.Value; + } + + [Test] + public void TestReadLenStringValueShortTripleByte() + { + StringBuilder sb = new StringBuilder(); + //sb.Append('1',127); //first char of euro at the end of the boundry. + //sb.Append(euro, 5); + //sb.Append('1',128); + sb.Append(euro); + + string expected = sb.ToString(); + Assert.AreEqual(expected, WriteAndReadStringValue(expected)); + } + + [Test] + public void TestReadLenStringValueTripleByteCharBufferBoundry0() + { + StringBuilder sb = new StringBuilder(); + sb.Append('1', 127); //first char of euro at the end of the boundry. + sb.Append(euro, 5); + sb.Append('1', 128); + sb.Append(euro); + + string expected = sb.ToString(); + Assert.AreEqual(expected, WriteAndReadStringValue(expected)); + } + + [Test] + public void TestReadLenStringValueTripleByteCharBufferBoundry1() + { + StringBuilder sb = new StringBuilder(); + sb.Append('1', 126); + sb.Append(euro, 5); //middle char of euro at the end of the boundry. + sb.Append('1', 128); + sb.Append(euro); + + string expected = sb.ToString(); + string result = WriteAndReadStringValue(expected); + Assert.AreEqual(expected, result); + } + + [Test] + public void TestReadLenStringValueTripleByteCharOne() + { + StringBuilder sb = new StringBuilder(); + sb.Append(euro, 1); //Just one triple byte char in the string. + + string expected = sb.ToString(); + Assert.AreEqual(expected, WriteAndReadStringValue(expected)); + } + + [Test] + public void TestReadLenStringValueTripleByteCharBufferBoundry2() + { + StringBuilder sb = new StringBuilder(); + sb.Append('1', 125); + sb.Append(euro, 5); //last char of the eruo at the end of the boundry. + sb.Append('1', 128); + sb.Append(euro); + + string expected = sb.ToString(); + Assert.AreEqual(expected, WriteAndReadStringValue(expected)); + } + + [Test] + public void TestReadStringValue() + { + string expected = "test"; + Assert.AreEqual(expected, WriteAndReadStringValue(expected)); + } + + [Test] + public void TestReadStringValueLong() + { + StringBuilder sb = new StringBuilder(); + sb.Append('t', 150); + string expected = sb.ToString(); + Assert.AreEqual(expected, WriteAndReadStringValue(expected)); + } + + [Test] + public void TestReadStringPropertyNameShortTripleByte() + { + StringBuilder sb = new StringBuilder(); + //sb.Append('1',127); //first char of euro at the end of the boundry. + //sb.Append(euro, 5); + //sb.Append('1',128); + sb.Append(euro); + + string expected = sb.ToString(); + Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected)); + } + + [Test] + public void TestReadStringPropertyNameTripleByteCharBufferBoundry0() + { + StringBuilder sb = new StringBuilder(); + sb.Append('1', 127); //first char of euro at the end of the boundry. + sb.Append(euro, 5); + sb.Append('1', 128); + sb.Append(euro); + + string expected = sb.ToString(); + string result = WriteAndReadStringPropertyName(expected); + Assert.AreEqual(expected, result); + } + + [Test] + public void TestReadStringPropertyNameTripleByteCharBufferBoundry1() + { + StringBuilder sb = new StringBuilder(); + sb.Append('1', 126); + sb.Append(euro, 5); //middle char of euro at the end of the boundry. + sb.Append('1', 128); + sb.Append(euro); + + string expected = sb.ToString(); + Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected)); + } + + [Test] + public void TestReadStringPropertyNameTripleByteCharOne() + { + StringBuilder sb = new StringBuilder(); + sb.Append(euro, 1); //Just one triple byte char in the string. + + string expected = sb.ToString(); + Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected)); + } + + [Test] + public void TestReadStringPropertyNameTripleByteCharBufferBoundry2() + { + StringBuilder sb = new StringBuilder(); + sb.Append('1', 125); + sb.Append(euro, 5); //last char of the eruo at the end of the boundry. + sb.Append('1', 128); + sb.Append(euro); + + string expected = sb.ToString(); + Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected)); + } + + [Test] + public void TestReadStringPropertyName() + { + string expected = "test"; + Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected)); + } + + [Test] + public void TestReadStringPropertyNameLong() + { + StringBuilder sb = new StringBuilder(); + sb.Append('t', 150); + string expected = sb.ToString(); + Assert.AreEqual(expected, WriteAndReadStringPropertyName(expected)); + } } } \ No newline at end of file diff --git a/Src/Newtonsoft.Json/Bson/BsonReader.cs b/Src/Newtonsoft.Json/Bson/BsonReader.cs index a88deda9..f7858d06 100644 --- a/Src/Newtonsoft.Json/Bson/BsonReader.cs +++ b/Src/Newtonsoft.Json/Bson/BsonReader.cs @@ -39,12 +39,17 @@ namespace Newtonsoft.Json.Bson public class BsonReader : JsonReader { private const int MaxCharBytesSize = 128; + private static readonly byte[] _seqRange1 = new byte[] { 0, 127 }; // range of 1-byte sequence + private static readonly byte[] _seqRange2 = new byte[] { 194, 223 }; // range of 2-byte sequence + private static readonly byte[] _seqRange3 = new byte[] { 224, 239 }; // range of 3-byte sequence + private static readonly byte[] _seqRange4 = new byte[] { 240, 244 }; // range of 4-byte sequence private readonly BinaryReader _reader; private readonly List _stack; private byte[] _byteBuffer; private char[] _charBuffer; + private BsonType _currentElementType; private BsonReaderState _bsonReaderState; private ContainerContext _currentContext; @@ -489,33 +494,57 @@ namespace Newtonsoft.Json.Bson StringBuilder builder = null; int totalBytesRead = 0; + // used in case of left over multibyte characters in the buffer + int offset = 0; do { - int byteCount = 0; + int count = offset; byte b; - while (byteCount < MaxCharBytesSize && (b = _reader.ReadByte()) > 0) + while (count < MaxCharBytesSize && (b = _reader.ReadByte()) > 0) { - _byteBuffer[byteCount++] = b; + _byteBuffer[count++] = b; } + int byteCount = count - offset; totalBytesRead += byteCount; - int length = Encoding.UTF8.GetChars(_byteBuffer, 0, byteCount, _charBuffer, 0); - - if (byteCount < MaxCharBytesSize && builder == null) + if (count < MaxCharBytesSize && builder == null) { + // pref optimization to avoid reading into a string builder + // if string is smaller than the buffer then return it directly + int length = Encoding.UTF8.GetChars(_byteBuffer, 0, byteCount, _charBuffer, 0); + MovePosition(totalBytesRead + 1); return new string(_charBuffer, 0, length); } - - if (builder == null) - builder = new StringBuilder(MaxCharBytesSize * 2); - - builder.Append(_charBuffer, 0, length); - - if (byteCount < MaxCharBytesSize) + else { - MovePosition(totalBytesRead + 1); - return builder.ToString(); + // calculate the index of the end of the last full character in the buffer + int lastFullCharStop = GetLastFullCharStop(count - 1); + + int charCount = Encoding.UTF8.GetChars(_byteBuffer, 0, lastFullCharStop + 1, _charBuffer, 0); + + if (builder == null) + builder = new StringBuilder(MaxCharBytesSize * 2); + + builder.Append(_charBuffer, 0, charCount); + + if (lastFullCharStop < byteCount - 1) + { + offset = byteCount - lastFullCharStop - 1; + // copy left over multi byte characters to beginning of buffer for next iteration + Array.Copy(_byteBuffer, lastFullCharStop + 1, _byteBuffer, 0, offset); + } + else + { + // reached end of string + if (count < MaxCharBytesSize) + { + MovePosition(totalBytesRead + 1); + return builder.ToString(); + } + + offset = 0; + } } } while (true); @@ -543,28 +572,98 @@ namespace Newtonsoft.Json.Bson StringBuilder builder = null; int totalBytesRead = 0; + int offset = 0; do { - int count = ((length - totalBytesRead) > MaxCharBytesSize) ? MaxCharBytesSize : (length - totalBytesRead); - int byteCount = _reader.BaseStream.Read(_byteBuffer, 0, count); + int count = ((length - totalBytesRead) > MaxCharBytesSize - offset) + ? MaxCharBytesSize - offset + : length - totalBytesRead; + + int byteCount = _reader.BaseStream.Read(_byteBuffer, offset, count); + if (byteCount == 0) throw new EndOfStreamException("Unable to read beyond the end of the stream."); - int charCount = Encoding.UTF8.GetChars(_byteBuffer, 0, byteCount, _charBuffer, 0); + byteCount += offset; + if (totalBytesRead == 0 && byteCount == length) + { + // pref optimization to avoid reading into a string builder + // first iteration and all bytes read then return string directly + int charCount = Encoding.UTF8.GetChars(_byteBuffer, 0, byteCount, _charBuffer, 0); return new string(_charBuffer, 0, charCount); + } + else + { + int lastFullCharStop = GetLastFullCharStop(byteCount - 1); - if (builder == null) - builder = new StringBuilder(length); + if (builder == null) + builder = new StringBuilder(length); - builder.Append(_charBuffer, 0, charCount); - totalBytesRead += byteCount; + int charCount = Encoding.UTF8.GetChars(_byteBuffer, 0, lastFullCharStop + 1, _charBuffer, 0); + builder.Append(_charBuffer, 0, charCount); + + if (lastFullCharStop < byteCount - 1) + { + offset = byteCount - lastFullCharStop - 1; + // copy left over multi byte characters to beginning of buffer for next iteration + Array.Copy(_byteBuffer, lastFullCharStop + 1, _byteBuffer, 0, offset); + } + else + { + offset = 0; + } + + totalBytesRead += (byteCount - offset); + } } while (totalBytesRead < length); return builder.ToString(); } + private int GetLastFullCharStop(int start) + { + int lookbackPos = start; + int bis = 0; + while (lookbackPos >= 0) + { + bis = BytesInSequence(_byteBuffer[lookbackPos]); + if (bis == 0) + { + lookbackPos--; + continue; + } + else if (bis == 1) + { + break; + } + else + { + lookbackPos--; + break; + } + } + if (bis == start - lookbackPos) + { + //Full character. + return start; + } + else + { + return lookbackPos; + } + } + + private int BytesInSequence(byte b) + { + if (b <= _seqRange1[1]) return 1; + if (b >= _seqRange2[0] && b <= _seqRange2[1]) return 2; + if (b >= _seqRange3[0] && b <= _seqRange3[1]) return 3; + if (b >= _seqRange4[0] && b <= _seqRange4[1]) return 4; + return 0; + } + private void EnsureBuffers() { if (_byteBuffer == null)