-Fix JTokenReader.ReadAsBytes and null values

-Fix JsonProperty inheritance on overriden properties

Src/Newtonsoft.Json.Tests/Bson/BsonReaderTests.cs

@@ -1064,5 +1064,35 @@ namespace Newtonsoft.Json.Tests.Bson
 
       Assert.IsFalse(reader.Read());
     }
+
+    [Test]
+    public void MultibyteCharacterPropertyNamesAndStrings()
+    {
+      string json = @"{
+  ""ΕΝΤΟΛΗ ΧΧΧ ΧΧΧΧΧΧΧΧΧ ΤΑ ΠΡΩΤΑΣΦΑΛΙΣΤΗΡΙΑ ΠΟΥ ΔΕΝ ΕΧΟΥΝ ΥΠΟΛΟΙΠΟ ΝΑ ΤΑ ΣΤΕΛΝΟΥΜΕ ΑΠΕΥΘΕΙΑΣ ΣΤΟΥΣ ΠΕΛΑΤΕΣ"": ""ΕΝΤΟΛΗ ΧΧΧ ΧΧΧΧΧΧΧΧΧ ΤΑ ΠΡΩΤΑΣΦΑΛΙΣΤΗΡΙΑ ΠΟΥ ΔΕΝ ΕΧΟΥΝ ΥΠΟΛΟΙΠΟ ΝΑ ΤΑ ΣΤΕΛΝΟΥΜΕ ΑΠΕΥΘΕΙΑΣ ΣΤΟΥΣ ΠΕΛΑΤΕΣ""
+}";
+      JObject parsed = JObject.Parse(json);
+      var memoryStream = new MemoryStream();
+      var bsonWriter = new BsonWriter(memoryStream);
+      parsed.WriteTo(bsonWriter);
+      bsonWriter.Flush();
+      memoryStream.Position = 0;
+
+      BsonReader reader = new BsonReader(memoryStream);
+
+      Assert.IsTrue(reader.Read());
+      Assert.AreEqual(JsonToken.StartObject, reader.TokenType);
+
+      Assert.IsTrue(reader.Read());
+      Assert.AreEqual(JsonToken.PropertyName, reader.TokenType);
+      Assert.AreEqual("ΕΝΤΟΛΗ ΧΧΧ ΧΧΧΧΧΧΧΧΧ ΤΑ ΠΡΩΤΑΣΦΑΛΙΣΤΗΡΙΑ ΠΟΥ ΔΕΝ ΕΧΟΥΝ ΥΠΟΛΟΙΠΟ ΝΑ ΤΑ ΣΤΕΛΝΟΥΜΕ ΑΠΕΥΘΕΙΑΣ ΣΤΟΥΣ ΠΕΛΑΤΕΣ", reader.Value);
+
+      Assert.IsTrue(reader.Read());
+      Assert.AreEqual(JsonToken.String, reader.TokenType);
+      Assert.AreEqual("ΕΝΤΟΛΗ ΧΧΧ ΧΧΧΧΧΧΧΧΧ ΤΑ ΠΡΩΤΑΣΦΑΛΙΣΤΗΡΙΑ ΠΟΥ ΔΕΝ ΕΧΟΥΝ ΥΠΟΛΟΙΠΟ ΝΑ ΤΑ ΣΤΕΛΝΟΥΜΕ ΑΠΕΥΘΕΙΑΣ ΣΤΟΥΣ ΠΕΛΑΤΕΣ", reader.Value);
+
+      Assert.IsTrue(reader.Read());
+      Assert.AreEqual(JsonToken.EndObject, reader.TokenType);
+    }
   }
 }

Src/Newtonsoft.Json.Tests/JsonTextReaderTest.cs

@@ -238,7 +238,7 @@ Parameter name: reader")]
     }
 
     [Test]
-    [ExpectedException(typeof(JsonReaderException), ExpectedMessage = "Unexpected character encountered while parsing value: t. Line 1, position 1.")]
+    [ExpectedException(typeof(JsonReaderException), ExpectedMessage = "Unexpected token when reading bytes: Boolean. Line 1, position 4.")]
     public void ReadBytesWithBadCharacter()
     {
       JsonReader reader = new JsonTextReader(new StringReader(@"true"));

Src/Newtonsoft.Json.Tests/Linq/JTokenReaderTest.cs

@@ -290,5 +290,20 @@ namespace Newtonsoft.Json.Tests.Linq
 
       Assert.AreEqual(new byte[0], result2.Bytes);
     }
+
+    public class ReadAsBytesTestObject
+    {
+      public byte[] Data;
+    }
+
+    [Test]
+    public void ReadAsBytesNull()
+    {
+      JsonSerializer s = new JsonSerializer();
+
+      JToken nullToken = JToken.ReadFrom(new JsonTextReader(new StringReader("{ Data: null }")));
+      ReadAsBytesTestObject x = s.Deserialize<ReadAsBytesTestObject>(new JTokenReader(nullToken));
+      Assert.IsNull(x.Data);
+    }
   }
 }

Src/Newtonsoft.Json.Tests/Serialization/JsonSerializerTest.cs

@@ -3156,7 +3156,7 @@ keyword such as type of business.""
       public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
       {
         List<string> existingStrings = (List<string>)existingValue;
-        List<string> newStrings = new List<string>(existingStrings);        
+        List<string> newStrings = new List<string>(existingStrings);
 
         reader.Read();
 
@@ -3173,7 +3173,7 @@ keyword such as type of business.""
 
       public override bool CanConvert(Type objectType)
       {
-        return (objectType == typeof (List<string>));
+        return (objectType == typeof(List<string>));
       }
     }
 
@@ -3357,7 +3357,7 @@ keyword such as type of business.""
       joe.Manager = mike;
       mike.Manager = mike;
 
-      string json = JsonConvert.SerializeObject(new []{ joe, mike }, Formatting.Indented);
+      string json = JsonConvert.SerializeObject(new[] { joe, mike }, Formatting.Indented);
       // [
       //   {
       //     "Name": "Joe Employee",
@@ -3384,7 +3384,7 @@ keyword such as type of business.""
 
       public static implicit operator DictionaryKey(string value)
       {
-        return new DictionaryKey() {Value = value};
+        return new DictionaryKey() { Value = value };
       }
     }
 
@@ -3533,5 +3533,45 @@ keyword such as type of business.""
 
       Assert.AreEqual(int.MaxValue, newStruct.MyInt);
     }
+
+    //public class TimeZoneOffsetObject
+    //{
+    //  public DateTimeOffset Offset { get; set; }
+    //}
+
+    //[Test]
+    //public void ReadWriteTimeZoneOffset()
+    //{
+    //  var serializeObject = JsonConvert.SerializeObject(new TimeZoneOffsetObject
+    //  {
+    //    Offset = new DateTimeOffset(new DateTime(2000, 1, 1), TimeSpan.FromHours(6))
+    //  });
+
+    //  Assert.AreEqual("{\"Offset\":\"\\/Date(946663200000+0600)\\/\"}", serializeObject);
+    //  var deserializeObject = JsonConvert.DeserializeObject<TimeZoneOffsetObject>(serializeObject);
+    //  Assert.AreEqual(TimeSpan.FromHours(6), deserializeObject.Offset.Offset); // fails here
+    //  Assert.AreEqual(new DateTime(2000, 1, 1), deserializeObject.Offset.Date);
+    //}
+
+    public abstract class LogEvent
+    {
+      [JsonProperty("event")]
+      public abstract string EventName { get; }
+    }
+
+    public class DerivedEvent : LogEvent
+    {
+      public override string EventName { get { return "derived"; } }
+    }
+
+    [Test]
+    public void OverridenPropertyMembers()
+    {
+      string json = JsonConvert.SerializeObject(new DerivedEvent(), Formatting.Indented);
+
+      Assert.AreEqual(@"{
+  ""event"": ""derived""
+}", json);
+    }
   }
 }

Src/Newtonsoft.Json/JsonTextReader.cs

@@ -38,12 +38,19 @@ namespace Newtonsoft.Json
   /// </summary>
   public class JsonTextReader : JsonReader, IJsonLineInfo
   {
+    private enum ReadType
+    {
+      Read,
+      ReadAsBytes
+    }
+
     private readonly TextReader _reader;
     private readonly StringBuffer _buffer;
     private char? _lastChar;
     private int _currentLinePosition;
     private int _currentLineNumber;
     private bool _end;
+    private ReadType _readType;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="JsonReader"/> class with the specified <see cref="TextReader"/>.
@@ -63,17 +70,35 @@ namespace Newtonsoft.Json
     {
       ReadStringIntoBuffer(quote);
 
-      string text = _buffer.ToString();
-      _buffer.Position = 0;
-
-      if (text.StartsWith("/Date(", StringComparison.Ordinal) && text.EndsWith(")/", StringComparison.Ordinal))
+      if (_readType == ReadType.ReadAsBytes)
       {
-        ParseDate(text);
+        byte[] data;
+        if (_buffer.Position == 0)
+        {
+          data = new byte[0];
+        }
+        else
+        {
+          data = Convert.FromBase64CharArray(_buffer.GetInternalBuffer(), 0, _buffer.Position);
+          _buffer.Position = 0;
+        }
+
+        SetToken(JsonToken.Bytes, data);
       }
       else
       {
-        SetToken(JsonToken.String, text);
-        QuoteChar = quote;
+        string text = _buffer.ToString();
+        _buffer.Position = 0;
+
+        if (text.StartsWith("/Date(", StringComparison.Ordinal) && text.EndsWith(")/", StringComparison.Ordinal))
+        {
+          ParseDate(text);
+        }
+        else
+        {
+          SetToken(JsonToken.String, text);
+          QuoteChar = quote;
+        } 
       }
     }
 
@@ -249,6 +274,32 @@ namespace Newtonsoft.Json
     /// true if the next token was read successfully; false if there are no more tokens to read.
     /// </returns>
     public override bool Read()
+    {
+      _readType = ReadType.Read;
+      return ReadInternal();
+    }
+
+    /// <summary>
+    /// Reads the next JSON token from the stream as a <see cref="T:Byte[]"/>.
+    /// </summary>
+    /// <returns>
+    /// A <see cref="T:Byte[]"/> or a null reference if the next JSON token is null.
+    /// </returns>
+    public override byte[] ReadAsBytes()
+    {
+      _readType = ReadType.ReadAsBytes;
+      if (!ReadInternal())
+        throw CreateJsonReaderException("Unexpected end when reading bytes: Line {0}, position {1}.", _currentLineNumber, _currentLinePosition);
+
+      if (TokenType == JsonToken.Null)
+        return null;
+      if (TokenType == JsonToken.Bytes)
+        return (byte[]) Value;
+
+      throw CreateJsonReaderException("Unexpected token when reading bytes: {0}. Line {1}, position {2}.", TokenType, _currentLineNumber, _currentLinePosition);
+    }
+
+    private bool ReadInternal()
     {
       while (true)
       {
@@ -296,92 +347,6 @@ namespace Newtonsoft.Json
       }
     }
 
-    /// <summary>
-    /// Reads the next JSON token from the stream as a <see cref="T:Byte[]"/>.
-    /// </summary>
-    /// <returns>
-    /// A <see cref="T:Byte[]"/> or a null reference if the next JSON token is null.
-    /// </returns>
-    public override byte[] ReadAsBytes()
-    {
-      while (true)
-      {
-        char currentChar;
-        if (_lastChar != null)
-        {
-          currentChar = _lastChar.Value;
-          _lastChar = null;
-        }
-        else
-        {
-          currentChar = MoveNext();
-        }
-
-        if (currentChar == '\0' && _end)
-          throw CreateJsonReaderException("Unexpected end when reading bytes: Line {0}, position {1}.", _currentLineNumber, _currentLinePosition);
-
-        switch (CurrentState)
-        {
-          case State.PostValue:
-          case State.Start:
-          case State.Property:
-          case State.Array:
-          case State.ArrayStart:
-          case State.Constructor:
-          case State.ConstructorStart:
-            do
-            {
-              switch (currentChar)
-              {
-                case '"':
-                case '\'':
-                  ReadStringIntoBuffer(currentChar);
-
-                  byte[] data;
-                  if (_buffer.Position == 0)
-                  {
-                    data = new byte[0];
-                  }
-                  else
-                  {
-                    data = Convert.FromBase64CharArray(_buffer.GetInternalBuffer(), 0, _buffer.Position);
-                    _buffer.Position = 0;
-                  }
-
-                  SetToken(JsonToken.Bytes, data);
-
-                  return data;
-                case 'n':
-                  ParseNull();
-                  return null;
-                case ' ':
-                case StringUtils.Tab:
-                case StringUtils.LineFeed:
-                case StringUtils.CarriageReturn:
-                  // eat
-                  break;
-                case ',':
-                  SetStateBasedOnCurrent();
-                  break;
-                default:
-                  if (char.IsWhiteSpace(currentChar))
-                  {
-                    // eat
-                  }
-                  else
-                  {
-                    throw CreateJsonReaderException("Unexpected character encountered while parsing value: {0}. Line {1}, position {2}.", currentChar, _currentLineNumber, _currentLinePosition);
-                  }
-                  break;
-              }
-            } while ((currentChar = MoveNext()) != '\0' || !_end);
-            break;
-          default:
-            throw CreateJsonReaderException("Unexpected state: {0}. Line {1}, position {2}.", CurrentState, _currentLineNumber, _currentLinePosition);
-        }
-      }
-    }
-
     private bool ParsePostValue(char currentChar)
     {
       do

Src/Newtonsoft.Json/Linq/JTokenReader.cs

@@ -46,10 +46,12 @@ namespace Newtonsoft.Json.Linq
         SetToken(JsonToken.Bytes, data);
       }
 
-      if (TokenType != JsonToken.Bytes)
-        throw new JsonReaderException("Error reading bytes. Expected bytes but got {0}.".FormatWith(CultureInfo.InvariantCulture, TokenType));
+      if (TokenType == JsonToken.Bytes)
+        return (byte[])Value;
+      if (TokenType == JsonToken.Null)
+        return null;
 
-      return (byte[])Value;
+      throw new JsonReaderException("Error reading bytes. Expected bytes but got {0}.".FormatWith(CultureInfo.InvariantCulture, TokenType));
     }
 
     /// <summary>

Src/Newtonsoft.Json/Serialization/JsonSerializerInternalReader.cs

@@ -865,8 +865,7 @@ namespace Newtonsoft.Json.Serialization
             else
             {
               if (!reader.Read())
-                throw new JsonSerializationException(
-                  "Unexpected end when setting {0}'s value.".FormatWith(CultureInfo.InvariantCulture, memberName));
+                throw new JsonSerializationException("Unexpected end when setting {0}'s value.".FormatWith(CultureInfo.InvariantCulture, memberName));
             }
 
             SetRequiredProperty(reader, property, requiredProperties);

Src/Newtonsoft.Json/Utilities/ReflectionUtils.cs

@@ -696,6 +696,21 @@ namespace Newtonsoft.Json.Utilities
     {
       ValidationUtils.ArgumentNotNull(attributeProvider, "attributeProvider");
 
+      // http://hyperthink.net/blog/getcustomattributes-gotcha/
+      // ICustomAttributeProvider doesn't do inheritance
+
+      if (attributeProvider is Assembly)
+        return (T[])Attribute.GetCustomAttributes((Assembly)attributeProvider, typeof(T), inherit);
+
+      if (attributeProvider is MemberInfo)
+        return (T[])Attribute.GetCustomAttributes((MemberInfo)attributeProvider, typeof(T), inherit);
+
+      if (attributeProvider is Module)
+        return (T[])Attribute.GetCustomAttributes((Module)attributeProvider, typeof(T), inherit);
+
+      if (attributeProvider is ParameterInfo)
+        return (T[])Attribute.GetCustomAttributes((ParameterInfo)attributeProvider, typeof(T), inherit);
+
       return (T[])attributeProvider.GetCustomAttributes(typeof(T), inherit);
     }