jpath regex timeout support added for a single regex expression, global umbrella for all regex calls, and support for allowing regex calls to get compiled if necessary

require users to set a global cancellation token themselves, to avoid a potential unmanaged memory leak, which cleans up the code a bunch and fixes the failing tests

fix tests so cached regex's don't cause one test to fail using prior tests timeout...

handle the global timeout of multitoken regex internally and test for it
This commit is contained in:
serpentfabric
2021-01-17 18:23:19 +13:00
committed by James Newton-King
parent 1403f5d329
commit 95a6eb3ac5
8 changed files with 199 additions and 18 deletions
@@ -31,6 +31,13 @@ using System.Numerics;
#endif
using Newtonsoft.Json.Linq.JsonPath;
using Newtonsoft.Json.Tests.Bson;
#if HAVE_REGEX_TIMEOUTS
using Bogus;
using Nito.AsyncEx;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Threading;
#endif
#if DNXCORE50
using Xunit;
using Test = Xunit.FactAttribute;
@@ -44,7 +51,6 @@ using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Utilities.LinqBridge;
#else
using System.Linq;
#endif
namespace Newtonsoft.Json.Tests.Linq.JsonPath
@@ -71,6 +77,75 @@ namespace Newtonsoft.Json.Tests.Linq.JsonPath
Assert.AreEqual(jObj, dd);
}
#if HAVE_REGEX_TIMEOUTS
[Test]
public void BacktrackingRegex_SingleMatch_TimeoutRespected()
{
var RegexBacktrackingPattern = "(?<a>(.*?))[|].*(?<b>(.*?))[|].*(?<c>(.*?))[|].*(?<d>[1-3])[|].*(?<e>(.*?))[|].*[|].*[|].*(?<f>(.*?))[|].*[|].*(?<g>(.*?))[|].*(?<h>(.*))";
var faker = new Faker();
var regexBacktrackingData = new JArray();
for (var i = 0; i < 1000; i++)
{
var value = $"{faker.Date.Past()}|1|{faker.Lorem.Words()}|3|{faker.Lorem.Sentences(3, ". ")}|||.\\{faker.Lorem.Word()}.cpp||{faker.Random.UShort()}|-1";
regexBacktrackingData.Add(new JObject(new JProperty("b", value)));
}
Xunit.Assert.Throws<RegexMatchTimeoutException>(() =>
{
var tokens = regexBacktrackingData.SelectTokens(
$"[?(@.b =~ /{RegexBacktrackingPattern}/)]",
errorWhenNoMatch: false,
singleRegexMatchTimeout: TimeSpan.FromSeconds(.01)).ToArray();
});
}
[Test]
public void BacktrackingRegex_GlobalMatch_TimeoutRespected()
{
var faker = new Faker();
var regexBacktrackingData = new JArray();
for (var i = 0; i < 1000; i++)
{
var value = $"{faker.Date.Past()}|1|{faker.Lorem.Words()}|3|{faker.Lorem.Sentences(3, ". ")}|||.\\{faker.Lorem.Word()}.cpp||{faker.Random.UShort()}|-1";
regexBacktrackingData.Add(new JObject(new JProperty("b", value)));
}
var RegexBacktrackingPattern = "(?<i>(.*?))[|].*(?<b>(.*?))[|].*(?<c>(.*?))[|].*(?<d>[1-3])[|].*(?<e>(.*?))[|].*[|].*[|].*(?<f>(.*?))[|].*[|].*(?<g>(.*?))[|].*(?<h>(.*))";
var jpathExpression = $"[?(@.b =~ /{RegexBacktrackingPattern}/c)]";
var exceptionThrow = Xunit.Assert.ThrowsAny<Exception>(() =>
{
var tokens = regexBacktrackingData.SelectTokens(
jpathExpression,
errorWhenNoMatch: false,
singleRegexMatchTimeout: TimeSpan.FromSeconds(2),
globalRegexMatchTimeout: TimeSpan.FromSeconds(.5)).ToArray();
});
Assert.IsTrue(exceptionThrow is OperationCanceledException or RegexMatchTimeoutException);
}
[Test]
public async Task BacktrackingRegexCanBeVerySlowWithoutTimeouts()
{
var RegexBacktrackingPattern = "(?<j>(.*?))[|].*(?<b>(.*?))[|].*(?<c>(.*?))[|].*(?<d>[1-3])[|].*(?<e>(.*?))[|].*[|].*[|].*(?<f>(.*?))[|].*[|].*(?<g>(.*?))[|].*(?<h>(.*))";
var faker = new Faker();
var regexBacktrackingData = new JArray();
for (var i = 0; i < 10; i++)
{
var value = $"{faker.Date.Past()}|1|{faker.Lorem.Words()}|3|{faker.Lorem.Sentences(3, ". ")}|||.\\{faker.Lorem.Word()}.cpp||{faker.Random.UShort()}|-1";
regexBacktrackingData.Add(new JObject(new JProperty("b", value)));
}
var selectTokenTask = Task.Run(() => regexBacktrackingData.SelectTokens($"[?(@.b =~ /{RegexBacktrackingPattern}/)]").ToArray());
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(.3));
using var ctts = new CancellationTokenTaskSource<bool>(cts.Token);
var finishedTask = await Task.WhenAny(selectTokenTask, ctts.Task);
Assert.AreEqual(finishedTask, ctts.Task);
}
#endif
[Test]
public void GreaterThanWithIntegerParameterAndStringValue()
{
@@ -2,7 +2,7 @@
<PropertyGroup>
<TargetFrameworks Condition="'$(TestFrameworks)'==''">net46;net40;net35;net20;net5.0;netcoreapp3.1;netcoreapp2.1</TargetFrameworks>
<TargetFrameworks Condition="'$(TestFrameworks)'!=''">$(TestFrameworks)</TargetFrameworks>
<LangVersion>8.0</LangVersion>
<LangVersion>9.0</LangVersion>
<VersionPrefix>1.0</VersionPrefix>
<Authors>James Newton-King</Authors>
<Company>Newtonsoft</Company>
@@ -76,6 +76,7 @@
<Reference Include="System.ComponentModel.DataAnnotations" />
<Reference Include="System.Web.Extensions" />
<Reference Include="System.Data.DataSetExtensions" />
<PackageReference Include="Bogus" Version="$(BogusPackageVersion)" />
</ItemGroup>
<PropertyGroup Condition="'$(TargetFramework)'=='net40'">
<AssemblyTitle>Json.NET Tests .NET 4.0</AssemblyTitle>
@@ -121,11 +122,14 @@
<PackageReference Include="xunit" Version="$(XunitPackageVersion)" />
<PackageReference Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioPackageVersion)" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkPackageVersion)" />
<PackageReference Include="Bogus" Version="$(BogusPackageVersion)" />
<PackageReference Include="Nito.AsyncEx.Tasks" Version="$(AsyncExPackageVersion)" />
</ItemGroup>
<PropertyGroup Condition="'$(TargetFramework)'=='net5.0'">
<AssemblyTitle>Json.NET Tests .NET Standard 2.0</AssemblyTitle>
<ReferringTargetFrameworkForProjectReferences>.NETStandard,Version=v2.0</ReferringTargetFrameworkForProjectReferences>
<DefineConstants>NETSTANDARD2_0;DNXCORE50;PORTABLE;HAVE_BENCHMARKS;$(AdditionalConstants)</DefineConstants>
<DefineConstants>NETSTANDARD2_0;DNXCORE50;PORTABLE;HAVE_BENCHMARKS;HAVE_REGEX_TIMEOUTS;$(AdditionalConstants)</DefineConstants>
</PropertyGroup>
<ItemGroup Condition="'$(TargetFramework)'=='netcoreapp3.1'">
@@ -142,11 +146,13 @@
<PackageReference Include="xunit" Version="$(XunitPackageVersion)" />
<PackageReference Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioPackageVersion)" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkPackageVersion)" />
<PackageReference Include="Bogus" Version="$(BogusPackageVersion)" />
<PackageReference Include="Nito.AsyncEx.Tasks" Version="$(AsyncExPackageVersion)" />
</ItemGroup>
<PropertyGroup Condition="'$(TargetFramework)'=='netcoreapp3.1'">
<AssemblyTitle>Json.NET Tests .NET Standard 1.3</AssemblyTitle>
<ReferringTargetFrameworkForProjectReferences>.NETStandard,Version=v1.3</ReferringTargetFrameworkForProjectReferences>
<DefineConstants>NETSTANDARD1_3;DNXCORE50;PORTABLE;HAVE_BENCHMARKS;$(AdditionalConstants)</DefineConstants>
<DefineConstants>NETSTANDARD1_3;DNXCORE50;PORTABLE;HAVE_BENCHMARKS;HAVE_REGEX_TIMEOUTS;$(AdditionalConstants)</DefineConstants>
</PropertyGroup>
<ItemGroup Condition="'$(TargetFramework)'=='netcoreapp2.1'">
@@ -162,10 +168,13 @@
<PackageReference Include="xunit" Version="$(XunitPackageVersion)" />
<PackageReference Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioPackageVersion)" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkPackageVersion)" />
<PackageReference Include="Bogus" Version="$(BogusPackageVersion)" />
<PackageReference Include="Nito.AsyncEx.Tasks" Version="$(AsyncExPackageVersion)" />
</ItemGroup>
<PropertyGroup Condition="'$(TargetFramework)'=='netcoreapp2.1'">
<AssemblyTitle>Json.NET Tests .NET Standard 1.0</AssemblyTitle>
<ReferringTargetFrameworkForProjectReferences>.NETStandard,Version=v1.0</ReferringTargetFrameworkForProjectReferences>
<DefineConstants>NETSTANDARD1_0;DNXCORE50;PORTABLE;$(AdditionalConstants)</DefineConstants>
<DefineConstants>NETSTANDARD1_0;DNXCORE50;PORTABLE;HAVE_REGEX_TIMEOUTS;$(AdditionalConstants)</DefineConstants>
</PropertyGroup>
</Project>