Skip to content

Commit 0c264c6

Browse files
ahsonkhanahsonkhan
authored
Reduce inner and outer loop test time for S.T.Json (dotnet#35776)
* Reduce inner and outer loop test time for S.T.Json * Remove redundant sequence slicing, only compare the SequencePositions.
1 parent ae14268 commit 0c264c6

File tree

5 files changed

+264
-144
lines changed

5 files changed

+264
-144
lines changed

‎src/System.Text.Json/tests/BitStackTests.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ public static void BitStackPushPop(int bitLength)
8484
}
8585

8686
[Theory]
87+
[OuterLoop]
8788
[InlineData(3_200_000)]
8889
[InlineData(int.MaxValue / 32 + 1)] // 67_108_864
8990
public static void BitStackPushPopLarge(int bitLength)

‎src/System.Text.Json/tests/JsonTestHelper.cs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
using System.Globalization;
88
using System.IO;
99
using Newtonsoft.Json;
10+
using Newtonsoft.Json.Linq;
1011
using Xunit;
1112

1213
namespace System.Text.Json.Tests
@@ -157,6 +158,24 @@ public static ReadOnlySequence<byte> GetSequence(byte[] _dataUtf8, int segmentSi
157158
return BufferFactory.Create(buffers);
158159
}
159160

161+
public static List<ReadOnlySequence<byte>> GetSequences(ReadOnlyMemory<byte> dataMemory)
162+
{
163+
var sequences = new List<ReadOnlySequence<byte>>
164+
{
165+
new ReadOnlySequence<byte>(dataMemory)
166+
};
167+
168+
for (int i = 0; i < dataMemory.Length; i++)
169+
{
170+
var firstSegment = new BufferSegment<byte>(dataMemory.Slice(0, i));
171+
ReadOnlyMemory<byte> secondMem = dataMemory.Slice(i);
172+
BufferSegment<byte> secondSegment = firstSegment.Append(secondMem);
173+
var sequence = new ReadOnlySequence<byte>(firstSegment, 0, secondSegment, secondMem.Length);
174+
sequences.Add(sequence);
175+
}
176+
return sequences;
177+
}
178+
160179
internal static ReadOnlySequence<byte> SegmentInto(ReadOnlyMemory<byte> data, int segmentCount)
161180
{
162181
if (segmentCount < 2)
@@ -543,5 +562,20 @@ public static decimal NextDecimal(Random random, double minValue, double maxValu
543562
double value = random.NextDouble() * (maxValue - minValue) + minValue;
544563
return (decimal)value;
545564
}
565+
566+
public static string GetCompactString(string jsonString)
567+
{
568+
using (JsonTextReader jsonReader = new JsonTextReader(new StringReader(jsonString)))
569+
{
570+
jsonReader.FloatParseHandling = FloatParseHandling.Decimal;
571+
JToken jtoken = JToken.ReadFrom(jsonReader);
572+
var stringWriter = new StringWriter();
573+
using (JsonTextWriter jsonWriter = new JsonTextWriter(stringWriter))
574+
{
575+
jtoken.WriteTo(jsonWriter);
576+
return stringWriter.ToString();
577+
}
578+
}
579+
}
546580
}
547581
}

‎src/System.Text.Json/tests/Utf8JsonReaderTests.MultiSegment.cs‎

Lines changed: 83 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,6 @@
66
using System.Collections.Generic;
77
using System.Diagnostics;
88
using System.IO;
9-
using Newtonsoft.Json;
10-
using Newtonsoft.Json.Linq;
119
using Xunit;
1210

1311
namespace System.Text.Json.Tests
@@ -16,23 +14,42 @@ public static partial class Utf8JsonReaderTests
1614
{
1715
// TestCaseType is only used to give the json strings a descriptive name.
1816
[Theory]
19-
[MemberData(nameof(TestCases))]
17+
// Skipping large JSON since slicing them (O(n^2)) is too slow.
18+
[MemberData(nameof(SmallTestCases))]
2019
public static void TestJsonReaderUtf8SegmentSizeOne(bool compactData, TestCaseType type, string jsonString)
20+
{
21+
ReadPartialSegmentSizeOne(compactData, type, jsonString);
22+
}
23+
24+
// TestCaseType is only used to give the json strings a descriptive name.
25+
[Theory]
26+
[MemberData(nameof(LargeTestCases))]
27+
public static void TestJsonReaderLargeUtf8SegmentSizeOne(bool compactData, TestCaseType type, string jsonString)
28+
{
29+
ReadFullySegmentSizeOne(compactData, type, jsonString);
30+
}
31+
32+
// TestCaseType is only used to give the json strings a descriptive name.
33+
[Theory]
34+
[OuterLoop]
35+
[MemberData(nameof(LargeTestCases))]
36+
public static void TestJsonReaderLargestUtf8SegmentSizeOne(bool compactData, TestCaseType type, string jsonString)
37+
{
38+
// Skipping really large JSON since slicing them (O(n^2)) is too slow.
39+
if (type == TestCaseType.Json40KB || type == TestCaseType.Json400KB || type == TestCaseType.ProjectLockJson)
40+
{
41+
return;
42+
}
43+
44+
ReadPartialSegmentSizeOne(compactData, type, jsonString);
45+
}
46+
47+
private static void ReadPartialSegmentSizeOne(bool compactData, TestCaseType type, string jsonString)
2148
{
2249
// Remove all formatting/indendation
2350
if (compactData)
2451
{
25-
using (JsonTextReader jsonReader = new JsonTextReader(new StringReader(jsonString)))
26-
{
27-
jsonReader.FloatParseHandling = FloatParseHandling.Decimal;
28-
JToken jtoken = JToken.ReadFrom(jsonReader);
29-
var stringWriter = new StringWriter();
30-
using (JsonTextWriter jsonWriter = new JsonTextWriter(stringWriter))
31-
{
32-
jtoken.WriteTo(jsonWriter);
33-
jsonString = stringWriter.ToString();
34-
}
35-
}
52+
jsonString = JsonTestHelper.GetCompactString(jsonString);
3653
}
3754

3855
byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
@@ -43,16 +60,6 @@ public static void TestJsonReaderUtf8SegmentSizeOne(bool compactData, TestCaseTy
4360

4461
ReadOnlySequence<byte> sequence = JsonTestHelper.GetSequence(dataUtf8, 1);
4562

46-
// Skipping really large JSON since slicing them (O(n^2)) is too slow.
47-
if (type == TestCaseType.Json40KB || type == TestCaseType.Json400KB || type == TestCaseType.ProjectLockJson)
48-
{
49-
var utf8JsonReader = new Utf8JsonReader(sequence, isFinalBlock: true, default);
50-
byte[] resultSequence = JsonTestHelper.ReaderLoop(dataUtf8.Length, out int length, ref utf8JsonReader);
51-
string actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length);
52-
Assert.Equal(expectedStr, actualStrSequence);
53-
return;
54-
}
55-
5663
for (int j = 0; j < dataUtf8.Length; j++)
5764
{
5865
var utf8JsonReader = new Utf8JsonReader(sequence.Slice(0, j), isFinalBlock: false, default);
@@ -71,63 +78,73 @@ public static void TestJsonReaderUtf8SegmentSizeOne(bool compactData, TestCaseTy
7178
}
7279
}
7380

74-
[Theory]
75-
[MemberData(nameof(TestCases))]
76-
public static void TestPartialJsonReaderMultiSegment(bool compactData, TestCaseType type, string jsonString)
81+
private static void ReadFullySegmentSizeOne(bool compactData, TestCaseType type, string jsonString)
7782
{
78-
// Skipping really large JSON since slicing them (O(n^2)) is too slow.
79-
if (type == TestCaseType.Json40KB || type == TestCaseType.Json400KB || type == TestCaseType.ProjectLockJson
80-
|| type == TestCaseType.DeepTree || type == TestCaseType.BroadTree || type == TestCaseType.LotsOfNumbers
81-
|| type == TestCaseType.LotsOfStrings || type == TestCaseType.Json4KB)
83+
// Remove all formatting/indendation
84+
if (compactData)
8285
{
83-
return;
86+
jsonString = JsonTestHelper.GetCompactString(jsonString);
8487
}
8588

89+
byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
90+
91+
Stream stream = new MemoryStream(dataUtf8);
92+
TextReader reader = new StreamReader(stream, Encoding.UTF8, false, 1024, true);
93+
string expectedStr = JsonTestHelper.NewtonsoftReturnStringHelper(reader);
94+
95+
ReadOnlySequence<byte> sequence = JsonTestHelper.GetSequence(dataUtf8, 1);
96+
97+
var utf8JsonReader = new Utf8JsonReader(sequence, isFinalBlock: true, default);
98+
byte[] resultSequence = JsonTestHelper.ReaderLoop(dataUtf8.Length, out int length, ref utf8JsonReader);
99+
string actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length);
100+
Assert.Equal(expectedStr, actualStrSequence);
101+
}
102+
103+
[Theory]
104+
[MemberData(nameof(SmallTestCases))]
105+
public static void TestPartialJsonReaderMultiSegment(bool compactData, TestCaseType type, string jsonString)
106+
{
86107
// Remove all formatting/indendation
87108
if (compactData)
88109
{
89-
using (JsonTextReader jsonReader = new JsonTextReader(new StringReader(jsonString)))
90-
{
91-
jsonReader.FloatParseHandling = FloatParseHandling.Decimal;
92-
JToken jtoken = JToken.ReadFrom(jsonReader);
93-
var stringWriter = new StringWriter();
94-
using (JsonTextWriter jsonWriter = new JsonTextWriter(stringWriter))
95-
{
96-
jtoken.WriteTo(jsonWriter);
97-
jsonString = stringWriter.ToString();
98-
}
99-
}
110+
jsonString = JsonTestHelper.GetCompactString(jsonString);
100111
}
101112

102113
byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
103114
ReadOnlyMemory<byte> dataMemory = dataUtf8;
104115

105-
var sequences = new List<ReadOnlySequence<byte>>
106-
{
107-
new ReadOnlySequence<byte>(dataMemory)
108-
};
109-
110-
for (int i = 0; i < dataUtf8.Length; i++)
111-
{
112-
var firstSegment = new BufferSegment<byte>(dataMemory.Slice(0, i));
113-
ReadOnlyMemory<byte> secondMem = dataMemory.Slice(i);
114-
BufferSegment<byte> secondSegment = firstSegment.Append(secondMem);
115-
var sequence = new ReadOnlySequence<byte>(firstSegment, 0, secondSegment, secondMem.Length);
116-
sequences.Add(sequence);
117-
}
116+
List<ReadOnlySequence<byte>> sequences = JsonTestHelper.GetSequences(dataMemory);
118117

119118
for (int i = 0; i < sequences.Count; i++)
120119
{
121-
var json = new Utf8JsonReader(sequences[i], isFinalBlock: true, default);
120+
ReadOnlySequence<byte> sequence = sequences[i];
121+
var json = new Utf8JsonReader(sequence, isFinalBlock: true, default);
122122
while (json.Read())
123123
;
124-
Assert.Equal(sequences[i].Length, json.BytesConsumed);
125-
Assert.Equal(sequences[i].Length, json.CurrentState.BytesConsumed);
124+
Assert.Equal(sequence.Length, json.BytesConsumed);
125+
Assert.Equal(sequence.Length, json.CurrentState.BytesConsumed);
126+
127+
Assert.True(sequence.Slice(json.Position).IsEmpty);
128+
Assert.True(sequence.Slice(json.CurrentState.Position).IsEmpty);
129+
}
130+
}
126131

127-
Assert.True(sequences[i].Slice(json.Position).IsEmpty);
128-
Assert.True(sequences[i].Slice(json.CurrentState.Position).IsEmpty);
132+
[Theory]
133+
[OuterLoop]
134+
[MemberData(nameof(SmallTestCases))]
135+
public static void TestPartialJsonReaderSlicesMultiSegment(bool compactData, TestCaseType type, string jsonString)
136+
{
137+
// Remove all formatting/indendation
138+
if (compactData)
139+
{
140+
jsonString = JsonTestHelper.GetCompactString(jsonString);
129141
}
130142

143+
byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
144+
ReadOnlyMemory<byte> dataMemory = dataUtf8;
145+
146+
List<ReadOnlySequence<byte>> sequences = JsonTestHelper.GetSequences(dataMemory);
147+
131148
for (int i = 0; i < sequences.Count; i++)
132149
{
133150
ReadOnlySequence<byte> sequence = sequences[i];
@@ -141,8 +158,8 @@ public static void TestPartialJsonReaderMultiSegment(bool compactData, TestCaseT
141158
JsonReaderState jsonState = json.CurrentState;
142159
byte[] consumedArray = sequence.Slice(0, consumed).ToArray();
143160
Assert.Equal(consumedArray, sequence.Slice(0, json.Position).ToArray());
144-
Assert.Equal(consumedArray, sequence.Slice(0, jsonState.Position).ToArray());
145-
json = new Utf8JsonReader(sequence.Slice(consumed), isFinalBlock: true, json.CurrentState);
161+
Assert.True(json.Position.Equals(jsonState.Position));
162+
json = new Utf8JsonReader(sequence.Slice(consumed), isFinalBlock: true, jsonState);
146163
while (json.Read())
147164
;
148165
Assert.Equal(dataUtf8.Length - consumed, json.BytesConsumed);
@@ -338,7 +355,7 @@ private static void TestReadingJsonWithComments(byte[] inputData, ReadOnlySequen
338355

339356
Assert.Equal(expectedWithComments, builder.ToString());
340357
Assert.Equal(inputData, sequence.Slice(0, json.Position).ToArray());
341-
Assert.Equal(inputData, sequence.Slice(0, json.CurrentState.Position).ToArray());
358+
Assert.True(json.Position.Equals(json.CurrentState.Position));
342359

343360
state = new JsonReaderState(options: new JsonReaderOptions { CommentHandling = JsonCommentHandling.Skip });
344361
json = new Utf8JsonReader(sequence, isFinalBlock: true, state);
@@ -362,7 +379,7 @@ private static void TestReadingJsonWithComments(byte[] inputData, ReadOnlySequen
362379

363380
Assert.Equal(expectedWithoutComments, builder.ToString());
364381
Assert.Equal(inputData, sequence.Slice(0, json.Position).ToArray());
365-
Assert.Equal(inputData, sequence.Slice(0, json.CurrentState.Position).ToArray());
382+
Assert.True(json.Position.Equals(json.CurrentState.Position));
366383
}
367384

368385
[Theory]
@@ -408,7 +425,7 @@ private static void TestReadingSingleValueJson(byte[] inputData, ReadOnlySequenc
408425

409426
Assert.Equal(json.BytesConsumed, json.CurrentState.BytesConsumed);
410427
Assert.Equal(inputData, sequence.Slice(0, json.Position).ToArray());
411-
Assert.Equal(inputData, sequence.Slice(0, json.CurrentState.Position).ToArray());
428+
Assert.True(json.Position.Equals(json.CurrentState.Position));
412429
}
413430
}
414431

0 commit comments

Comments
 (0)