Skip to content

Commit 3dd0ccc

Browse files
CXuesongCXuesong
committed
#118 Materialize continuation params into strings for easier comparison.
1 parent 9658c3e commit 3dd0ccc

File tree

6 files changed

+92
-24
lines changed

6 files changed

+92
-24
lines changed

‎UnitTestProject1/Endpoints.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ public static class Endpoints
2525
/// </summary>
2626
public const string WikipediaLzh = "https://zh-classical.wikipedia.org/w/api.php";
2727

28+
public const string WikimediaCommons = "https://commons.wikimedia.org/w/api.php";
29+
2830
public const string WikimediaCommonsBeta = "https://commons.wikimedia.beta.wmflabs.org/w/api.php";
2931

3032
public const string Wikidata = "https://www.wikidata.org/w/api.php";

‎UnitTestProject1/Tests/ValidationTests.cs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
using WikiClientLibrary.Generators;
22
using WikiClientLibrary.Infrastructures;
3+
using WikiClientLibrary.Pages;
4+
using WikiClientLibrary.Pages.Queries;
5+
using WikiClientLibrary.Pages.Queries.Properties;
36
using WikiClientLibrary.Tests.UnitTestProject1.Fixtures;
47
using Xunit;
58
using Xunit.Abstractions;
@@ -86,4 +89,29 @@ public async Task Issue89()
8689
Assert.False(site.MagicWords.ContainsAlias("__non_existing_magic__"));
8790
}
8891

92+
/// <summary>
93+
/// [B]Infinite Continuation on Wiki Commons Site
94+
/// </summary>
95+
[Fact]
96+
public async Task Issue118()
97+
{
98+
var commonsSite = await GetWikiSiteAsync(Endpoints.WikimediaCommons);
99+
var page = new WikiPage(commonsSite, "File:Harku mõisa park.JPG");
100+
101+
// Just in case RefreshAsync gets into an infinite loop.
102+
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10));
103+
104+
await Assert.ThrowsAsync<UnexpectedContinuationLoopException>(() => page.RefreshAsync(new WikiPageQueryProvider
105+
{
106+
Properties =
107+
{
108+
new FileInfoPropertyProvider { QueryExtMetadata = true },
109+
new PageImagesPropertyProvider { QueryOriginalImage = true },
110+
new LanguageLinksPropertyProvider(LanguageLinkProperties.Url),
111+
new PageInfoPropertyProvider(),
112+
new PagePropertiesPropertyProvider(),
113+
},
114+
}, cts.Token));
115+
}
116+
89117
}

‎WikiClientLibrary/Exceptions.cs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,3 +347,29 @@ public UnexpectedDataException(string message, Exception inner)
347347
}
348348

349349
}
350+
351+
/// <summary>
352+
/// Raises when the received MediaWiki continuation parameter is exactly the same
353+
/// as the parameter used to send the MediaWiki API query. This means we are not
354+
/// moving forward in the paginated results.
355+
/// </summary>
356+
public class UnexpectedContinuationLoopException : UnexpectedDataException
357+
{
358+
359+
public UnexpectedContinuationLoopException()
360+
: this(Prompts.ExceptionUnexpectedContinuationLoop)
361+
{
362+
}
363+
364+
public UnexpectedContinuationLoopException(string message)
365+
: base(message)
366+
{
367+
}
368+
369+
public UnexpectedContinuationLoopException(string message, Exception inner)
370+
: base(message, inner)
371+
{
372+
}
373+
374+
}
375+

‎WikiClientLibrary/Generators/Primitive/WikiList.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ public async IAsyncEnumerable<T> EnumItemsAsync([EnumeratorCancellation] Cancell
142142
foreach (var p in EnumListParameters())
143143
baseQueryParams.Add(p.Key, p.Value);
144144
cancellationToken.ThrowIfCancellationRequested();
145-
var continuationParams = new Dictionary<string, object?>();
145+
var continuationParams = new Dictionary<string, string>();
146146
using var scope = Site.BeginActionScope(this);
147147
// query parameters for this batch. The content/ref will be modified below.
148148
var queryParams = new Dictionary<string, object?>();

‎WikiClientLibrary/RequestHelper.cs‎

Lines changed: 19 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -29,43 +29,39 @@ internal static class RequestHelper
2929
?? jresult["query-continue"]?.AsObject().FirstOrDefault().Value);
3030
}
3131

32-
public static int ParseContinuationParameters(JsonNode jresult, IDictionary<string, object?> queryParams,
33-
IDictionary<string, object?>? continuationParams)
32+
public static int ParseContinuationParameters(JsonNode jresult, IDictionary<string, object> queryParams,
33+
IDictionary<string, string>? continuationParams)
3434
{
3535
var continuation = FindQueryContinuationParameterRoot(jresult);
3636
// No more results.
3737
if (continuation == null || continuation.Count == 0)
3838
return CONTINUATION_DONE;
39+
3940
var anyNewValue = false;
4041
continuationParams?.Clear();
4142
foreach (var p in continuation)
4243
{
4344
if (p.Value == null) continue;
4445

45-
var parsed = p.Value switch
46+
var parsed = p.Value.GetValueKind() switch
4647
{
47-
JsonValue value => value.GetValue<object>(),
48+
// Trivial: unwrap strings.
49+
JsonValueKind.String => p.Value.GetValue<string>(),
50+
// Retrieve JSON representation so we won't need to consider which int/float type to use.
51+
JsonValueKind.Number => p.Value.ToJsonString(),
52+
// Ignore nulls.
53+
JsonValueKind.Null or JsonValueKind.Undefined => null,
54+
// We cannot help -- this is the best we can do.
4855
_ => p.Value.ToJsonString(),
4956
};
50-
if (!queryParams.TryGetValue(p.Key, out var existingValue) || !ValueEquals(existingValue, parsed))
57+
if (parsed == null) continue;
58+
59+
if (!queryParams.TryGetValue(p.Key, out var existing) || !Equals(existing, parsed))
5160
anyNewValue = true;
61+
5262
continuationParams?.Add(new(p.Key, parsed));
5363
}
5464
return anyNewValue ? CONTINUATION_AVAILABLE : CONTINUATION_LOOP;
55-
56-
static bool ValueEquals(object? existing, object? incoming)
57-
{
58-
if (Equals(existing, incoming)) return true;
59-
if (existing is DateTime dt && incoming is string s)
60-
{
61-
if (MediaWikiHelper.TryParseDateTime(s, out var dt2))
62-
{
63-
// We have called ToUniversalTime() in ToWikiStringValuePairs.
64-
return dt.ToUniversalTime() == dt2.ToUniversalTime();
65-
}
66-
}
67-
return false;
68-
}
6965
}
7066

7167
public static JsonNode? FindQueryResponseItemsRoot(JsonNode jresult, string actionName)
@@ -115,7 +111,7 @@ public static async IAsyncEnumerable<JsonObject> QueryWithContinuation(WikiSite
115111
// Defensive copy.
116112
var baseQueryParams = new Dictionary<string, object?>(parameters);
117113
Debug.Assert("query".Equals(baseQueryParams["action"]));
118-
var continuationParams = new Dictionary<string, object?>();
114+
var continuationParams = new Dictionary<string, string>();
119115
while (true)
120116
{
121117
var queryParams = new Dictionary<string, object?>(baseQueryParams);
@@ -158,7 +154,7 @@ public static async IAsyncEnumerable<JsonObject> QueryWithContinuation(WikiSite
158154
// Continue the loop and fetch for the next page of query.
159155
break;
160156
case CONTINUATION_LOOP:
161-
throw new UnexpectedDataException();
157+
throw new UnexpectedContinuationLoopException();
162158
}
163159
}
164160
}
@@ -207,13 +203,13 @@ public static async Task RefreshPagesAsync(IEnumerable<WikiPage> pages, IWikiPag
207203
if (continuationStatus != CONTINUATION_DONE)
208204
{
209205
var queryParams1 = new Dictionary<string, object?>();
210-
var continuationParams = new Dictionary<string, object?>();
206+
var continuationParams = new Dictionary<string, string>();
211207
var jobj1 = jobj;
212208
ParseContinuationParameters(jobj1, queryParams1, continuationParams);
213209
while (continuationStatus != CONTINUATION_DONE)
214210
{
215211
if (continuationStatus == CONTINUATION_LOOP)
216-
throw new UnexpectedDataException(Prompts.ExceptionUnexpectedContinuationLoop);
212+
throw new UnexpectedContinuationLoopException();
217213
Debug.Assert(continuationStatus == CONTINUATION_AVAILABLE);
218214
site.Logger.LogDebug("Detected query continuation. PartitionCount={PartitionCount}.", partition.Count);
219215
queryParams1.Clear();

‎WikiClientLibrary/Utility.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,22 @@ public static void MergeFrom<TKey, TValue>(this IDictionary<TKey, TValue> dict,
4343
dict[item.Key] = item.Value;
4444
}
4545

46+
public static void MergeFrom<TKey, TValue>(this IDictionary<TKey, object?> dict, IEnumerable<KeyValuePair<TKey, TValue>> items)
47+
{
48+
foreach (var item in items)
49+
dict[item.Key] = item.Value;
50+
}
51+
52+
/*
53+
Causes ambiguity with MergeFrom<TKey, TValue>
54+
public static void MergeFrom<TKey, TValue, TValue2>(this IDictionary<TKey,TValue> dict, IEnumerable<KeyValuePair<TKey, TValue2>> items)
55+
where TValue2 : TValue
56+
{
57+
foreach (var item in items)
58+
dict[item.Key] = item.Value;
59+
}
60+
*/
61+
4662
public static string? ToWikiQueryValue(object? value)
4763
{
4864
return value switch

0 commit comments

Comments
 (0)