Skip to content

Commit 37abbf0

Browse files
chutch3chutch3
committed
feat: acquire back-catalogue issues via covering pack torrents with deduped pack tags
1 parent dcad666 commit 37abbf0

14 files changed

Lines changed: 402 additions & 36 deletions

‎api/API/Acquirers/PackTag.cs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
using System.Security.Cryptography;
2+
using System.Text;
3+
4+
namespace API.Acquirers;
5+
6+
/// <summary>
7+
/// Deterministic download-client tag for a pack release: every chapter that selects the same pack
8+
/// computes the same tag, so the torrent is added to the client once and finalised once (fanning out
9+
/// to all the chapters its files cover). The series key is encoded so completion can find the
10+
/// chapters without re-searching the indexers.
11+
/// </summary>
12+
public static class PackTag
13+
{
14+
private const string Prefix = "pack:";
15+
16+
public static string For(string seriesKey, string downloadUrl)
17+
{
18+
string digest = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(downloadUrl)));
19+
return $"{Prefix}{seriesKey}:{digest[..8].ToLowerInvariant()}";
20+
}
21+
22+
/// <summary>The series key inside a pack tag, or null when <paramref name="tag"/> is not a pack tag.</summary>
23+
public static string? SeriesKeyOf(string tag)
24+
{
25+
if (!tag.StartsWith(Prefix, StringComparison.Ordinal))
26+
return null;
27+
int lastColon = tag.LastIndexOf(':');
28+
return lastColon > Prefix.Length ? tag[Prefix.Length..lastColon] : null;
29+
}
30+
}

‎api/API/Acquirers/TorrentAcquirer.cs‎

Lines changed: 48 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -53,25 +53,44 @@ public async Task<AcquireResult> AcquireAsync(
5353
Categories: settings.IndexerCategories);
5454

5555
IndexerSearchResult[] results = await indexer.Search(query, ct);
56-
if (results.Length == 0)
57-
{
58-
Log.InfoFormat("No torrent releases found for {0} ch.{1}", series.Name, ch.ChapterNumber);
59-
return new AcquireResult.Failed($"no torrent releases found for {series.Name} ch.{ch.ChapterNumber}");
60-
}
61-
6256
IndexerSearchResult? best = selector.SelectBest(results);
57+
string tagKey = chapter.Key;
58+
6359
if (best is null)
6460
{
65-
Log.InfoFormat("No torrent releases passed selection criteria for {0} ch.{1} ({2} candidates)",
66-
series.Name, ch.ChapterNumber, results.Length);
67-
return new AcquireResult.Failed(
68-
$"no release passed selection for {series.Name} ch.{ch.ChapterNumber} ({results.Length} candidates, min seeders {selector.MinSeeders})");
61+
// No usable single-issue release — back catalogues often exist only as packs, so fall
62+
// back to a pack release whose issue range covers this chapter. Packs get a tag derived
63+
// from the release itself, so every chapter of the run converges on one client entry.
64+
IndexerSearchResult[] packs = await SearchCoveringPacks(series, ch, ct);
65+
best = selector.SelectBest(packs);
66+
if (best is null)
67+
{
68+
if (results.Length == 0 && packs.Length == 0)
69+
{
70+
Log.InfoFormat("No torrent releases found for {0} ch.{1} (singles or packs)", series.Name, ch.ChapterNumber);
71+
return new AcquireResult.Failed($"no torrent releases found for {series.Name} ch.{ch.ChapterNumber} (singles or packs)");
72+
}
73+
Log.InfoFormat("No torrent releases passed selection criteria for {0} ch.{1} ({2} candidates)",
74+
series.Name, ch.ChapterNumber, results.Length + packs.Length);
75+
return new AcquireResult.Failed(
76+
$"no release passed selection for {series.Name} ch.{ch.ChapterNumber} ({results.Length + packs.Length} candidates, min seeders {selector.MinSeeders})");
77+
}
78+
79+
tagKey = PackTag.For(series.Key, best.DownloadUrl);
80+
switch (await downloadClient.GetStatus(tagKey, ct))
81+
{
82+
case DownloadStatus.Downloading or DownloadStatus.Completed:
83+
Log.DebugFormat("Pack covering {0} ch.{1} already in the client; deferring to completion.", series.Name, ch.ChapterNumber);
84+
return new AcquireResult.Deferred();
85+
case DownloadStatus.Errored errored:
86+
return new AcquireResult.Failed($"the pack torrent errored in the download client: {errored.Reason}");
87+
}
6988
}
7089

71-
string stagingDir = Path.Combine(settings.StagingDirectory, chapter.Key);
90+
string stagingDir = Path.Combine(settings.StagingDirectory, tagKey.Replace(':', '_'));
7291
Directory.CreateDirectory(stagingDir);
7392

74-
string? tag = await downloadClient.Add(best.DownloadUrl, stagingDir, chapter.Key, ct);
93+
string? tag = await downloadClient.Add(best.DownloadUrl, stagingDir, tagKey, ct);
7594
if (tag is null)
7695
{
7796
Log.WarnFormat("Torrent client refused release {0} for {1} ch.{2}", best.Title, series.Name, ch.ChapterNumber);
@@ -82,6 +101,23 @@ public async Task<AcquireResult> AcquireAsync(
82101
best.Title, series.Name, ch.ChapterNumber);
83102
return new AcquireResult.Deferred();
84103
}
104+
105+
/// <summary>Pack releases of this series whose issue range covers <paramref name="ch"/>.</summary>
106+
private async Task<IndexerSearchResult[]> SearchCoveringPacks(Series series, Chapter ch, CancellationToken ct)
107+
{
108+
// Decimal specials ("60.5") never appear in integer issue ranges.
109+
if (!int.TryParse(ch.ChapterNumber, out int issue))
110+
return [];
111+
112+
IndexerSearchResult[] results = await indexer.Search(
113+
new IndexerQuery(series.Name, null, series.Year?.ToString(), settings.IndexerCategories), ct);
114+
return results.Where(r =>
115+
{
116+
ParsedRelease p = ReleaseTitleParser.Parse(r.Title);
117+
return string.Equals(p.SeriesTitle, series.Name, StringComparison.OrdinalIgnoreCase)
118+
&& p.IssueRange is { } range && issue >= range.Start && issue <= range.End;
119+
}).ToArray();
120+
}
85121
}
86122

87123
/// <summary>Settings the TorrentAcquirer needs at construction time. Populated from KenkuSettings via DI.</summary>

‎api/API/DownloadClients/Interfaces/IDownloadClient.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,17 @@ public interface IDownloadClient
1919

2020
/// <summary>Removes the torrent tagged with <paramref name="tag"/>. No-ops if not found.</summary>
2121
Task Remove(string tag, bool deleteData, CancellationToken ct);
22+
23+
/// <summary>
24+
/// Every download in Kenku's category, with its tag and status. Lets callers find downloads whose
25+
/// tags aren't chapter-keyed (pack torrents) and surface in-flight progress.
26+
/// </summary>
27+
Task<IReadOnlyList<DownloadEntry>> List(CancellationToken ct);
2228
}
2329

30+
/// <summary>One download as the client reports it.</summary>
31+
public record DownloadEntry(string Tag, string Name, DownloadStatus Status, double Progress, int Seeders);
32+
2433
public abstract record DownloadStatus
2534
{
2635
public sealed record Downloading(double Progress) : DownloadStatus;

‎api/API/DownloadClients/QBittorrentClient.cs‎

Lines changed: 40 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -61,21 +61,51 @@ public class QBittorrentClient(HttpClient http, string baseUrl, string username,
6161
string tags = el.TryGetProperty("tags", out var t) ? t.GetString() ?? "" : "";
6262
if (!TagsContain(tags, tag)) continue;
6363

64-
string state = el.TryGetProperty("state", out var s) ? s.GetString() ?? "" : "";
65-
double progress = el.TryGetProperty("progress", out var p) && p.TryGetDouble(out double pv) ? pv : 0.0;
66-
string savePath = el.TryGetProperty("save_path", out var sp) ? sp.GetString() ?? "" : "";
64+
return StatusOf(el);
65+
}
6766

68-
if (state.Equals("error", StringComparison.OrdinalIgnoreCase) ||
69-
state.Equals("missingFiles", StringComparison.OrdinalIgnoreCase))
70-
return new DownloadStatus.Errored(state);
67+
return null;
68+
}
7169

72-
if (progress >= 1.0)
73-
return new DownloadStatus.Completed(savePath);
70+
public async Task<IReadOnlyList<DownloadEntry>> List(CancellationToken ct)
71+
{
72+
if (!await EnsureAuth(ct)) return [];
7473

75-
return new DownloadStatus.Downloading(progress);
74+
using HttpResponseMessage resp = await GetWithCookie("/api/v2/torrents/info?category=kenku", ct);
75+
if (!resp.IsSuccessStatusCode)
76+
return [];
77+
78+
string body = await resp.Content.ReadAsStringAsync(ct);
79+
using JsonDocument doc = JsonDocument.Parse(body);
80+
if (doc.RootElement.ValueKind != JsonValueKind.Array)
81+
return [];
82+
83+
var entries = new List<DownloadEntry>();
84+
foreach (JsonElement el in doc.RootElement.EnumerateArray())
85+
{
86+
string tags = el.TryGetProperty("tags", out var t) ? t.GetString() ?? "" : "";
87+
string tag = tags.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
88+
.FirstOrDefault() ?? "";
89+
string name = el.TryGetProperty("name", out var n) ? n.GetString() ?? "" : "";
90+
int seeders = el.TryGetProperty("num_seeds", out var ns) && ns.TryGetInt32(out int sv) ? sv : 0;
91+
double progress = el.TryGetProperty("progress", out var p) && p.TryGetDouble(out double pv) ? pv : 0.0;
92+
entries.Add(new DownloadEntry(tag, name, StatusOf(el), progress, seeders));
7693
}
94+
return entries;
95+
}
7796

78-
return null;
97+
private static DownloadStatus StatusOf(JsonElement el)
98+
{
99+
string state = el.TryGetProperty("state", out var s) ? s.GetString() ?? "" : "";
100+
double progress = el.TryGetProperty("progress", out var p) && p.TryGetDouble(out double pv) ? pv : 0.0;
101+
string savePath = el.TryGetProperty("save_path", out var sp) ? sp.GetString() ?? "" : "";
102+
103+
if (state.Equals("error", StringComparison.OrdinalIgnoreCase) ||
104+
state.Equals("missingFiles", StringComparison.OrdinalIgnoreCase))
105+
return new DownloadStatus.Errored(state);
106+
if (progress >= 1.0)
107+
return new DownloadStatus.Completed(savePath);
108+
return new DownloadStatus.Downloading(progress);
79109
}
80110

81111
public async Task Remove(string tag, bool deleteData, CancellationToken ct)

‎api/API/Extensions/ApplicationServiceCollectionExtensions.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ public static IServiceCollection AddJobRuntime(this IServiceCollection services,
164164
services.AddSingleton<IJobHandler, PlaceChapterFileHandler>();
165165
services.AddSingleton<IJobHandler, DownloadCoverHandler>();
166166
services.AddSingleton<IJobHandler, FinalizeTorrentHandler>();
167+
services.AddSingleton<IJobHandler, FinalizePackHandler>();
167168
services.AddSingleton<IJobHandler, VerifyDownloadStateHandler>();
168169
services.AddSingleton<IJobHandler, MoveDataHandler>();
169170
services.AddSingleton<IJobHandler, RefreshDiscoveryFeedHandler>();

‎api/API/JobRuntime/Handlers/FinalizePackHandler.cs‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
using API.DownloadClients.Interfaces;
2+
using API.JobRuntime.Interfaces;
3+
using System.Text.Json;
4+
using API.Schema.ActionsContext;
5+
using API.Schema.JobsContext;
6+
using API.Schema.SeriesContext;
7+
using API.Services;
8+
using Microsoft.Extensions.DependencyInjection;
9+
10+
namespace API.JobRuntime.Handlers;
11+
12+
/// <summary>Payload for <see cref="FinalizePackHandler"/>: the pack's client tag, its series, and the save path.</summary>
13+
public record FinalizePackPayload(string Tag, string SeriesKey, string SavePath);
14+
15+
/// <summary>
16+
/// Finalises one completed pack torrent (tag <c>pack:{seriesKey}:{hash}</c>) by fanning its archives
17+
/// out to every chapter of the series they cover — the pack counterpart of
18+
/// <see cref="FinalizeTorrentHandler"/>.
19+
/// </summary>
20+
public class FinalizePackHandler(IServiceScopeFactory scopeFactory) : IJobHandler
21+
{
22+
public const string Type = "FinalizePack";
23+
public string JobType => Type;
24+
25+
public static string PayloadFor(string tag, string seriesKey, string savePath) =>
26+
JsonSerializer.Serialize(new FinalizePackPayload(tag, seriesKey, savePath));
27+
28+
public async Task ExecuteAsync(Job job, CancellationToken ct)
29+
{
30+
FinalizePackPayload payload = JsonSerializer.Deserialize<FinalizePackPayload>(job.Payload)
31+
?? throw new InvalidOperationException($"Invalid {Type} payload: {job.Payload}");
32+
33+
using IServiceScope scope = scopeFactory.CreateScope();
34+
var provider = scope.ServiceProvider;
35+
await provider.GetRequiredService<TorrentFinalizationService>().FinalizePackAsync(
36+
provider.GetRequiredService<SeriesContext>(),
37+
provider.GetRequiredService<ActionsContext>(),
38+
provider.GetRequiredService<IDownloadClient>(),
39+
provider.GetRequiredService<KenkuSettings>(),
40+
payload.Tag, payload.SeriesKey, payload.SavePath, ct);
41+
}
42+
}

‎api/API/JobRuntime/Reconcilers/TorrentCompletionReconciler.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,19 @@ await store.EnqueueAsync(new Job(FinalizeTorrentHandler.Type,
6161
resourceKey: chId.Obj.ParentMangaId, dedupKey: DedupKey(chId.Key)), ct);
6262
enqueued++;
6363
}
64+
65+
// Pack torrents aren't chapter-keyed, so they're discovered from the client itself: any
66+
// completed download carrying a pack tag gets a FinalizePack job (deduped per tag).
67+
foreach (DownloadEntry entry in await downloadClient.List(ct))
68+
{
69+
if (Acquirers.PackTag.SeriesKeyOf(entry.Tag) is not { } seriesKey) continue;
70+
if (entry.Status is not DownloadStatus.Completed packCompleted) continue;
71+
72+
await store.EnqueueAsync(new Job(FinalizePackHandler.Type,
73+
FinalizePackHandler.PayloadFor(entry.Tag, seriesKey, packCompleted.SavePath), now,
74+
resourceKey: seriesKey, dedupKey: $"finalize-pack:{entry.Tag}"), ct);
75+
enqueued++;
76+
}
6477
return enqueued;
6578
}
6679
}

‎api/API/Services/TorrentFinalizationService.cs‎

Lines changed: 69 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -67,17 +67,7 @@ public async Task FinalizeAsync(SeriesContext seriesContext, ActionsContext acti
6767
List<Chapter> seriesChapters = await seriesContext.Chapters
6868
.Where(c => c.ParentMangaId == chapter.ParentMangaId && !c.Downloaded)
6969
.ToListAsync(ct);
70-
placed = 0;
71-
foreach (string archive in archives)
72-
{
73-
ParsedRelease parsed = ReleaseTitleParser.Parse(Path.GetFileNameWithoutExtension(archive));
74-
// Same-series check keeps a pack's extras/specials from claiming a main-run issue number.
75-
if (parsed.IssueNumber is null) continue;
76-
if (!string.Equals(parsed.SeriesTitle, chapter.ParentManga.Name, StringComparison.OrdinalIgnoreCase)) continue;
77-
Chapter? target = seriesChapters.FirstOrDefault(c => c.ChapterNumber == parsed.IssueNumber && !c.Downloaded);
78-
if (target is null) continue;
79-
if (Place(archive, target, settings, actionsContext)) placed++;
80-
}
70+
placed = FanOut(archives, chapter.ParentManga.Name, seriesChapters, settings, actionsContext);
8171
}
8272

8373
if (placed == 0)
@@ -100,6 +90,74 @@ public async Task FinalizeAsync(SeriesContext seriesContext, ActionsContext acti
10090
Log.InfoFormat("Finalised torrent for {0}: placed {1} chapter file(s).", chapter.ParentManga.Name, placed);
10191
}
10292

93+
/// <summary>
94+
/// Finalises a completed pack torrent (tag <c>pack:{seriesKey}:{hash}</c>): fans its archives out
95+
/// to every undownloaded chapter of the series they parse to, then removes the torrent. The pack
96+
/// is removed even when nothing could be placed — its contents won't change, so leaving it would
97+
/// only make the completion reconciler re-enqueue a hopeless finalise forever.
98+
/// </summary>
99+
public async Task FinalizePackAsync(SeriesContext seriesContext, ActionsContext actionsContext,
100+
IDownloadClient downloadClient, KenkuSettings settings, string tag, string seriesKey, string savePath,
101+
CancellationToken ct)
102+
{
103+
Series? series = await seriesContext.Series
104+
.Include(s => s.Library)
105+
.FirstOrDefaultAsync(s => s.Key == seriesKey, ct);
106+
if (series is null)
107+
{
108+
Log.ErrorFormat("Could not finalise pack {0}: series {1} not found.", tag, seriesKey);
109+
return;
110+
}
111+
112+
if (!Directory.Exists(savePath))
113+
{
114+
Log.ErrorFormat("Pack {0} reports completion at {1} but the directory does not exist.", tag, savePath);
115+
return;
116+
}
117+
118+
List<Chapter> chapters = await seriesContext.Chapters
119+
.Where(c => c.ParentMangaId == seriesKey && !c.Downloaded)
120+
.ToListAsync(ct);
121+
122+
string[] archives = Directory.EnumerateFiles(savePath, "*.cbz", SearchOption.AllDirectories).ToArray();
123+
int placed = FanOut(archives, series.Name, chapters, settings, actionsContext);
124+
125+
if (placed > 0)
126+
{
127+
var syncs = await Task.WhenAll(
128+
seriesContext.Sync(ct, typeof(TorrentFinalizationService), nameof(FinalizePackAsync)),
129+
actionsContext.Sync(ct, typeof(TorrentFinalizationService), nameof(FinalizePackAsync)));
130+
foreach (var s in syncs)
131+
if (!s.success) Log.ErrorFormat("Sync failed during pack finalise: {0}", s.exceptionMessage);
132+
}
133+
else
134+
{
135+
Log.ErrorFormat("Pack {0} at {1} contained {2} archive(s) but none matched an undownloaded chapter of {3}.",
136+
tag, savePath, archives.Length, series.Name);
137+
}
138+
139+
await downloadClient.Remove(tag, deleteData: false, ct);
140+
Log.InfoFormat("Finalised pack {0} for {1}: placed {2} chapter file(s).", tag, series.Name, placed);
141+
}
142+
143+
/// <summary>Fans pack archives out to the chapters their filenames parse to. Returns how many were placed.</summary>
144+
private static int FanOut(string[] archives, string seriesName, List<Chapter> chapters,
145+
KenkuSettings settings, ActionsContext actionsContext)
146+
{
147+
int placed = 0;
148+
foreach (string archive in archives)
149+
{
150+
ParsedRelease parsed = ReleaseTitleParser.Parse(Path.GetFileNameWithoutExtension(archive));
151+
// Same-series check keeps a pack's extras/specials from claiming a main-run issue number.
152+
if (parsed.IssueNumber is null) continue;
153+
if (!string.Equals(parsed.SeriesTitle, seriesName, StringComparison.OrdinalIgnoreCase)) continue;
154+
Chapter? target = chapters.FirstOrDefault(c => c.ChapterNumber == parsed.IssueNumber && !c.Downloaded);
155+
if (target is null) continue;
156+
if (Place(archive, target, settings, actionsContext)) placed++;
157+
}
158+
return placed;
159+
}
160+
103161
/// <summary>Moves one archive into <paramref name="chapter"/>'s publication path and marks it downloaded.</summary>
104162
private static bool Place(string archive, Chapter chapter, KenkuSettings settings, ActionsContext actionsContext)
105163
{

‎api/Tests/Integration/SeriesDeletionEndToEndTests.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ public Task Remove(string tag, bool deleteData, CancellationToken ct)
2828
Removed.Add(tag);
2929
return Task.CompletedTask;
3030
}
31+
32+
public Task<IReadOnlyList<DownloadEntry>> List(CancellationToken ct) =>
33+
Task.FromResult<IReadOnlyList<DownloadEntry>>([]);
3134
}
3235

3336
private readonly PostgresFixture _postgres = new();

‎api/Tests/Integration/TorrentDownloadEndToEndTests.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,10 @@ public Task Remove(string tag, bool deleteData, CancellationToken ct)
4949
Status = null;
5050
return Task.CompletedTask;
5151
}
52+
53+
public Task<IReadOnlyList<DownloadEntry>> List(CancellationToken ct) =>
54+
Task.FromResult<IReadOnlyList<DownloadEntry>>(
55+
Added.Select(a => new DownloadEntry(a.Tag, a.Url, Status ?? new DownloadStatus.Downloading(0), 0, 0)).ToList());
5256
}
5357

5458
private readonly string _tempRoot = Path.Combine(Path.GetTempPath(), "kenku-tor-e2e-" + Guid.NewGuid().ToString("N"));

0 commit comments

Comments
 (0)