Skip to content

Commit 5c84da5

Browse files
justcoding121justcoding121
committed
Fix harvest bugs
1 parent 2e64b4a commit 5c84da5

File tree

4 files changed

+252
-27
lines changed

4 files changed

+252
-27
lines changed

‎.tools/Bible.Alarm.Audio.Links.Harvestor/Harvestors/Bible/JwBibleHarvester.cs‎

Lines changed: 110 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,29 @@ Task Harvest_Bible_Links(Dictionary<string, string> biblePublicationCodeToNameMa
2929
var jsonString = await DownloadUtility.GetAsync(harvestLink);
3030
using var doc = JsonDocument.Parse(jsonString);
3131
var root = doc.RootElement;
32-
var languages = root.GetProperty("languages");
32+
33+
if (!root.TryGetProperty("languages", out var languages))
34+
{
35+
Console.WriteLine($"Warning: 'languages' property not found in response for publication {publicationCode}. Skipping.");
36+
continue;
37+
}
3338

3439
foreach (var item in languages.EnumerateObject())
3540
{
3641
var languageCode = item.Name;
37-
var language = item.Value.GetProperty("name").GetString();
42+
43+
if (!item.Value.TryGetProperty("name", out var nameElement))
44+
{
45+
Console.WriteLine($"Warning: 'name' property not found for language {languageCode}. Skipping.");
46+
continue;
47+
}
48+
49+
var language = nameElement.GetString();
50+
if (string.IsNullOrEmpty(language))
51+
{
52+
Console.WriteLine($"Warning: Language name is null or empty for language {languageCode}. Skipping.");
53+
continue;
54+
}
3855

3956
languageCodeToNameMappings.TryAdd(languageCode, language);
4057

@@ -73,12 +90,20 @@ private static async Task<bool> harvestBibleLinks(string languageCode, string pu
7390
try
7491
{
7592
doc = JsonDocument.Parse(jsonString);
76-
files = doc.RootElement.GetProperty("files");
93+
if (!doc.RootElement.TryGetProperty("files", out files))
94+
{
95+
Console.WriteLine($"Warning: 'files' property not found in response for book {bookNumber}, language {languageCode}. Skipping.");
96+
doc?.Dispose();
97+
bookNumber++;
98+
harvestLink = $"{AppConstants.ApiEndpoints.JwOrgIndexServiceBaseUrl}?output=json&pub={publicationCode}&booknum={bookNumber}&fileformat=MP3&alllangs=0&langwritten={languageCode}&txtCMSLang=E";
99+
continue;
100+
}
77101
}
78102
catch (Exception e)
79103
{
80-
if (e is JsonException or ArgumentException)
104+
if (e is JsonException or ArgumentException or KeyNotFoundException)
81105
{
106+
Console.WriteLine($"Warning: Error parsing JSON for book {bookNumber}, language {languageCode}: {e.Message}. Skipping.");
82107
doc?.Dispose();
83108
bookNumber++;
84109
harvestLink = $"{AppConstants.ApiEndpoints.JwOrgIndexServiceBaseUrl}?output=json&pub={publicationCode}&booknum={bookNumber}&fileformat=MP3&alllangs=0&langwritten={languageCode}&txtCMSLang=E";
@@ -90,30 +115,93 @@ private static async Task<bool> harvestBibleLinks(string languageCode, string pu
90115

91116
try
92117
{
93-
var bookFiles = files.GetProperty(languageCode).GetProperty("MP3");
118+
if (!files.TryGetProperty(languageCode, out var languageFiles))
119+
{
120+
Console.WriteLine($"Warning: Language '{languageCode}' not found in files. Skipping book {bookNumber}.");
121+
bookNumber++;
122+
harvestLink = $"{AppConstants.ApiEndpoints.JwOrgIndexServiceBaseUrl}?output=json&pub={publicationCode}&booknum={bookNumber}&fileformat=MP3&alllangs=0&langwritten={languageCode}&txtCMSLang=E";
123+
continue;
124+
}
125+
126+
if (!languageFiles.TryGetProperty("MP3", out var bookFiles))
127+
{
128+
Console.WriteLine($"Warning: 'MP3' property not found for language '{languageCode}'. Skipping book {bookNumber}.");
129+
bookNumber++;
130+
harvestLink = $"{AppConstants.ApiEndpoints.JwOrgIndexServiceBaseUrl}?output=json&pub={publicationCode}&booknum={bookNumber}&fileformat=MP3&alllangs=0&langwritten={languageCode}&txtCMSLang=E";
131+
continue;
132+
}
133+
94134
foreach (var bookFile in bookFiles.EnumerateArray())
95135
{
96-
string url = bookFile.GetProperty("file").GetProperty("url").GetString()!;
97-
var track = bookFile.GetProperty("track").GetInt32();
136+
if (!bookFile.TryGetProperty("file", out var fileElement) ||
137+
!fileElement.TryGetProperty("url", out var urlElement))
138+
{
139+
Console.WriteLine($"Warning: Missing 'file.url' property in book file. Skipping.");
140+
continue;
141+
}
142+
143+
string url = urlElement.GetString();
144+
if (string.IsNullOrEmpty(url))
145+
{
146+
Console.WriteLine($"Warning: URL is null or empty. Skipping.");
147+
continue;
148+
}
149+
150+
if (!bookFile.TryGetProperty("track", out var trackElement))
151+
{
152+
Console.WriteLine($"Warning: Missing 'track' property in book file. Skipping.");
153+
continue;
154+
}
98155

99-
if (track == 0
100-
|| url.EndsWith(".zip")) continue;
156+
var track = trackElement.GetInt32();
101157

102-
bookNumber = bookFile.GetProperty("booknum").GetInt32();
158+
if (track == 0 || url.EndsWith(".zip")) continue;
159+
160+
if (!bookFile.TryGetProperty("booknum", out var bookNumElement))
161+
{
162+
Console.WriteLine($"Warning: Missing 'booknum' property in book file. Skipping.");
163+
continue;
164+
}
165+
166+
bookNumber = bookNumElement.GetInt32();
103167

104168
if (!bookNumberBookMap.ContainsKey(bookNumber))
105169
{
106-
var name = harvestLink.Contains("booknum=") ? doc.RootElement.GetProperty("pubName").GetString()! : bookFile.GetProperty("title").GetString()!.Split('-')[0].Trim();
170+
string name;
171+
if (harvestLink.Contains("booknum="))
172+
{
173+
if (!doc.RootElement.TryGetProperty("pubName", out var pubNameElement))
174+
{
175+
Console.WriteLine($"Warning: Missing 'pubName' property. Skipping book {bookNumber}.");
176+
continue;
177+
}
178+
name = pubNameElement.GetString()!;
179+
}
180+
else
181+
{
182+
if (!bookFile.TryGetProperty("title", out var titleElement))
183+
{
184+
Console.WriteLine($"Warning: Missing 'title' property. Skipping book {bookNumber}.");
185+
continue;
186+
}
187+
name = titleElement.GetString()!.Split('-')[0].Trim();
188+
}
107189
name = name == "Psalm 1" ? "Psalms" : name;
108190
bookNumberBookMap[bookNumber] = new BibleBook
109191
{
110-
Number = bookFile.GetProperty("booknum").GetInt32(),
192+
Number = bookNumber,
111193
Name = name
112194
};
113195
}
114196

115-
var trackNumber = bookFile.GetProperty("track").GetInt32();
116-
var duration = bookFile.GetProperty("duration").GetDouble();
197+
var trackNumber = track;
198+
199+
if (!bookFile.TryGetProperty("duration", out var durationElement))
200+
{
201+
Console.WriteLine($"Warning: Missing 'duration' property. Using default value.");
202+
}
203+
var duration = durationElement.ValueKind != JsonValueKind.Undefined ? durationElement.GetDouble() : 0.0;
204+
117205
if (!bookNumberChapterMap.ContainsKey(bookNumber))
118206
{
119207
bookNumberChapterMap[bookNumber] = new Dictionary<int, BibleChapter>();
@@ -125,11 +213,18 @@ private static async Task<bool> harvestBibleLinks(string languageCode, string pu
125213
new BibleChapter
126214
{
127215
Number = trackNumber,
128-
Url = bookFile.GetProperty("file").GetProperty("url").GetString()!,
216+
Url = url,
129217
});
130218
}
131219
}
132220
}
221+
catch (KeyNotFoundException ex)
222+
{
223+
Console.WriteLine($"Warning: KeyNotFoundException in book processing: {ex.Message}. Skipping book {bookNumber}.");
224+
bookNumber++;
225+
harvestLink = $"{AppConstants.ApiEndpoints.JwOrgIndexServiceBaseUrl}?output=json&pub={publicationCode}&booknum={bookNumber}&fileformat=MP3&alllangs=0&langwritten={languageCode}&txtCMSLang=E";
226+
continue;
227+
}
133228
finally
134229
{
135230
doc?.Dispose();

‎.tools/Bible.Alarm.Audio.Links.Harvestor/Harvestors/Music/MusicHarvester.cs‎

Lines changed: 72 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,29 @@ internal async static Task Harvest_Vocal_Music_Links()
3131
var jsonString = await DownloadUtility.GetAsync(harvestLink);
3232
using var doc = JsonDocument.Parse(jsonString);
3333
var root = doc.RootElement;
34-
var languages = root.GetProperty("languages");
34+
35+
if (!root.TryGetProperty("languages", out var languages))
36+
{
37+
Console.WriteLine($"Warning: 'languages' property not found in response for publication {publication.Key}. Skipping.");
38+
continue;
39+
}
3540

3641
foreach (var item in languages.EnumerateObject())
3742
{
3843
var languageCode = item.Name;
39-
var language = item.Value.GetProperty("name").GetString();
44+
45+
if (!item.Value.TryGetProperty("name", out var nameElement))
46+
{
47+
Console.WriteLine($"Warning: 'name' property not found for language {languageCode}. Skipping.");
48+
continue;
49+
}
50+
51+
var language = nameElement.GetString();
52+
if (string.IsNullOrEmpty(language))
53+
{
54+
Console.WriteLine($"Warning: Language name is null or empty for language {languageCode}. Skipping.");
55+
continue;
56+
}
4057

4158
Console.WriteLine($"Harvesting Music track links for {publication.Value} of {language} language.");
4259

@@ -153,21 +170,67 @@ private static async Task<bool> harvestMusicLinks(string publicationCode, List<s
153170

154171
var lc = languageCode ?? "E";
155172

156-
var musicFiles = root.GetProperty("files").GetProperty(lc).GetProperty("MP3");
173+
if (!root.TryGetProperty("files", out var filesElement))
174+
{
175+
Console.WriteLine($"Warning: 'files' property not found for publication {publicationDownloadCode}. Skipping.");
176+
continue;
177+
}
178+
179+
if (!filesElement.TryGetProperty(lc, out var languageFiles))
180+
{
181+
Console.WriteLine($"Warning: Language '{lc}' not found in files for publication {publicationDownloadCode}. Skipping.");
182+
continue;
183+
}
184+
185+
if (!languageFiles.TryGetProperty("MP3", out var musicFiles))
186+
{
187+
Console.WriteLine($"Warning: 'MP3' property not found for language '{lc}' in publication {publicationDownloadCode}. Skipping.");
188+
continue;
189+
}
190+
157191
foreach (var musicFile in musicFiles.EnumerateArray())
158192
{
159-
string url = musicFile.GetProperty("file").GetProperty("url").GetString()!;
160-
var track = musicFile.GetProperty("track").GetInt32();
193+
if (!musicFile.TryGetProperty("file", out var fileElement) ||
194+
!fileElement.TryGetProperty("url", out var urlElement))
195+
{
196+
Console.WriteLine($"Warning: Missing 'file.url' property in music file. Skipping.");
197+
continue;
198+
}
161199

162-
if (track == 0
163-
|| url.EndsWith(".zip"))
200+
string url = urlElement.GetString();
201+
if (string.IsNullOrEmpty(url))
202+
{
203+
Console.WriteLine($"Warning: URL is null or empty. Skipping.");
164204
continue;
205+
}
206+
207+
if (!musicFile.TryGetProperty("track", out var trackElement))
208+
{
209+
Console.WriteLine($"Warning: Missing 'track' property in music file. Skipping.");
210+
continue;
211+
}
212+
213+
var track = trackElement.GetInt32();
214+
215+
if (track == 0 || url.EndsWith(".zip"))
216+
continue;
217+
218+
if (!musicFile.TryGetProperty("duration", out var durationElement))
219+
{
220+
Console.WriteLine($"Warning: Missing 'duration' property. Using default value.");
221+
}
222+
var duration = durationElement.ValueKind != JsonValueKind.Undefined ? durationElement.GetDouble() : 0.0;
223+
224+
if (!musicFile.TryGetProperty("title", out var titleElement))
225+
{
226+
Console.WriteLine($"Warning: Missing 'title' property. Using default value.");
227+
}
228+
var title = titleElement.ValueKind != JsonValueKind.Undefined ? titleElement.GetString()! : "Unknown";
165229

166-
var duration = musicFile.GetProperty("duration").GetDouble();
167230
musicTracks.Add(new MusicTrack
168231
{
169232
Number = trackNumber,
170-
Title = musicFile.GetProperty("title").GetString()!,
233+
Title = title,
171234
Url = url,
172235
LookUpPath = $"?output=json&pub={publicationDownloadCode}&fileformat=MP3" +
173236
$"{(languageCode == null ? "&langwritten=E" : $"&langwritten={languageCode}")}" +

‎.tools/Bible.Alarm.Audio.Links.Harvestor/Utility/DirectoryHelper.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ public static class DirectoryHelper
1616
currentDir = currentDir.Parent;
1717
}
1818

19-
return Path.Combine(currentDir.FullName, "src", "_tools", "_index");
19+
return Path.Combine(currentDir.FullName, ".tools", "_index");
2020
});
2121

2222
public static void Ensure(string directory)

‎src/Bible.Alarm/Services/Media/MediaUrlRefreshService.cs‎

Lines changed: 69 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#nullable enable
2+
using System.Linq;
23
using System.Text;
34
using System.Text.Json;
45
using Bible.Alarm.Services.Media.Interfaces;
@@ -72,7 +73,40 @@ public MediaUrlRefreshService(ILogger logger, IDownloadService downloadService)
7273
using var doc = JsonDocument.Parse(jsonString);
7374
var root = doc.RootElement;
7475

75-
return root.GetProperty("files").GetProperty(languageCode).GetProperty("MP3")[0].GetProperty("file").GetProperty("url").GetString();
76+
if (!root.TryGetProperty("files", out var files))
77+
{
78+
_logger.Warning("'files' property not found in JSON response for Bible chapter URL refresh");
79+
return null;
80+
}
81+
82+
if (!files.TryGetProperty(languageCode, out var languageFiles))
83+
{
84+
_logger.Warning("Language '{LanguageCode}' not found in files for Bible chapter URL refresh", languageCode);
85+
return null;
86+
}
87+
88+
if (!languageFiles.TryGetProperty("MP3", out var mp3Files))
89+
{
90+
_logger.Warning("'MP3' property not found for language '{LanguageCode}' in Bible chapter URL refresh", languageCode);
91+
return null;
92+
}
93+
94+
var mp3Array = mp3Files.EnumerateArray().ToList();
95+
if (mp3Array.Count == 0)
96+
{
97+
_logger.Warning("No MP3 files found for language '{LanguageCode}' in Bible chapter URL refresh", languageCode);
98+
return null;
99+
}
100+
101+
var firstMp3 = mp3Array[0];
102+
if (!firstMp3.TryGetProperty("file", out var fileElement) ||
103+
!fileElement.TryGetProperty("url", out var urlElement))
104+
{
105+
_logger.Warning("Missing 'file.url' property in MP3 file for Bible chapter URL refresh");
106+
return null;
107+
}
108+
109+
return urlElement.GetString();
76110
}
77111
catch
78112
{
@@ -96,7 +130,40 @@ public MediaUrlRefreshService(ILogger logger, IDownloadService downloadService)
96130

97131
if (lc == AppConstants.Media.LanguageCodePatchFrom) lc = AppConstants.Media.LanguageCodePatchTo;
98132

99-
return root.GetProperty("files").GetProperty(lc).GetProperty("MP3")[0].GetProperty("file").GetProperty("url").GetString();
133+
if (!root.TryGetProperty("files", out var files))
134+
{
135+
_logger.Warning("'files' property not found in JSON response for music track URL refresh");
136+
return null;
137+
}
138+
139+
if (!files.TryGetProperty(lc, out var languageFiles))
140+
{
141+
_logger.Warning("Language '{LanguageCode}' not found in files for music track URL refresh", lc);
142+
return null;
143+
}
144+
145+
if (!languageFiles.TryGetProperty("MP3", out var mp3Files))
146+
{
147+
_logger.Warning("'MP3' property not found for language '{LanguageCode}' in music track URL refresh", lc);
148+
return null;
149+
}
150+
151+
var mp3Array = mp3Files.EnumerateArray().ToList();
152+
if (mp3Array.Count == 0)
153+
{
154+
_logger.Warning("No MP3 files found for language '{LanguageCode}' in music track URL refresh", lc);
155+
return null;
156+
}
157+
158+
var firstMp3 = mp3Array[0];
159+
if (!firstMp3.TryGetProperty("file", out var fileElement) ||
160+
!fileElement.TryGetProperty("url", out var urlElement))
161+
{
162+
_logger.Warning("Missing 'file.url' property in MP3 file for music track URL refresh");
163+
return null;
164+
}
165+
166+
return urlElement.GetString();
100167
}
101168
catch
102169
{

0 commit comments

Comments
 (0)