Skip to content

Commit 7fe6a4c

Browse files
SamTV12345SamTV12345
authored
Feature/optimize rss feeds (#1526)
* chore: merge every pr * chore: add merge retry sleep * fix: fix timeline to be handled by queryclient * fix: fixed opmladd * chore: optimized rss feed retrieval by 20x * chore: format * chore: format
1 parent 0548ddf commit 7fe6a4c

File tree

11 files changed

+81
-66
lines changed

11 files changed

+81
-66
lines changed

‎.github/workflows/pr-build.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ jobs:
1616
- uses: dtolnay/rust-toolchain@stable
1717
with:
1818
toolchain: stable
19+
components: clippy, rustfmt
1920
- name: Set up cargo cache
2021
uses: actions/cache@v3
2122
continue-on-error: false
@@ -41,6 +42,7 @@ jobs:
4142
- uses: dtolnay/rust-toolchain@stable
4243
with:
4344
toolchain: stable
45+
components: clippy, rustfmt
4446
- name: Set up cargo cache
4547
uses: actions/cache@v3
4648
continue-on-error: false
@@ -66,6 +68,7 @@ jobs:
6668
- uses: dtolnay/rust-toolchain@stable
6769
with:
6870
toolchain: stable
71+
components: clippy, rustfmt
6972
- name: Set up cargo cache
7073
uses: actions/cache@v3
7174
continue-on-error: false

‎.github/workflows/rust.yml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,7 @@ jobs:
4444
uses: dtolnay/rust-toolchain@stable
4545
with:
4646
toolchain: stable
47-
default: true
48-
override: true
47+
components: clippy, rustfmt
4948

5049
- name: Build
5150
run: cargo build --all --release && strip target/release/podfetch && mv target/release/podfetch target/release/podfetch_amd64
@@ -112,8 +111,7 @@ jobs:
112111
uses: dtolnay/rust-toolchain@stable
113112
with:
114113
toolchain: stable
115-
default: true
116-
override: true
114+
components: clippy, rustfmt
117115

118116
- name: Build
119117
run: cargo build --all --release

‎src/commands/startup.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,6 @@ async fn index() -> Result<Response<String>, CustomError> {
116116
.filter(|x| x.starts_with("index") && x.ends_with(".css"))
117117
.collect::<Vec<&String>>()[0];
118118

119-
120119
let config = ENVIRONMENT_SERVICE.get_config();
121120
let config_string = serde_json::to_string(&config).unwrap();
122121
let html = html! {

‎src/constants/inner_constants.rs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
use std::convert::Into;
21
use crate::service::environment_service::EnvironmentService;
2+
use std::convert::Into;
33
use std::fmt;
44
use std::fmt::Formatter;
55
use std::string::ToString;
@@ -60,8 +60,6 @@ pub enum Role {
6060
User,
6161
}
6262

63-
64-
6563
impl fmt::Display for Role {
6664
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
6765
match self {
@@ -145,7 +143,7 @@ pub const COMMON_USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) A
145143

146144
pub const OIDC_JWKS: &str = "OIDC_JWKS";
147145
pub const OIDC_REFRESH_INTERVAL: &str = "OIDC_REFRESH_INTERVAL";
148-
pub const DEFAULT_OIDC_REFRESH_INTERVAL: u64 = 1000*60*2; // 2 minutes
146+
pub const DEFAULT_OIDC_REFRESH_INTERVAL: u64 = 1000 * 60 * 2; // 2 minutes
149147

150148
// Default device when viewing via web interface
151149
pub const DEFAULT_DEVICE: &str = "webview";

‎src/controllers/podcast_controller.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
use crate::constants::inner_constants::{BASIC_AUTH, COMMON_USER_AGENT, DEFAULT_IMAGE_URL, ENVIRONMENT_SERVICE, OIDC_AUTH};
1+
use crate::constants::inner_constants::{
2+
BASIC_AUTH, COMMON_USER_AGENT, DEFAULT_IMAGE_URL, ENVIRONMENT_SERVICE, OIDC_AUTH,
3+
};
24
use crate::models::dto_models::PodcastFavorUpdateModel;
35
use crate::models::misc_models::{PodcastAddModel, PodcastInsertModel};
46
use crate::models::opml_model::OpmlModel;
@@ -60,7 +62,9 @@ pub async fn get_filter(
6062
username: "".into(),
6163
};
6264

63-
let filter = Filter::get_filter_by_username(&requester.username).await?.unwrap_or(default_filter);
65+
let filter = Filter::get_filter_by_username(&requester.username)
66+
.await?
67+
.unwrap_or(default_filter);
6468
Ok(Json(filter))
6569
}
6670

‎src/controllers/sys_info_controller.rs‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,9 @@ responses(
1919
body = SysExtraInfo)),
2020
tag="sys"
2121
)]
22-
pub async fn get_sys_info(Extension(requester): Extension<User>) -> Result<Json<SysExtraInfo>, CustomError> {
23-
22+
pub async fn get_sys_info(
23+
Extension(requester): Extension<User>,
24+
) -> Result<Json<SysExtraInfo>, CustomError> {
2425
if !requester.is_admin() {
2526
return Err(CustomErrorInner::Forbidden(Info).into());
2627
}
@@ -58,11 +59,13 @@ pub async fn get_sys_info(Extension(requester): Extension<User>) -> Result<Json<
5859
use crate::constants::inner_constants::ENVIRONMENT_SERVICE;
5960
use crate::models::settings::ConfigModel;
6061
use crate::utils::error::ErrorSeverity::Info;
61-
use crate::utils::error::{CustomError, CustomErrorInner, ErrorSeverity, map_io_extra_error, ErrorType, ApiError};
62+
use crate::utils::error::ErrorType::CustomErrorType;
63+
use crate::utils::error::{
64+
ApiError, CustomError, CustomErrorInner, ErrorSeverity, ErrorType, map_io_extra_error,
65+
};
6266
use utoipa::ToSchema;
6367
use utoipa_axum::router::OpenApiRouter;
6468
use utoipa_axum::routes;
65-
use crate::utils::error::ErrorType::CustomErrorType;
6669

6770
#[derive(Debug, Serialize, ToSchema)]
6871
pub struct SysExtraInfo {
@@ -185,7 +188,7 @@ pub async fn login(auth: Json<LoginRequest>) -> Result<StatusCode, ErrorType> {
185188
log::warn!("Login failed for user {}", auth.0.username);
186189
return Err(ApiError::wrong_user_or_password().into());
187190
}
188-
return Err(CustomErrorType(err))
191+
return Err(CustomErrorType(err));
189192
}
190193
};
191194

‎src/controllers/websocket_controller.rs‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -243,32 +243,30 @@ fn get_podcast_items_rss(downloaded_episodes: &[PodcastEpisodeDto]) -> Vec<Item>
243243
downloaded_episodes
244244
.iter()
245245
.map(|episode| {
246-
let episode = episode.clone();
247-
248246
let enclosure = EnclosureBuilder::default()
249-
.url(episode.local_url.clone())
250-
.length(episode.clone().total_time.to_string())
247+
.url(&episode.local_url)
248+
.length(episode.total_time.to_string())
251249
.mime_type(format!(
252250
"audio/{}",
253-
PodcastEpisodeService::get_url_file_suffix(&episode.clone().local_url).unwrap()
251+
PodcastEpisodeService::get_url_file_suffix(&episode.local_url).unwrap()
254252
))
255253
.build();
256254

257255
let itunes_extension = ITunesItemExtensionBuilder::default()
258-
.duration(Some(episode.clone().total_time.to_string()))
259-
.image(Some(episode.clone().local_image_url))
256+
.duration(Some(episode.total_time.to_string()))
257+
.image(Some(episode.local_image_url.to_string()))
260258
.build();
261259

262260
let guid = GuidBuilder::default()
263261
.permalink(false)
264-
.value(episode.clone().episode_id)
262+
.value(&episode.episode_id)
265263
.build();
266264

267265
ItemBuilder::default()
268266
.guid(Some(guid))
269-
.pub_date(Some(episode.clone().date_of_recording))
270-
.title(Some(episode.clone().name))
271-
.description(Some(episode.clone().description))
267+
.pub_date(Some(episode.date_of_recording.to_string()))
268+
.title(Some(episode.name.to_string()))
269+
.description(Some(episode.description.to_string()))
272270
.enclosure(Some(enclosure))
273271
.itunes_ext(itunes_extension)
274272
.build()

‎src/models/favorites.rs‎

Lines changed: 14 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@ use crate::utils::error::{CustomError, map_db_error};
1313
use diesel::insert_into;
1414
use diesel::prelude::*;
1515
use diesel::sql_types::{Bool, Integer, Text};
16+
use indexmap::IndexMap;
1617
use serde::{Deserialize, Serialize};
1718
use std::collections::BTreeMap;
18-
use indexmap::IndexMap;
1919
use utoipa::ToSchema;
2020

2121
#[derive(
@@ -135,7 +135,6 @@ impl Favorite {
135135
latest_pub: OrderOption,
136136
designated_username: &str,
137137
) -> Result<Vec<(Podcast, Favorite, Vec<Tag>)>, CustomError> {
138-
139138
use crate::adapters::persistence::dbconfig::schema::podcasts::dsl::id as podcastsid;
140139
use crate::adapters::persistence::dbconfig::schema::podcasts::dsl::*;
141140
use crate::adapters::persistence::dbconfig::schema::tags_podcasts as t_join_table;
@@ -189,19 +188,15 @@ impl Favorite {
189188
let mut matching_podcast_ids: BTreeMap<i32, (Podcast, Favorite, Vec<Tag>)> =
190189
BTreeMap::new();
191190
let pr = query
192-
.load::<(
193-
Podcast,
194-
Favorite,
195-
Option<TagsPodcast>,
196-
Option<Tag>,
197-
)>(&mut get_connection())
191+
.load::<(Podcast, Favorite, Option<TagsPodcast>, Option<Tag>)>(&mut get_connection())
198192
.map_err(|e| map_db_error(e, Critical))?;
199193
pr.iter().for_each(|c| {
200194
if let Some(existing) = matching_podcast_ids.get_mut(&c.0.id) {
201195
if let Some(tag) = &c.3
202-
&& !existing.2.iter().any(|t| t.id == tag.id) {
203-
existing.2.push(tag.clone());
204-
}
196+
&& !existing.2.iter().any(|t| t.id == tag.id)
197+
{
198+
existing.2.push(tag.clone());
199+
}
205200
} else {
206201
let mut tags = vec![];
207202
if let Some(tag) = &c.3 {
@@ -214,7 +209,6 @@ impl Favorite {
214209
Ok(matching_podcast_ids.values().cloned().collect())
215210
}
216211

217-
218212
pub fn search_podcasts(
219213
order: OrderCriteria,
220214
title: Option<String>,
@@ -224,7 +218,7 @@ impl Favorite {
224218
use crate::adapters::persistence::dbconfig::schema::favorites::dsl::favorites as f_db;
225219
use crate::adapters::persistence::dbconfig::schema::favorites::dsl::podcast_id as f_id;
226220
use crate::adapters::persistence::dbconfig::schema::favorites::dsl::username as f_username;
227-
221+
228222
use crate::adapters::persistence::dbconfig::schema::podcasts::dsl::id as podcastsid;
229223
use crate::adapters::persistence::dbconfig::schema::podcasts::dsl::*;
230224

@@ -277,19 +271,17 @@ impl Favorite {
277271
let mut matching_podcast_ids: IndexMap<i32, (Podcast, Option<Favorite>, Vec<Tag>)> =
278272
IndexMap::new();
279273
let pr = query
280-
.load::<(
281-
Podcast,
282-
Option<Favorite>,
283-
Option<TagsPodcast>,
284-
Option<Tag>,
285-
)>(&mut get_connection())
274+
.load::<(Podcast, Option<Favorite>, Option<TagsPodcast>, Option<Tag>)>(
275+
&mut get_connection(),
276+
)
286277
.map_err(|e| map_db_error(e, Critical))?;
287278
pr.iter().for_each(|c| {
288279
if let Some(existing) = matching_podcast_ids.get_mut(&c.0.id) {
289280
if let Some(tag) = &c.3
290-
&& !existing.2.iter().any(|t| t.id == tag.id) {
291-
existing.2.push(tag.clone());
292-
}
281+
&& !existing.2.iter().any(|t| t.id == tag.id)
282+
{
283+
existing.2.push(tag.clone());
284+
}
293285
} else {
294286
let mut tags = vec![];
295287
if let Some(tag) = &c.3 {

‎src/service/environment_service.rs‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,16 @@ use crate::adapters::file::file_handler::FileHandlerType;
33
#[cfg(feature = "postgresql")]
44
use crate::constants::inner_constants::CONNECTION_NUMBERS;
55

6-
use crate::constants::inner_constants::{API_KEY, BASIC_AUTH, DATABASE_URL, DATABASE_URL_DEFAULT_SQLITE, DEFAULT_OIDC_REFRESH_INTERVAL, DEFAULT_PODFETCH_FOLDER, FILE_HANDLER, GPODDER_INTEGRATION_ENABLED, OIDC_AUTH, OIDC_AUTHORITY, OIDC_CLIENT_ID, OIDC_JWKS, OIDC_REDIRECT_URI, OIDC_REFRESH_INTERVAL, OIDC_SCOPE, PASSWORD, PODFETCH_FOLDER, PODFETCH_PROXY_FOR_REQUESTS, PODINDEX_API_KEY, PODINDEX_API_SECRET, POLLING_INTERVAL, POLLING_INTERVAL_DEFAULT, REVERSE_PROXY, REVERSE_PROXY_AUTO_SIGN_UP, REVERSE_PROXY_HEADER, S3_ACCESS_KEY, S3_PROFILE, S3_REGION, S3_SECRET_KEY, S3_SECURITY_TOKEN, S3_SESSION_TOKEN, S3_URL, SERVER_URL, SUB_DIRECTORY, TELEGRAM_API_ENABLED, TELEGRAM_BOT_CHAT_ID, TELEGRAM_BOT_TOKEN, USERNAME};
6+
use crate::constants::inner_constants::{
7+
API_KEY, BASIC_AUTH, DATABASE_URL, DATABASE_URL_DEFAULT_SQLITE, DEFAULT_OIDC_REFRESH_INTERVAL,
8+
DEFAULT_PODFETCH_FOLDER, FILE_HANDLER, GPODDER_INTEGRATION_ENABLED, OIDC_AUTH, OIDC_AUTHORITY,
9+
OIDC_CLIENT_ID, OIDC_JWKS, OIDC_REDIRECT_URI, OIDC_REFRESH_INTERVAL, OIDC_SCOPE, PASSWORD,
10+
PODFETCH_FOLDER, PODFETCH_PROXY_FOR_REQUESTS, PODINDEX_API_KEY, PODINDEX_API_SECRET,
11+
POLLING_INTERVAL, POLLING_INTERVAL_DEFAULT, REVERSE_PROXY, REVERSE_PROXY_AUTO_SIGN_UP,
12+
REVERSE_PROXY_HEADER, S3_ACCESS_KEY, S3_PROFILE, S3_REGION, S3_SECRET_KEY, S3_SECURITY_TOKEN,
13+
S3_SESSION_TOKEN, S3_URL, SERVER_URL, SUB_DIRECTORY, TELEGRAM_API_ENABLED,
14+
TELEGRAM_BOT_CHAT_ID, TELEGRAM_BOT_TOKEN, USERNAME,
15+
};
716
use crate::models::settings::ConfigModel;
817
use crate::utils::environment_variables::is_env_var_present_and_true;
918
use s3::creds::Credentials;
@@ -22,7 +31,7 @@ pub struct OidcConfig {
2231
pub redirect_uri: String,
2332
pub scope: String,
2433
pub jwks_uri: String,
25-
pub refresh_interval: u64
34+
pub refresh_interval: u64,
2635
}
2736

2837
pub struct EnvironmentService {
@@ -119,8 +128,10 @@ impl EnvironmentService {
119128
client_id: var(OIDC_CLIENT_ID).expect("OIDC client id not configured"),
120129
scope: var(OIDC_SCOPE).unwrap_or("openid profile email".to_string()),
121130
jwks_uri: var(OIDC_JWKS).unwrap(),
122-
refresh_interval: var(OIDC_REFRESH_INTERVAL).unwrap_or
123-
(DEFAULT_OIDC_REFRESH_INTERVAL.to_string()).parse::<u64>().unwrap_or(DEFAULT_OIDC_REFRESH_INTERVAL),
131+
refresh_interval: var(OIDC_REFRESH_INTERVAL)
132+
.unwrap_or(DEFAULT_OIDC_REFRESH_INTERVAL.to_string())
133+
.parse::<u64>()
134+
.unwrap_or(DEFAULT_OIDC_REFRESH_INTERVAL),
124135
})
125136
} else {
126137
None

‎src/service/podcast_episode_service.rs‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,12 @@ use crate::utils::podcast_builder::PodcastBuilder;
2424
use crate::utils::reqwest_client::get_sync_client;
2525
use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl};
2626
use log::error;
27-
use regex::Regex;
2827
use reqwest::header::{ACCEPT, HeaderMap};
2928
use reqwest::redirect::Policy;
3029
use rss::{Channel, Item};
30+
use std::ffi::OsStr;
3131
use std::io::Error;
32+
use std::path::Path;
3233
use std::sync::{Arc, Mutex};
3334

3435
pub struct PodcastEpisodeService;
@@ -357,13 +358,25 @@ impl PodcastEpisodeService {
357358
}
358359
}
359360

361+
#[inline]
360362
pub fn get_url_file_suffix(url: &str) -> Result<String, Error> {
361-
let re = Regex::new(r"\.(\w+)(?:\?.*)?$").unwrap();
362-
let capture = re.captures(url);
363-
if capture.is_none() {
364-
return Err(Error::other("No"));
365-
}
366-
Ok(capture.unwrap().get(1).unwrap().as_str().to_string())
363+
/*let url = url::Url::parse(url);
364+
365+
let converted_url = match url {
366+
Ok(url) => {
367+
Ok(url)
368+
},
369+
Err(_) => {
370+
Err(Error::other("No"))
371+
}
372+
}?;
373+
let file_name = converted_url.path_segments().iter().last().ok_or(Error::other("No"))?;
374+
file_name.rsplit('.').next().filter(|s| *s != file_name).map(|s| s.to_string())*/
375+
Ok(Path::new(url)
376+
.extension()
377+
.unwrap_or(OsStr::new(""))
378+
.to_string_lossy()
379+
.into_owned())
367380
}
368381

369382
pub fn query_for_podcast(query: &str) -> Result<Vec<PodcastEpisode>, CustomError> {

0 commit comments

Comments
 (0)