Merge branch 'redlib-org:main' into main

Author: espacedev3019

Cargo.lock

@@ -3,16 +3,20 @@ name = "redlib"
 description = " Alternative private front-end to Reddit"
 license = "AGPL-3.0-only"
 repository = "https://github.com/redlib-org/redlib"
-version = "0.35.1"
+version = "0.36.0"
 authors = [
     "Matthew Esposito <matt+cargo@matthew.science>",
     "spikecodes <19519553+spikecodes@users.noreply.github.com>",
 ]
 edition = "2021"
+rust-version = "1.81"
 default-run = "redlib"
 
 [dependencies]
-rinja = { version = "0.3.4", default-features = false }
+askama = { version = "0.14.0", default-features = false, features = [
+    "std",
+    "derive",
+] }
 cached = { version = "0.54.0", features = ["async"] }
 clap = { version = "4.4.11", default-features = false, features = [
     "std",
@@ -27,14 +31,13 @@ hyper = { version = "0.14.31", features = ["full"] }
 percent-encoding = "2.3.1"
 route-recognizer = "0.3.1"
 serde_json = "1.0.133"
-tokio = { version = "1.35.1", features = ["full"] }
+tokio = { version = "1.44.2", features = ["full"] }
 time = { version = "0.3.31", features = ["local-offset"] }
 url = "2.5.0"
 rust-embed = { version = "8.1.0", features = ["include-exclude"] }
 libflate = "2.0.0"
 brotli = { version = "7.0.0", features = ["std"] }
 toml = "0.8.8"
-once_cell = "1.19.0"
 serde_yaml = "0.9.29"
 build_html = "2.4.0"
 uuid = { version = "1.6.1", features = ["v4"] }
@@ -56,7 +59,8 @@ htmlescape = "0.3.1"
 bincode = "1.3.3"
 base2048 = "2.0.2"
 revision = "0.10.0"
-
+fake_user_agent = "0.2.2"
+rustls = "0.21.12"
 
 [dev-dependencies]
 lipsum = "0.9.0"

Cargo.toml

@@ -37,7 +37,7 @@ USER redlib
 EXPOSE 8080
 
 # Run a healthcheck every minute to make sure redlib is functional
-HEALTHCHECK --interval=1m --timeout=3s CMD wget --spider --q http://localhost:8080/settings || exit 1
+HEALTHCHECK --interval=1m --timeout=3s CMD wget --spider -q http://localhost:8080/settings || exit 1
 
 # Add container metadata
 LABEL org.opencontainers.image.authors="sigaloid"

Dockerfile.alpine

@@ -43,7 +43,7 @@ USER redlib
 EXPOSE 8080
 
 # Run a healthcheck every minute to make sure redlib is functional
-HEALTHCHECK --interval=1m --timeout=3s CMD wget --spider --q http://localhost:8080/settings || exit 1
+HEALTHCHECK --interval=1m --timeout=3s CMD wget --spider -q http://localhost:8080/settings || exit 1
 
 # Add container metadata
 LABEL org.opencontainers.image.authors="sigaloid"

Dockerfile.ubuntu

@@ -6,7 +6,7 @@
 
 ---
 
-**10-second pitch:** Redlib is a private front-end like [Invidious](https://github.com/iv-org/invidious) but for Reddit. Browse the coldest takes of [r/unpopularopinion](https://redlib.matthew.science/r/unpopularopinion) without being [tracked](#reddit).
+**10-second pitch:** Redlib is a private front-end like [Invidious](https://github.com/iv-org/invidious) but for Reddit. Browse the coldest takes of [r/unpopularopinion](https://farside.link/redlib/r/unpopularopinion) without being [tracked](#reddit).
 
 - 🚀 Fast: written in Rust for blazing-fast speeds and memory safety
 - ☁️ Light: no JavaScript, no ads, no tracking, no bloat
@@ -30,7 +30,6 @@
      - [Reddit](#reddit)
      - [Redlib](#redlib-1)
        - [Server](#server)
-       - [Official instance (redlib.matthew.science)](#official-instance-redlibmatthewscience)
 5. [Deployment](#deployment)
    - [Docker](#docker)
      - [Docker Compose](#docker-compose)
@@ -75,7 +74,7 @@ Redlib currently implements most of Reddit's (signed-out) functionalities but st
 
 - [Rust](https://www.rust-lang.org/) - Programming language
 - [Hyper](https://github.com/hyperium/hyper) - HTTP server and client
-- [Rinja](https://github.com/rinja-rs/rinja) - Templating engine
+- [Askama](https://github.com/askama-rs/askama) - Templating engine
 - [Rustls](https://github.com/rustls/rustls) - TLS library
 
 ## How is it different from other Reddit front ends?
@@ -159,17 +158,7 @@ For transparency, I hope to describe all the ways Redlib handles user privacy.
 
 - **Logging:** In production (when running the binary, hosting with docker, or using the official instances), Redlib logs nothing. When debugging (running from source without `--release`), Redlib logs post IDs fetched to aid with troubleshooting.
 
-- **Cookies:** Redlib uses optional cookies to store any configured settings in [the settings menu](https://redlib.matthew.science/settings). These are not cross-site cookies and the cookies hold no personal data.
-
-#### Official instance (redlib.matthew.science)
-
-The official instance is hosted at https://redlib.matthew.science.
-
-- **Server:** The official instance runs a production binary, and thus logs nothing.
-
-- **DNS:** The domain for the official instance uses Cloudflare as the DNS resolver. However, this site is not proxied through Cloudflare, and thus Cloudflare doesn't have access to user traffic.
-
-- **Hosting:** The official instance is hosted on [Replit](https://replit.com/), which monitors usage to prevent abuse. I can understand if this invalidates certain users' threat models, and therefore, self-hosting, using unofficial instances, and browsing through Tor are welcomed.
+- **Cookies:** Redlib uses optional cookies to store any configured settings in the settings menu. These are not cross-site cookies and the cookies hold no personal data.
 
 ---
 

README.md

@@ -76,6 +76,9 @@
     },
     "REDLIB_FULL_URL": {
       "required": false
+    },
+    "REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS": {
+      "required": false
     }
   }
 }

app.json

@@ -5,19 +5,19 @@ use futures_lite::{future::Boxed, FutureExt};
 use hyper::client::HttpConnector;
 use hyper::header::HeaderValue;
 use hyper::{body, body::Buf, header, Body, Client, Method, Request, Response, Uri};
-use hyper_rustls::HttpsConnector;
+use hyper_rustls::{ConfigBuilderExt, HttpsConnector};
 use libflate::gzip;
 use log::{error, trace, warn};
-use once_cell::sync::Lazy;
 use percent_encoding::{percent_encode, CONTROLS};
 use serde_json::Value;
 
 use std::sync::atomic::Ordering;
 use std::sync::atomic::{AtomicBool, AtomicU16};
+use std::sync::LazyLock;
 use std::{io, result::Result};
 
 use crate::dbg_msg;
-use crate::oauth::{force_refresh_token, token_daemon, Oauth};
+use crate::oauth::{force_refresh_token, token_daemon, Oauth, OauthBackendImpl};
 use crate::server::RequestExt;
 use crate::utils::{format_url, Post};
 
@@ -30,12 +30,40 @@ const REDDIT_SHORT_URL_BASE_HOST: &str = "redd.it";
 const ALTERNATIVE_REDDIT_URL_BASE: &str = "https://www.reddit.com";
 const ALTERNATIVE_REDDIT_URL_BASE_HOST: &str = "www.reddit.com";
 
-pub static HTTPS_CONNECTOR: Lazy<HttpsConnector<HttpConnector>> =
-	Lazy::new(|| hyper_rustls::HttpsConnectorBuilder::new().with_native_roots().https_only().enable_http2().build());
+pub static HTTPS_CONNECTOR: LazyLock<HttpsConnector<HttpConnector>> = LazyLock::new(|| {
+	hyper_rustls::HttpsConnectorBuilder::new()
+		.with_tls_config(
+			rustls::ClientConfig::builder()
+				// These are the Firefox 145.0 cipher suite,
+				// minus the suites missing forward-secrecy support,
+				// in the same order.
+				// https://github.com/redlib-org/redlib/issues/446#issuecomment-3609306592
+				.with_cipher_suites(&[
+					rustls::cipher_suite::TLS13_AES_256_GCM_SHA384,
+					rustls::cipher_suite::TLS13_AES_128_GCM_SHA256,
+					rustls::cipher_suite::TLS13_CHACHA20_POLY1305_SHA256,
+					rustls::cipher_suite::TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+					rustls::cipher_suite::TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+					rustls::cipher_suite::TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,
+					rustls::cipher_suite::TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
+					rustls::cipher_suite::TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+					rustls::cipher_suite::TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+				])
+				// .with_safe_default_cipher_suites()
+				.with_safe_default_kx_groups()
+				.with_safe_default_protocol_versions()
+				.unwrap()
+				.with_native_roots()
+				.with_no_client_auth(),
+		)
+		.https_only()
+		.enable_http2()
+		.build()
+});
 
-pub static CLIENT: Lazy<Client<HttpsConnector<HttpConnector>>> = Lazy::new(|| Client::builder().build::<_, Body>(HTTPS_CONNECTOR.clone()));
+pub static CLIENT: LazyLock<Client<HttpsConnector<HttpConnector>>> = LazyLock::new(|| Client::builder().build::<_, Body>(HTTPS_CONNECTOR.clone()));
 
-pub static OAUTH_CLIENT: Lazy<ArcSwap<Oauth>> = Lazy::new(|| {
+pub static OAUTH_CLIENT: LazyLock<ArcSwap<Oauth>> = LazyLock::new(|| {
 	let client = block_on(Oauth::new());
 	tokio::spawn(token_daemon());
 	ArcSwap::new(client.into())
@@ -154,7 +182,7 @@ async fn stream(url: &str, req: &Request<Body>) -> Result<Response<Body>, String
 	let parsed_uri = url.parse::<Uri>().map_err(|_| "Couldn't parse URL".to_string())?;
 
 	// Build the hyper client from the HTTPS connector.
-	let client: &Lazy<Client<_, Body>> = &CLIENT;
+	let client: &LazyLock<Client<_, Body>> = &CLIENT;
 
 	let mut builder = Request::get(parsed_uri);
 
@@ -165,6 +193,12 @@ async fn stream(url: &str, req: &Request<Body>) -> Result<Response<Body>, String
 		}
 	}
 
+	// Add User-Agent header of the currently spoofed device
+	{
+		let client = OAUTH_CLIENT.load_full();
+		builder = builder.header("User-Agent", client.user_agent());
+	}
+
 	let stream_request = builder.body(Body::empty()).map_err(|_| "Couldn't build empty body in stream".to_string())?;
 
 	client
@@ -216,7 +250,7 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo
 	let url = format!("{base_path}{path}");
 
 	// Construct the hyper client from the HTTPS connector.
-	let client: &Lazy<Client<_, Body>> = &CLIENT;
+	let client: &LazyLock<Client<_, Body>> = &CLIENT;
 
 	// Build request to Reddit. When making a GET, request gzip compression.
 	// (Reddit doesn't do brotli yet.)
@@ -356,7 +390,7 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo
 	.boxed()
 }
 
-// Make a request to a Reddit API and parse the JSON response
+/// Make a request to a Reddit API and parse the JSON response
 #[cached(size = 100, time = 30, result = true)]
 pub async fn json(path: String, quarantine: bool) -> Result<Value, String> {
 	// Closure to quickly build errors
@@ -488,11 +522,17 @@ async fn self_check(sub: &str) -> Result<(), String> {
 }
 
 pub async fn rate_limit_check() -> Result<(), String> {
+	// First, test the Oauth client: we can perform a rate limit check if the OAuth backend is MobileSpoof; if GenericWeb, we skip the check.
+	if matches!(OAUTH_CLIENT.load().backend, OauthBackendImpl::GenericWeb(_)) {
+		warn!("[⚠️] Cannot perform rate limit check, running as GenericWeb. Skipping check.");
+		return Ok(());
+	}
+
 	// First, check a subreddit.
 	self_check("reddit").await?;
 	// This will reduce the rate limit to 99. Assert this check.
 	if OAUTH_RATELIMIT_REMAINING.load(Ordering::SeqCst) != 99 {
-		return Err(format!("Rate limit check failed: expected 99, got {}", OAUTH_RATELIMIT_REMAINING.load(Ordering::SeqCst)));
+		return Err(format!("Rate limit check 1 failed: expected 99, got {}", OAUTH_RATELIMIT_REMAINING.load(Ordering::SeqCst)));
 	}
 	// Now, we switch out the OAuth client.
 	// This checks for the IP rate limit association.
@@ -501,7 +541,7 @@ pub async fn rate_limit_check() -> Result<(), String> {
 	self_check("rust").await?;
 	// Again, assert the rate limit check.
 	if OAUTH_RATELIMIT_REMAINING.load(Ordering::SeqCst) != 99 {
-		return Err(format!("Rate limit check failed: expected 99, got {}", OAUTH_RATELIMIT_REMAINING.load(Ordering::SeqCst)));
+		return Err(format!("Rate limit check 2 failed: expected 99, got {}", OAUTH_RATELIMIT_REMAINING.load(Ordering::SeqCst)));
 	}
 
 	Ok(())

src/client.rs

@@ -1,16 +1,12 @@
-use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
-use std::{env::var, fs::read_to_string};
+use std::{env::var, fs::read_to_string, sync::LazyLock};
 
-// Waiting for https://github.com/rust-lang/rust/issues/74465 to land, so we
-// can reduce reliance on once_cell.
-//
-// This is the local static that is initialized at runtime (technically at
-// first request) and contains the instance settings.
-pub static CONFIG: Lazy<Config> = Lazy::new(Config::load);
+/// This is the local static that is initialized at runtime (technically at
+/// first request) and contains the instance settings.
+pub static CONFIG: LazyLock<Config> = LazyLock::new(Config::load);
 
-// This serves as the frontend for an archival API - on removed comments, this URL
-// will be the base of a link, to display removed content (on another site).
+/// This serves as the frontend for an archival API - on removed comments, this URL
+/// will be the base of a link, to display removed content (on another site).
 pub const DEFAULT_PUSHSHIFT_FRONTEND: &str = "undelete.pullpush.io";
 
 /// Stores the configuration parsed from the environment variables and the
@@ -109,6 +105,9 @@ pub struct Config {
 
 	#[serde(rename = "REDLIB_FULL_URL")]
 	pub(crate) full_url: Option<String>,
+
+	#[serde(rename = "REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS")]
+	pub(crate) default_remove_default_feeds: Option<String>,
 }
 
 impl Config {
@@ -156,6 +155,7 @@ impl Config {
 			pushshift: parse("REDLIB_PUSHSHIFT_FRONTEND"),
 			enable_rss: parse("REDLIB_ENABLE_RSS"),
 			full_url: parse("REDLIB_FULL_URL"),
+			default_remove_default_feeds: parse("REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS"),
 		}
 	}
 }
@@ -185,6 +185,7 @@ fn get_setting_from_config(name: &str, config: &Config) -> Option<String> {
 		"REDLIB_PUSHSHIFT_FRONTEND" => config.pushshift.clone(),
 		"REDLIB_ENABLE_RSS" => config.enable_rss.clone(),
 		"REDLIB_FULL_URL" => config.full_url.clone(),
+		"REDLIB_DEFAULT_REMOVE_DEFAULT_FEEDS" => config.default_remove_default_feeds.clone(),
 		_ => None,
 	}
 }

src/config.rs

@@ -1,12 +1,12 @@
-// Handler for post duplicates.
+//! Handler for post duplicates.
 
 use crate::client::json;
 use crate::server::RequestExt;
 use crate::subreddit::{can_access_quarantine, quarantine};
 use crate::utils::{error, filter_posts, get_filters, nsfw_landing, parse_post, template, Post, Preferences};
 
+use askama::Template;
 use hyper::{Body, Request, Response};
-use rinja::Template;
 use serde_json::Value;
 use std::borrow::ToOwned;
 use std::collections::HashSet;

src/duplicates.rs

@@ -3,17 +3,17 @@ use crate::{
 	server::RequestExt,
 	utils::{ErrorTemplate, Preferences},
 };
+use askama::Template;
 use build_html::{Container, Html, HtmlContainer, Table};
 use hyper::{http::Error, Body, Request, Response};
-use once_cell::sync::Lazy;
-use rinja::Template;
 use serde::{Deserialize, Serialize};
+use std::sync::LazyLock;
 use time::OffsetDateTime;
 
-// This is the local static that is intialized at runtime (technically at
-// the first request to the info endpoint) and contains the data
-// retrieved from the info endpoint.
-pub static INSTANCE_INFO: Lazy<InstanceInfo> = Lazy::new(InstanceInfo::new);
+/// This is the local static that is initialized at runtime (technically at
+/// the first request to the info endpoint) and contains the data
+/// retrieved from the info endpoint.
+pub static INSTANCE_INFO: LazyLock<InstanceInfo> = LazyLock::new(InstanceInfo::new);
 
 /// Handles instance info endpoint
 pub async fn instance_info(req: Request<Body>) -> Result<Response<Body>, String> {
@@ -128,6 +128,7 @@ impl InstanceInfo {
 				["Pushshift frontend", &convert(&self.config.pushshift)],
 				["RSS enabled", &convert(&self.config.enable_rss)],
 				["Full URL", &convert(&self.config.full_url)],
+				["Remove default feeds", &convert(&self.config.default_remove_default_feeds)],
 				//TODO: fallback to crate::config::DEFAULT_PUSHSHIFT_FRONTEND
 			])
 			.with_header_row(["Settings"]),
@@ -169,6 +170,7 @@ impl InstanceInfo {
 				Pushshift frontend: {:?}\n
 				RSS enabled: {:?}\n
 				Full URL: {:?}\n
+				Remove default feeds: {:?}\n
                 Config:\n
                     Banner: {:?}\n
                     Hide awards: {:?}\n
@@ -195,6 +197,7 @@ impl InstanceInfo {
 					self.config.sfw_only,
 					self.config.enable_rss,
 					self.config.full_url,
+					self.config.default_remove_default_feeds,
 					self.config.pushshift,
 					self.config.banner,
 					self.config.default_hide_awards,