From f5c48d29c3242cb0f4a71dfb5afeea163d3561e2 Mon Sep 17 00:00:00 2001 From: Perry Gibson Date: Sat, 13 Dec 2025 14:51:27 +0000 Subject: [PATCH 1/2] feat: clarify feed generator --- .pre-commit-config.yaml | 57 +++++++-------------------------------- README.md | 7 ++--- generate_feeds.py | 60 +++++++++++++++++++++++++++++++++-------- 3 files changed, 63 insertions(+), 61 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 62851e8..f9fe403 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,63 +1,26 @@ +minimum_pre_commit_version: "4.2.0" + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: - - id: check-added-large-files - args: ["--maxkb=100"] + - id: check-json - id: check-merge-conflict + - id: check-symlinks - id: check-yaml - - id: check-json - id: end-of-file-fixer - exclude_types: ["image"] - id: trailing-whitespace exclude_types: ["image"] - id: requirements-txt-fixer - files: (requirements|constraints).*\.(txt|in)$ - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.5 + rev: v0.11.8 hooks: - id: ruff args: [--fix] - id: ruff-format - - - repo: https://github.com/pre-commit/mirrors-clang-format - rev: v19.1.7 - hooks: - - id: clang-format - "types_or": [c++, c] - - - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.5.5 - hooks: - - id: forbid-tabs - exclude: ".gitmodules|Makefile" - - - repo: https://github.com/rhysd/actionlint - rev: v1.7.7 - hooks: - - id: actionlint - - - repo: https://github.com/shellcheck-py/shellcheck-py # TODO add this - rev: v0.10.0.1 - hooks: - - id: shellcheck - exclude: build_tools/credential-helper # This is an autogenerated file - - - repo: local + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v3.0.0 hooks: - - id: buildifier - name: buildifier - entry: buildifier - language: golang - additional_dependencies: [ - # v7.1.2 - "github.com/bazelbuild/buildtools/buildifier@1429e15ae755a6762d0edf9198062dc6ed04408d", - ] - files: '^(.*/)?(BUILD\.bazel|BUILD|WORKSPACE|WORKSPACE\.bazel|WORKSPACE\.bzlmod|MODULE\.bazel)$|\.BUILD$|\.bzl$' - language_version: "1.16" - - id: build_file_names - name: Check Bazel file names - entry: Files should be named BUILD.bazel instead of BUILD - language: fail - files: "BUILD$" + - id: prettier + args: [--write] + types_or: [yaml, json, markdown, css, html, javascript, ts, tsx] diff --git a/README.md b/README.md index 8601d36..bbc55ff 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,8 @@ Next, run the script to generate the feeds, with the optional inclusion of a tok ``` sh python3 generate_feeds.py \ --archive_dir /srv/www/petit-pois/pods \ - --base_url http://yourdomain.com/pods + --base_url http://pods.yourdomain.com/ \ + --map_file /etc/nginx/podcast_tokens.map ``` Now, each podcast will have a `archive.xml` file in its directory. @@ -120,7 +121,7 @@ server { } - ###### πŸ” TLS CONFIG (UNCHANGED) ###### + ###### πŸ” TLS CONFIG ###### listen 443 ssl; # managed by Certbot ssl_certificate /etc/letsencrypt/live/podcasts.archive.example.com/fullchain.pem; # managed by Certbot ssl_certificate_key /etc/letsencrypt/live/podcasts.archive.example.com/privkey.pem; # managed by Certbot @@ -149,7 +150,7 @@ sudo nginx -t && sudo systemctl reload nginx Further information on Nginx and web server configuration is outwith the scope of this guide. -# Bootstrap a Podcast from Local Files (Advanced) +# Bootstrap a Podcast from Local Files In some cases you may already have local MP3 files (or partial archives) and want to generate a valid podcast feed without downloading from an RSS source. For this, use `bootstrap_local_podcast.py`. diff --git a/generate_feeds.py b/generate_feeds.py index 9d76fd0..7c35455 100644 --- a/generate_feeds.py +++ b/generate_feeds.py @@ -150,21 +150,59 @@ def generate_rss_for_podcast( rss_path = os.path.join(podcast_dir, "archive.xml") tree = ET.ElementTree(rss) tree.write(rss_path, encoding="utf-8", xml_declaration=True) + # With the nginx config, only /secure// is reachable. + # If there's no token, the feed won't be accessible over HTTP. + url_prefix = f"{base_url}/secure/{token}" if token else None + feed_url = f"{url_prefix}/archive.xml" print(f"βœ… Generated RSS: {rss_path}") + print("token",token) + return feed_url -def generate_all_feeds(archive_root: str, base_url: str, map_file: str = None): + +def generate_all_feeds(archive_root: str, base_url: str, map_file: os.PathLike = None): + if map_file is None: + print("⚠️ No token map file provided. RSS feeds will not be obfuscated.") token_map = load_token_map(map_file) if map_file else {} - for dir_name in os.listdir(archive_root): + print("token_map", token_map, map_file) + results = [] + + for dir_name in sorted(os.listdir(archive_root)): dir_path = os.path.join(archive_root, dir_name) - if os.path.isdir(dir_path): - token = next( - (t for t, folder in token_map.items() if folder == dir_name), None - ) - if not token: - print(f"⚠️ No token found for {dir_name}.") - title = dir_name.replace("_", " ") - generate_rss_for_podcast(dir_path, title, base_url, token) + if not os.path.isdir(dir_path): + continue + + token = next( + (t for t, folder in token_map.items() if folder == dir_name), None + ) + + title = dir_name.replace("_", " ") + feed_url = generate_rss_for_podcast( + dir_path, title, base_url, token + ) + + if feed_url: + results.append((title, feed_url)) + + # ---- Print table at the end ---- + if results: + title_width = max(len(r[0]) for r in results) + url_width = max(len(r[1]) for r in results) + + print("\n" + "─" * (title_width + url_width + 5)) + print("Generated Podcast Feeds") + print("─" * (title_width + url_width + 5)) + print( + f"{'Podcast'.ljust(title_width)} {'Feed URL'.ljust(url_width)}" + ) + print( + f"{'-' * title_width} {'-' * url_width}" + ) + + for title, url in results: + print(f"{title.ljust(title_width)} {url}") + + print() if __name__ == "__main__": @@ -181,7 +219,7 @@ def generate_all_feeds(archive_root: str, base_url: str, map_file: str = None): ) parser.add_argument( "--map_file", - help="Optional path to token map (JSON format). Enables secure token-based URLs.", + help="Optional path to token map (nginx-formatted map). Enables secureish token-based URLs.", ) args = parser.parse_args() From 0b2543e17757b02c1eaf30f622b8558588565114 Mon Sep 17 00:00:00 2001 From: Perry Gibson Date: Sat, 13 Dec 2025 15:04:09 +0000 Subject: [PATCH 2/2] pre-commit: apply --- README.md | 26 ++++++++++++-------------- generate_feeds.py | 19 +++++-------------- 2 files changed, 17 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index bbc55ff..b6e3069 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Create a `feeds.jsonl` file with the following format: -``` sh +```sh {"url": "https://pod.url1.com/FFFFF", "name": "Podname XYZ"} {"url": "https://pod.url2.com/FFFFF", "name": "Podname ABC"} {"url": "https://pod.url3.com/FFFFF", "name": "Podname DEF"} @@ -25,19 +25,19 @@ Create a `feeds.jsonl` file with the following format: ### Download podcasts -Next, run the script to download all the missing episodes and metadata. By default it will be stored under the `pods` directory, with one sub-directory per podcast. +Next, run the script to download all the missing episodes and metadata. By default it will be stored under the `pods` directory, with one sub-directory per podcast. We recommend if you plan on serving this over the web to use a different directory, such as `/srv/www/petit-pois/pods`. -``` sh +```sh python3 download_podcasts.py \ --archive_dir /srv/www/petit-pois/pods ``` ### Generate podcast feed tokens (optional) -Again, if you're interested serving, we don't want expose the podcast to just anyone, so we need to create a token for each podcast. This is done by running the `generate_tokens.py` script: +Again, if you're interested serving, we don't want expose the podcast to just anyone, so we need to create a token for each podcast. This is done by running the `generate_tokens.py` script: -``` sh +```sh sudo python3 generate_token_map.py \ --archive_dir /srv/www/petit-pois/pods \ --map_file /etc/nginx/podcast_tokens.map @@ -47,7 +47,7 @@ sudo python3 generate_token_map.py \ Next, run the script to generate the feeds, with the optional inclusion of a token map file: -``` sh +```sh python3 generate_feeds.py \ --archive_dir /srv/www/petit-pois/pods \ --base_url http://pods.yourdomain.com/ \ @@ -56,7 +56,7 @@ python3 generate_feeds.py \ Now, each podcast will have a `archive.xml` file in its directory. -If you want to serve the files using a web-server, there are a few options. The next section gives an example using Nginx. +If you want to serve the files using a web-server, there are a few options. The next section gives an example using Nginx. ## ⚠️ Disclaimer @@ -74,14 +74,13 @@ Before archiving or sharing anything, it’s a good idea to: Install Nginx: -``` sh +```sh sudo apt update && sudo apt install nginx ``` Create a config file (e.g., `/etc/nginx/sites-available/petit-pois`): - -``` sh +```sh map $secure_token $podcast_dir { default ""; include /etc/nginx/podcast_tokens.map; @@ -143,7 +142,7 @@ server { Enable the site and restart nginx: -``` sh +```sh sudo ln -s /etc/nginx/sites-available/petit-pois /etc/nginx/sites-enabled/ sudo nginx -t && sudo systemctl reload nginx ``` @@ -166,14 +165,13 @@ Use `bootstrap_local_podcast.py` if: - The original feed no longer exists - You want a complete historical feed, even with gaps - ### Input: Episode Metadata JSONL The bootstrap script consumes a JSONL (JSON-per-line) file describing episodes. Each line represents one episode, with the following format: -``` sh +```sh {"episode":"Episode Title","date":"YYYY-MM-DD","file":"audio_file.mp3"} ``` @@ -184,7 +182,7 @@ When an episode has `"file": null`, this means the episode is missing. You can then run: -``` sh +```sh python3 bootstrap_local_podcast.py \ --jsonl metadata.jsonl \ --podcast_dir path/to/archive diff --git a/generate_feeds.py b/generate_feeds.py index 7c35455..37a36aa 100644 --- a/generate_feeds.py +++ b/generate_feeds.py @@ -155,11 +155,10 @@ def generate_rss_for_podcast( url_prefix = f"{base_url}/secure/{token}" if token else None feed_url = f"{url_prefix}/archive.xml" print(f"βœ… Generated RSS: {rss_path}") - print("token",token) + print("token", token) return feed_url - def generate_all_feeds(archive_root: str, base_url: str, map_file: os.PathLike = None): if map_file is None: print("⚠️ No token map file provided. RSS feeds will not be obfuscated.") @@ -172,14 +171,10 @@ def generate_all_feeds(archive_root: str, base_url: str, map_file: os.PathLike = if not os.path.isdir(dir_path): continue - token = next( - (t for t, folder in token_map.items() if folder == dir_name), None - ) + token = next((t for t, folder in token_map.items() if folder == dir_name), None) title = dir_name.replace("_", " ") - feed_url = generate_rss_for_podcast( - dir_path, title, base_url, token - ) + feed_url = generate_rss_for_podcast(dir_path, title, base_url, token) if feed_url: results.append((title, feed_url)) @@ -192,12 +187,8 @@ def generate_all_feeds(archive_root: str, base_url: str, map_file: os.PathLike = print("\n" + "─" * (title_width + url_width + 5)) print("Generated Podcast Feeds") print("─" * (title_width + url_width + 5)) - print( - f"{'Podcast'.ljust(title_width)} {'Feed URL'.ljust(url_width)}" - ) - print( - f"{'-' * title_width} {'-' * url_width}" - ) + print(f"{'Podcast'.ljust(title_width)} {'Feed URL'.ljust(url_width)}") + print(f"{'-' * title_width} {'-' * url_width}") for title, url in results: print(f"{title.ljust(title_width)} {url}")