Skip to content

Commit c134054

Browse files
bzgbzg
committed
fix: Prevent silent data loss in export, ingestion and supersession
1 parent 7dd2d11 commit c134054

31 files changed

Lines changed: 1154 additions & 474 deletions

‎config.edn.example‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@
2121
;;
2222
;; Both raise a clear error at startup if the file/var is missing,
2323
;; rather than silently using nil. Mutually exclusive with :password.
24+
;;
25+
;; For IMAP servers authenticating via OAuth2 (XOAUTH2), set
26+
;; :oauth2-token instead of :password (one of the two is required):
27+
;;
28+
;; :oauth2-token "ya29...." ; OAuth2 access token
2429
:mailboxes [{:name "primary"
2530
:type :imap
2631
:host "imap.example.com"
@@ -245,18 +250,39 @@
245250

246251
;; Optional -- notifications disabled if absent.
247252
;; The :smtp map accepts :password-file or #bone/env exactly like the
248-
;; mailbox section above.
253+
;; mailbox section above. :reply-to (optional) sets the Reply-To
254+
;; header on every outgoing notification.
249255
;; :notifications {:enabled false
250256
;; :smtp {:host "smtp.example.com"
251257
;; :port 587
252258
;; :tls true
253259
;; :user "notify@example.com"
254260
;; :password "secret"
255-
;; :from "bone@example.com"}
261+
;; :from "bone@example.com"
262+
;; :reply-to "maintainers@example.com"}
256263
;; ;; Optional -- copy this address (or vector of
257264
;; ;; addresses) on every notification (subscriber
258265
;; ;; digest and direct-to-author failure mail).
259-
;; :admin-bcc "ops@example.com"}
266+
;; :admin-bcc "ops@example.com"
267+
;; ;; Who receives the digests sent by `bb notify`:
268+
;; ;; a map of subscriber email => vector of
269+
;; ;; subscriptions, one per watched source. Each
270+
;; ;; subscription map:
271+
;; ;; :source source :name to watch [required]
272+
;; ;; :min-priority 0..3 -- only filters the "unacked
273+
;; ;; & unowned" section (default 1)
274+
;; ;; :min-status 0..7 -- same section (default 0)
275+
;; ;; :subject-match case-insensitive substring on the
276+
;; ;; report subject [optional]
277+
;; ;; :topic case-insensitive substring on the
278+
;; ;; report topic [optional]
279+
;; :subscribers
280+
;; {"maint@example.com" [{:source "public-list"
281+
;; :min-priority 1}
282+
;; {:source "team-alias"
283+
;; :topic "release"}]
284+
;; "dev@example.com" [{:source "public-list"
285+
;; :subject-match "parser"}]}}
260286

261287
;; ---- Logging -------------------------------------------------------------
262288

‎scripts/bone-docs.clj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -443,7 +443,7 @@
443443
escaped (html-escape display)
444444
range (cond
445445
(and from to)
446-
(str " <small>(" (format-date-iso from) " → " (format-date-iso to) ")</small>")
446+
(str " <small>(" (format-date-iso from) " -- " (format-date-iso to) ")</small>")
447447
from
448448
(str " <small>(since " (format-date-iso from) ")</small>")
449449
to

‎scripts/bone-export.clj‎

Lines changed: 65 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -336,9 +336,16 @@
336336
;; ---------------------------------------------------------------------------
337337

338338
(defn- attachment-basename
339-
"Return the basename of an attachment filename (handles absolute paths)."
339+
"Return the basename of an attachment filename (handles absolute paths).
340+
Hostile or degenerate filenames (nil, \"\", \".\", \"..\") fall back to
341+
a stable name derived from a hash of the raw filename, so a crafted
342+
attachment can neither crash the export nor escape its directory."
340343
[att]
341-
(.getName (io/file (:attachment/filename att))))
344+
(let [filename (:attachment/filename att)
345+
base (some-> filename io/file .getName)]
346+
(if (or (nil? base) (#{"" "." ".."} base))
347+
(str "attachment-" (mid-hash (str filename)) ".txt")
348+
base)))
342349

343350
(defn- close-flag [report]
344351
(if (:report/closed report)
@@ -677,11 +684,14 @@
677684
Outgoing relations (:rel/_from) cover all kinds; incoming :related-to
678685
(:rel/_to) is added because the symmetric kind canonicalises to the
679686
smaller-eid side, so the other report only sees it via incoming."
680-
[report src-type]
687+
[report source-map]
681688
(let [self-eid (:db/id report)
689+
;; Go through archive-url so linked reports get the same
690+
;; treatment as the report itself: https-only validation of the
691+
;; sender-controlled Archived-At header and :archive-format-string
692+
;; substitution, keyed on the linked report's own source.
682693
archive (fn [other-r]
683-
(when-not (#{:alias :mailbox} src-type)
684-
(archived-at (:report/email other-r))))
694+
(archive-url other-r (:report/email other-r) source-map))
685695
format-rel (fn [from-side? rel]
686696
(let [other (if from-side? (:rel/to rel) (:rel/from rel))
687697
a (archive other)
@@ -720,8 +730,7 @@
720730
from-name (or (get (ctx-author-names) (str/lower-case from))
721731
(:email/author-name email))
722732
arch (archive-url report email source-map)
723-
src-type (get-in source-map [source-name :source-type])
724-
relations (group-relations report src-type)
733+
relations (group-relations report source-map)
725734
role (sender-role from source-name source-map maintainers-map)
726735
awaiting? (awaiting-reply? report source-name source-map maintainers-map)]
727736
(-> {:type (name (:report/type report))
@@ -1566,35 +1575,40 @@
15661575

15671576
(defn dump-root-index!
15681577
"Generate public/index.html listing all exported sources.
1569-
Reads each source's reports/meta.json for summary counts."
1570-
[source-names]
1578+
Reads each source's reports/meta.json for summary counts.
1579+
Feed links honor each source's effective :export-formats so the
1580+
index never points at files the export does not produce."
1581+
[source-names source-map]
15711582
(let [rows (for [src-name source-names
15721583
:let [slug (slugify src-name)
15731584
base-dir (str "public/" slug)
15741585
meta (load-source-meta base-dir)]
15751586
:when meta]
15761587
{:name src-name
15771588
:slug slug
1589+
:formats (resolve-export-formats src-name source-map)
15781590
:total (or (:total meta) 0)
15791591
:open (or (:open-count meta) 0)
15801592
:closed (or (:closed-count meta) 0)
15811593
:list-archive (:list-archive meta)})
15821594
row-html
1583-
(fn [{:keys [name slug total open closed list-archive]}]
1584-
(str "<tr>"
1585-
"<td><a href=\"" slug "/index.html\">" (xml-escape name) "</a>"
1586-
(when list-archive
1587-
(str " <a class=\"archive\" href=\"" (xml-escape list-archive)
1588-
"\" title=\"List archive\">↗</a>"))
1589-
"</td>"
1590-
"<td class=\"num\">" open "</td>"
1591-
"<td class=\"num\">" closed "</td>"
1592-
"<td class=\"num\">" total "</td>"
1593-
"<td class=\"num feeds\">"
1594-
"<a href=\"" slug "/reports/all.xml\">RSS</a> · "
1595-
"<a href=\"" slug "/reports/all.json\">JSON</a>"
1596-
"</td>"
1597-
"</tr>\n"))
1595+
(fn [{:keys [name slug formats total open closed list-archive]}]
1596+
(let [feeds (cond-> []
1597+
(formats "rss")
1598+
(conj (str "<a href=\"" slug "/reports/all.xml\">RSS</a>"))
1599+
(formats "json")
1600+
(conj (str "<a href=\"" slug "/reports/all.json\">JSON</a>")))]
1601+
(str "<tr>"
1602+
"<td><a href=\"" slug "/index.html\">" (xml-escape name) "</a>"
1603+
(when list-archive
1604+
(str " <a class=\"archive\" href=\"" (xml-escape list-archive)
1605+
"\" title=\"List archive\">↗</a>"))
1606+
"</td>"
1607+
"<td class=\"num\">" open "</td>"
1608+
"<td class=\"num\">" closed "</td>"
1609+
"<td class=\"num\">" total "</td>"
1610+
"<td class=\"num feeds\">" (str/join " · " feeds) "</td>"
1611+
"</tr>\n")))
15981612
page
15991613
(str
16001614
"<!DOCTYPE html>\n<html lang=\"en\">\n"
@@ -1713,7 +1727,7 @@
17131727
(log/error "Invalid --min-priority:" min-priority "(must be 1, 2, or 3)")
17141728
(System/exit 1))
17151729
(when (and min-status (not (<= 1 min-status 7)))
1716-
(log/error "Invalid --min-status:" min-status "(must be 1–7)")
1730+
(log/error "Invalid --min-status:" min-status "(must be 1-7)")
17171731
(System/exit 1))
17181732
(let [;; Watermark for save-last-export!: taken *before* the DB snapshot,
17191733
;; otherwise daemon transactions racing with this export would fall
@@ -1876,18 +1890,18 @@
18761890
(if incremental? " (incremental)" "")))
18771891
(try
18781892
(delete-dir! (io/file staging))
1879-
;; Seed staging with the previous export so an
1880-
;; incremental run keeps the files it does not
1881-
;; rewrite (per-type reports/, per-mid patches/,
1882-
;; text/, events/); aggregates are always
1883-
;; rebuilt. Seed on EVERY incremental run:
1884-
;; :since is passed whenever incremental?, so
1885-
;; unseeded per-mid files would be lost in the
1886-
;; swap.
1887-
(when incremental?
1888-
(doseq [sub ["reports" "patches" "text" "events"]]
1889-
(copy-dir! (io/file final-dir sub)
1890-
(io/file staging sub))))
1893+
;; Single-format runs: FULL copy of final-dir,
1894+
;; or the swap would wipe the other formats.
1895+
;; "all": incremental seeds only the subdirs it
1896+
;; does not rewrite; a full run starts from an
1897+
;; empty staging so stale files are purged.
1898+
(if (not= format "all")
1899+
(copy-dir! (io/file final-dir)
1900+
(io/file staging))
1901+
(when incremental?
1902+
(doseq [sub ["reports" "patches" "text" "events"]]
1903+
(copy-dir! (io/file final-dir sub)
1904+
(io/file staging sub)))))
18911905
;; Preserve the previous HTML shells when not
18921906
;; regenerating them: they are top-level files
18931907
;; not covered by the subdir seed above, and
@@ -1916,7 +1930,7 @@
19161930
;; (the :when meta filter drops never-exported sources anyway).
19171931
(when (and (#{"all" "root"} format)
19181932
(or (= format "root") (seq exported-srcs)))
1919-
(dump-root-index! (mapv :name (:sources config))))
1933+
(dump-root-index! (mapv :name (:sources config)) source-map))
19201934
;; Cron notification: one stderr line iff real work was
19211935
;; published ("Wrote ..." progress goes to stdout). Names
19221936
;; the changed report types per source (pre-filter counts);
@@ -1940,6 +1954,18 @@
19401954
s))
19411955
exported-srcs))
19421956
(when-not incremental? " [full re-export]")))))))
1943-
(save-last-export! run-started))))
1957+
;; Advance the incremental watermark only after a full,
1958+
;; unfiltered run: a partial run (single format, -n, or a
1959+
;; priority/status/topics filter) does not publish everything,
1960+
;; so moving the watermark would hide those changes from the
1961+
;; next incremental export. --closed-retention is fine: it
1962+
;; only drops old closed reports, and day-crossed? forces a
1963+
;; daily full run to keep retention output converged.
1964+
(when (and (= format "all")
1965+
(nil? source-name)
1966+
(nil? min-priority)
1967+
(nil? min-status)
1968+
(nil? topics-filter))
1969+
(save-last-export! run-started)))))
19441970
(finally
19451971
(d/close conn))))

‎scripts/bone-maintenance.clj‎

Lines changed: 39 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,14 @@
3333
;; CLI parsing
3434
;; ---------------------------------------------------------------------------
3535

36+
(defn- missing-value!
37+
"A valued option with no value must abort loudly: silently ignoring
38+
it would widen the scope of a --delete run (e.g. `--delete -n` with
39+
the source name forgotten would delete orphans from ALL sources)."
40+
[opt expected]
41+
(log/error "Missing value for" opt (str "(expected " expected ")"))
42+
(System/exit 1))
43+
3644
(defn- parse-args [args]
3745
(loop [opts {:delete? false :verbose? false :failures? false}
3846
[a & [v & r :as more]] args]
@@ -41,8 +49,12 @@
4149
(= a "--delete") (recur (assoc opts :delete? true) more)
4250
(= a "--verbose") (recur (assoc opts :verbose? true) more)
4351
(= a "--failures") (recur (assoc opts :failures? true) more)
44-
(#{"-n" "--source"} a) (if v (recur (assoc opts :source-name v) r) opts)
45-
(= a "--retention") (if v (recur (assoc opts :retention v) r) opts)
52+
(#{"-n" "--source"} a) (if v
53+
(recur (assoc opts :source-name v) r)
54+
(missing-value! a "a source name"))
55+
(= a "--retention") (if v
56+
(recur (assoc opts :retention v) r)
57+
(missing-value! a "a duration like \"90d\" or an ISO date"))
4658
:else (recur opts more))))
4759

4860
;; ---------------------------------------------------------------------------
@@ -51,8 +63,9 @@
5163

5264
(defn- report-referenced-eids
5365
"All email entity IDs reachable from any report via any ref attribute,
54-
PLUS the emails referenced by qualified relations (:rel/email), series
55-
(:series/cover-letter, :series/closed) and votes (:vote/email).
66+
PLUS the emails referenced by qualified relations (:rel/email,
67+
:rel/retracted-by), series (:series/cover-letter, :series/closed)
68+
and votes (:vote/email).
5669
If an email is pointed to by any of these, it's protected."
5770
[db]
5871
(let [via-reports (dq '[:find [?e ...]
@@ -69,7 +82,7 @@
6982
[?e :email/message-id _]]
7083
db attr))]
7184
(set (concat via-reports
72-
(mapcat via-attr [:rel/email
85+
(mapcat via-attr [:rel/email :rel/retracted-by
7386
:series/cover-letter :series/closed
7487
:vote/email])))))
7588

@@ -177,17 +190,21 @@
177190
_ (when (and source-name (not (contains? source-map source-name)))
178191
(log/error "Unknown source:" source-name)
179192
(log/error "Available:" (str/join ", " (keys source-map)))
180-
(System/exit 1))
181-
;; Validated before the connection opens: System/exit here must
182-
;; not skip the (finally (d/close conn)) below.
183-
cutoff (resolve-retention-cutoff retention)
184-
;; Open with WAL for potential writes
185-
conn (d/get-conn dbp bone-schema {})]
186-
(try
187-
(let [db (d/db conn)]
188-
(if failures?
189-
(show-failures source-name)
190-
(let [orphans (find-orphans db source-map source-name cutoff)]
193+
(System/exit 1))]
194+
(if failures?
195+
;; --failures only reads the failures EDN file -- no DB connection
196+
;; (nor retention cutoff) needed, so the daemon can keep running.
197+
(do (when retention
198+
(log/warn "--retention is ignored with --failures"))
199+
(show-failures source-name))
200+
(let [;; Validated before the connection opens: System/exit here must
201+
;; not skip the (finally (d/close conn)) below.
202+
cutoff (resolve-retention-cutoff retention)
203+
;; Open with WAL for potential writes
204+
conn (d/get-conn dbp bone-schema {})]
205+
(try
206+
(let [db (d/db conn)
207+
orphans (find-orphans db source-map source-name cutoff)]
191208
(if (empty? orphans)
192209
(log/info "No orphan emails found.")
193210
(let [by-source (group-by :source orphans)]
@@ -199,14 +216,14 @@
199216
(println (str " " source " | " mid " | " from " | " date))))
200217
(if delete?
201218
(do
202-
(log/info "Deleting" (count orphans) "orphan email(s)…")
219+
(log/info "Deleting" (count orphans) "orphan email(s)...")
203220
(let [tx-data (mapv (fn [{:keys [eid]}]
204-
[:db/retractEntity eid])
205-
orphans)]
221+
[:db/retractEntity eid])
222+
orphans)]
206223
(d/transact! conn tx-data)
207224
(log/info "Done. Deleted" (count orphans) "email(s).")))
208225
(do
209226
(log/info "Dry run -- no changes made. Pass --delete to remove.")
210-
(log/info "Tip: use --verbose to list individual orphan message-ids."))))))))
211-
(finally
212-
(d/close conn))))
227+
(log/info "Tip: use --verbose to list individual orphan message-ids."))))))
228+
(finally
229+
(d/close conn))))))

‎scripts/bone-notify.clj‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,12 @@
361361
(let [db (d/db conn)
362362
src-map (build-source-map config)
363363
reports (all-reports db)
364+
;; Watermark candidate, captured BEFORE reading the
365+
;; failures file: a failure written by the daemon
366+
;; between the read and the send may be shown twice on
367+
;; the next run, but is never lost (storing the send
368+
;; time instead would skip it forever).
369+
run-started (java.util.Date.)
364370
all-failures (load-failures)
365371
pairs (for [[email subs] subscribers
366372
s subs]
@@ -386,7 +392,7 @@
386392
(when-not dry-run?
387393
(try
388394
(send-notification! smtp email source body admin-bcc)
389-
(swap! updated-shown assoc k (.getTime (java.util.Date.)))
395+
(swap! updated-shown assoc k (.getTime run-started))
390396
(swap! sent inc)
391397
(catch Exception e
392398
;; Don't advance the timestamp on failure so the

0 commit comments

Comments
 (0)