Skip to content

feat(rewrite): add OpenGraph and Twitter Card preview rules #4295

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ChrisJr404 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(rewrite): add OpenGraph and Twitter Card preview rules #4295

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 13 additions & 11 deletions internal/reader/processor/processor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, userID int64,
}

webpageBaseURL := ""
var scrapedMetadata map[string]string
entry.URL = rewrite.RewriteEntryURL(feed, entry)
entryIsNew := store.IsNewEntry(feed.ID, entry.Hash)
contentExtractedSuccessfully := false
Expand All @@ -108,15 +109,16 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, userID int64,

startTime := time.Now()

scrapedPageBaseURL, extractedContent, scraperErr := scraper.ScrapeWebsite(
scrapeResult, scraperErr := scraper.ScrapeWebsite(
requestBuilder,
entry.URL,
feed.ScraperRules,
)

if scrapedPageBaseURL != "" {
webpageBaseURL = scrapedPageBaseURL
if scrapeResult.BaseURL != "" {
webpageBaseURL = scrapeResult.BaseURL
}
scrapedMetadata = scrapeResult.Metadata

if config.Opts.HasMetricsCollector() {
status := metric.StatusSuccess
Expand All @@ -134,14 +136,14 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, userID int64,
slog.String("feed_url", feed.FeedURL),
slog.Any("error", scraperErr),
)
} else if extractedContent != "" {
} else if scrapeResult.ExtractedContent != "" {
// We replace the entry content only if the scraper doesn't return any error.
entry.Content = minifyContent(extractedContent)
entry.Content = minifyContent(scrapeResult.ExtractedContent)
contentExtractedSuccessfully = true
}
}

rewrite.ApplyContentRewriteRules(entry, feed.RewriteRules)
rewrite.ApplyContentRewriteRules(entry, feed.RewriteRules, &rewrite.RewriteContext{Metadata: scrapedMetadata})

// Re-run filters only when extracted content replaced entry.Content.
if contentExtractedSuccessfully && filter.IsBlockedEntry(blockRules, allowRules, feed, entry) {
Expand Down Expand Up @@ -192,7 +194,7 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User)
requestBuilder.IgnoreTLSErrors(feed.AllowSelfSignedCertificates)
requestBuilder.DisableHTTP2(feed.DisableHTTP2)

webpageBaseURL, extractedContent, scraperErr := scraper.ScrapeWebsite(
scrapeResult, scraperErr := scraper.ScrapeWebsite(
requestBuilder,
entry.URL,
feed.ScraperRules,
Expand All @@ -210,15 +212,15 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User)
return scraperErr
}

if extractedContent != "" {
entry.Content = minifyContent(extractedContent)
if scrapeResult.ExtractedContent != "" {
entry.Content = minifyContent(scrapeResult.ExtractedContent)
if user.ShowReadingTime {
entry.ReadingTime = readingtime.EstimateReadingTime(entry.Content, user.DefaultReadingSpeed, user.CJKReadingSpeed)
}
}

rewrite.ApplyContentRewriteRules(entry, entry.Feed.RewriteRules)
entry.Content = sanitizer.SanitizeHTML(webpageBaseURL, entry.Content, &sanitizer.SanitizerOptions{OpenLinksInNewTab: user.OpenExternalLinksInNewTab})
rewrite.ApplyContentRewriteRules(entry, entry.Feed.RewriteRules, &rewrite.RewriteContext{Metadata: scrapeResult.Metadata})
entry.Content = sanitizer.SanitizeHTML(scrapeResult.BaseURL, entry.Content, &sanitizer.SanitizerOptions{OpenLinksInNewTab: user.OpenExternalLinksInNewTab})

return nil
}
46 changes: 43 additions & 3 deletions internal/reader/rewrite/content_rewrite.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,27 @@ import (
"miniflux.app/v2/internal/urllib"
)

// RewriteContext carries optional inputs collected by earlier stages of the
// reader pipeline. It is passed to rewrite rules that need more than the
// entry content alone (for example, OpenGraph values extracted from the
// page's <head>).
//
// All fields are optional; rules that depend on them must degrade gracefully
// when the value is missing.
type RewriteContext struct {
// Metadata holds key/value pairs collected from <meta> tags in the
// scraped page. Keys keep their original prefix ("og:", "twitter:") and
// are normalized to lowercase. The map can be nil when no scraping took
// place (for example, when the feed crawler is disabled).
Metadata map[string]string
}

type rule struct {
name string
args []string
}

func (rule rule) applyRule(entryURL string, entry *model.Entry) {
func (rule rule) applyRule(entryURL string, entry *model.Entry, ctx *RewriteContext) {
switch rule.name {
case "add_image_title":
entry.Content = addImageTitle(entry.Content)
Expand Down Expand Up @@ -96,10 +111,35 @@ func (rule rule) applyRule(entryURL string, entry *model.Entry) {
entry.Content = fixGhostCards(entry.Content)
case "remove_img_blur_params":
entry.Content = removeImgBlurParams(entry.Content)
case "add_open_graph":
// Format: add_open_graph("description","image",...)
// Without arguments, falls back to the most useful defaults.
properties := rule.args
if len(properties) == 0 {
properties = defaultOpenGraphProperties
}
entry.Content = addMetaPreview(entry.Content, ctx.metadataLookup(), "og:", properties)
case "add_twitter_card":
// Format: add_twitter_card("description","image",...)
properties := rule.args
if len(properties) == 0 {
properties = defaultTwitterCardProperties
}
entry.Content = addMetaPreview(entry.Content, ctx.metadataLookup(), "twitter:", properties)
}
}

// metadataLookup returns the metadata map for the current rewrite context,
// returning an empty (but non-nil) map when the context or its Metadata field
// is nil. This keeps rule implementations free of nil checks.
func (ctx *RewriteContext) metadataLookup() map[string]string {
if ctx == nil || ctx.Metadata == nil {
return map[string]string{}
}
return ctx.Metadata
}

func ApplyContentRewriteRules(entry *model.Entry, customRewriteRules string) {
func ApplyContentRewriteRules(entry *model.Entry, customRewriteRules string, ctx *RewriteContext) {
rulesList := getPredefinedRewriteRules(entry.URL)
if customRewriteRules != "" {
rulesList = customRewriteRules
Expand All @@ -114,7 +154,7 @@ func ApplyContentRewriteRules(entry *model.Entry, customRewriteRules string) {
)

for _, rule := range rules {
rule.applyRule(entry.URL, entry)
rule.applyRule(entry.URL, entry, ctx)
}
}

Expand Down
116 changes: 116 additions & 0 deletions internal/reader/rewrite/content_rewrite_functions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,14 @@ var (
textLinkRegex = regexp.MustCompile(`(?mi)(\bhttps?:\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])`)
)

// defaultOpenGraphProperties is the suffix list applied when add_open_graph
// is invoked without explicit arguments.
var defaultOpenGraphProperties = []string{"description", "image"}

// defaultTwitterCardProperties is the suffix list applied when add_twitter_card
// is invoked without explicit arguments.
var defaultTwitterCardProperties = []string{"description", "image"}

// titlelize returns a copy of the string s with all Unicode letters that begin words
// mapped to their Unicode title case.
func titlelize(s string) string {
Expand Down Expand Up @@ -550,6 +558,114 @@ func fixGhostCards(entryContent string) string {
return strings.TrimSpace(output)
}

// addMetaPreview prepends a small block of preview content built from the
// supplied metadata map. The keyPrefix selects which family of values to read
// ("og:" for OpenGraph, "twitter:" for Twitter Cards) and properties is the
// list of suffixes to render ("description", "image", "title", ...).
//
// The block is rendered as a <figure> when an image suffix is requested and
// the metadata contains a value for it; otherwise a single <p> tag is
// emitted. When no requested property has a value the original content is
// returned unchanged so the rule is a no-op.
func addMetaPreview(entryContent string, metadata map[string]string, keyPrefix string, properties []string) string {
if len(metadata) == 0 || len(properties) == 0 {
return entryContent
}

var (
imageURL string
imageAlt string
title string
description string
extraLines []string
)

for _, prop := range properties {
prop = strings.TrimSpace(strings.ToLower(prop))
if prop == "" {
continue
}
// Allow callers to pass either the bare suffix ("description") or
// the fully-qualified key ("og:description"). When a fully-qualified
// key is given, only honor it if the prefix matches the rule family.
key := prop
if strings.HasPrefix(prop, "og:") || strings.HasPrefix(prop, "twitter:") {
if !strings.HasPrefix(prop, keyPrefix) {
continue
}
} else {
key = keyPrefix + prop
}

value, ok := metadata[key]
if !ok || value == "" {
continue
}

shortKey := strings.TrimPrefix(key, keyPrefix)
switch shortKey {
case "image", "image:url", "image:secure_url":
if imageURL == "" {
imageURL = value
}
case "image:alt":
if imageAlt == "" {
imageAlt = value
}
case "title":
if title == "" {
title = value
}
case "description":
if description == "" {
description = value
}
default:
extraLines = append(extraLines, fmt.Sprintf(
`<p><strong>%s:</strong> %s</p>`,
html.EscapeString(shortKey),
html.EscapeString(value),
))
}
}

var preview strings.Builder

if title != "" {
preview.WriteString("<p><strong>")
preview.WriteString(html.EscapeString(title))
preview.WriteString("</strong></p>")
}

if imageURL != "" {
preview.WriteString(`<figure><img src="`)
preview.WriteString(html.EscapeString(imageURL))
preview.WriteString(`" alt="`)
preview.WriteString(html.EscapeString(imageAlt))
preview.WriteString(`"/>`)
if description != "" {
preview.WriteString("<figcaption><p>")
preview.WriteString(html.EscapeString(description))
preview.WriteString("</p></figcaption>")
}
preview.WriteString("</figure>")
} else if description != "" {
preview.WriteString("<p>")
preview.WriteString(html.EscapeString(description))
preview.WriteString("</p>")
}

for _, line := range extraLines {
preview.WriteString(line)
}

if preview.Len() == 0 {
return entryContent
}

return preview.String() + entryContent
}

func removeImgBlurParams(entryContent string) string {
doc, err := goquery.NewDocumentFromReader(strings.NewReader(entryContent))
if err != nil {
Expand Down
Loading