Skip to content

Commit edc7704

Browse files
jpurnelljpurnell
committed
Add search index foundations: TextProcessor, SearchEntry, SearchIndexGenerator
Introduces standalone search indexing types for build-time generation of client-side search indices: - TextProcessor: HTML stripping, excerpt generation, tokenization with Porter stemming and stop-word removal - SearchEntry: Codable model for indexed content - SearchConfiguration: site-level search settings - SearchField/SearchLanguage: configuration enums - SearchIndexGenerator: produces JSON index from Article collections Purely additive. SearchBar component and pipeline integration are planned as follow-ups.
1 parent 64fb3ff commit edc7704

10 files changed

Lines changed: 806 additions & 0 deletions

‎Sources/Ignite/Search/SearchConfiguration.swift‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
//
2+
// SearchConfiguration.swift
3+
// Ignite
4+
// https://www.github.com/twostraws/Ignite
5+
// See LICENSE for license information.
6+
//
7+
8+
/// Configuration for Ignite's built-in search.
9+
public struct SearchConfiguration: Sendable {
10+
/// Whether search index generation is enabled.
11+
public var enabled: Bool
12+
13+
/// Which content types to include in the search index.
14+
/// Nil means index all published content.
15+
public var contentTypes: [String]?
16+
17+
/// Maximum length of content excerpts stored in the index (in characters).
18+
public var excerptLength: Int
19+
20+
/// Fields to include in the search index.
21+
public var indexedFields: Set<SearchField>
22+
23+
/// Whether to apply stemming to indexed content.
24+
public var enableStemming: Bool
25+
26+
/// Whether to remove common stop words from the index.
27+
public var removeStopWords: Bool
28+
29+
/// The language for stemming and stop-word lists.
30+
public var language: SearchLanguage
31+
32+
/// Output path for the search index JSON file.
33+
public var indexPath: String
34+
35+
/// Output path for the search JavaScript file.
36+
public var scriptPath: String
37+
38+
/// Default configuration with sensible values.
39+
public static let `default` = SearchConfiguration(
40+
enabled: true,
41+
contentTypes: nil,
42+
excerptLength: 300,
43+
indexedFields: [.title, .description, .tags, .content, .author],
44+
enableStemming: true,
45+
removeStopWords: true,
46+
language: .english,
47+
indexPath: "/search-index.json",
48+
scriptPath: "/js/ignite-search.js"
49+
)
50+
51+
/// Disabled configuration.
52+
public static let disabled = SearchConfiguration(
53+
enabled: false,
54+
contentTypes: nil,
55+
excerptLength: 0,
56+
indexedFields: [],
57+
enableStemming: false,
58+
removeStopWords: false,
59+
language: .english,
60+
indexPath: "",
61+
scriptPath: ""
62+
)
63+
}

‎Sources/Ignite/Search/SearchEntry.swift‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
//
2+
// SearchEntry.swift
3+
// Ignite
4+
// https://www.github.com/twostraws/Ignite
5+
// See LICENSE for license information.
6+
//
7+
8+
import Foundation
9+
10+
/// A single entry in the search index, representing one piece of content.
11+
public struct SearchEntry: Codable, Sendable {
12+
public let title: String
13+
public let description: String
14+
public let url: String
15+
public let tags: [String]
16+
public let author: String
17+
public let excerpt: String
18+
public let date: String
19+
public let tokens: [String]
20+
}

‎Sources/Ignite/Search/SearchField.swift‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
//
2+
// SearchField.swift
3+
// Ignite
4+
// https://www.github.com/twostraws/Ignite
5+
// See LICENSE for license information.
6+
//
7+
8+
/// Fields that can be included in the search index.
9+
public enum SearchField: String, Sendable, CaseIterable {
10+
case title
11+
case description
12+
case tags
13+
case content
14+
case author
15+
16+
/// The relative weight of this field in search ranking.
17+
var weight: Double {
18+
switch self {
19+
case .title: 3.0
20+
case .tags: 2.5
21+
case .description: 2.0
22+
case .author: 1.5
23+
case .content: 1.0
24+
}
25+
}
26+
}

‎Sources/Ignite/Search/SearchIndexGenerator.swift‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
//
2+
// SearchIndexGenerator.swift
3+
// Ignite
4+
// https://www.github.com/twostraws/Ignite
5+
// See LICENSE for license information.
6+
//
7+
8+
import Foundation
9+
10+
/// Generates a search index JSON file from a collection of articles.
11+
public struct SearchIndexGenerator: Sendable {
12+
public let configuration: SearchConfiguration
13+
public let articles: [Article]
14+
15+
public init(configuration: SearchConfiguration, articles: [Article]) {
16+
self.configuration = configuration
17+
self.articles = articles
18+
}
19+
20+
/// Generates the search index as a JSON string.
21+
public func generateIndex() throws -> String {
22+
guard configuration.enabled else { return "" }
23+
24+
let entries = articles
25+
.filter { $0.isPublished }
26+
.filter { article in
27+
guard let types = configuration.contentTypes else { return true }
28+
guard let articleType = article.metadata["type"] as? String else { return false }
29+
return types.contains(articleType)
30+
}
31+
.map { buildEntry(for: $0) }
32+
33+
let encoder = JSONEncoder()
34+
encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
35+
let data = try encoder.encode(entries)
36+
return String(data: data, encoding: .utf8) ?? "[]"
37+
}
38+
39+
private func buildEntry(for article: Article) -> SearchEntry {
40+
let plainText = TextProcessor.stripHTML(article.text)
41+
let excerpt = TextProcessor.excerpt(from: plainText, length: configuration.excerptLength)
42+
let tokens = tokenize(article: article)
43+
44+
return SearchEntry(
45+
title: article.title,
46+
description: article.description,
47+
url: "/\(article.path)",
48+
tags: article.tags ?? [],
49+
author: article.author ?? "",
50+
excerpt: excerpt,
51+
date: ISO8601DateFormatter().string(from: article.date),
52+
tokens: tokens
53+
)
54+
}
55+
56+
private func tokenize(article: Article) -> [String] {
57+
var allText = ""
58+
59+
if configuration.indexedFields.contains(.title) {
60+
allText += " " + article.title
61+
}
62+
if configuration.indexedFields.contains(.description) {
63+
allText += " " + article.description
64+
}
65+
if configuration.indexedFields.contains(.tags) {
66+
allText += " " + (article.tags ?? []).joined(separator: " ")
67+
}
68+
if configuration.indexedFields.contains(.content) {
69+
allText += " " + TextProcessor.stripHTML(article.text)
70+
}
71+
if configuration.indexedFields.contains(.author) {
72+
allText += " " + (article.author ?? "")
73+
}
74+
75+
return TextProcessor.tokenize(
76+
allText,
77+
language: configuration.language,
78+
stem: configuration.enableStemming,
79+
removeStopWords: configuration.removeStopWords
80+
)
81+
}
82+
}

‎Sources/Ignite/Search/SearchLanguage.swift‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
//
2+
// SearchLanguage.swift
3+
// Ignite
4+
// https://www.github.com/twostraws/Ignite
5+
// See LICENSE for license information.
6+
//
7+
8+
/// Languages supported for stemming and stop-word removal.
9+
public enum SearchLanguage: String, Sendable {
10+
case english = "en"
11+
case spanish = "es"
12+
case french = "fr"
13+
case german = "de"
14+
15+
/// Stop words for this language.
16+
var stopWords: Set<String> {
17+
switch self {
18+
case .english:
19+
return ["a", "an", "the", "is", "are", "was", "were", "be", "been",
20+
"being", "have", "has", "had", "do", "does", "did", "will",
21+
"would", "could", "should", "may", "might", "shall", "can",
22+
"to", "of", "in", "for", "on", "with", "at", "by", "from",
23+
"as", "into", "through", "during", "before", "after", "and",
24+
"but", "or", "nor", "not", "so", "yet", "both", "either",
25+
"neither", "each", "every", "all", "any", "few", "more",
26+
"most", "other", "some", "such", "no", "only", "own",
27+
"same", "than", "too", "very", "just", "about", "above",
28+
"below", "between", "up", "down", "out", "off", "over",
29+
"under", "again", "further", "then", "once", "it", "its",
30+
"this", "that", "these", "those", "i", "me", "my", "we",
31+
"our", "you", "your", "he", "him", "his", "she", "her",
32+
"they", "them", "their", "what", "which", "who", "whom",
33+
"how", "when", "where", "why"]
34+
default:
35+
return []
36+
}
37+
}
38+
}

‎Sources/Ignite/Search/TextProcessor.swift‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
//
2+
// TextProcessor.swift
3+
// Ignite
4+
// https://www.github.com/twostraws/Ignite
5+
// See LICENSE for license information.
6+
//
7+
8+
import Foundation
9+
10+
/// Text processing utilities for search indexing.
11+
public enum TextProcessor {
12+
/// Removes all HTML tags and entities, collapsing whitespace.
13+
public static func stripHTML(_ html: String) -> String {
14+
html.replacingOccurrences(of: "<[^>]+>", with: " ", options: .regularExpression)
15+
.replacingOccurrences(of: "&[^;]+;", with: " ", options: .regularExpression)
16+
.replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression)
17+
.trimmingCharacters(in: .whitespacesAndNewlines)
18+
}
19+
20+
/// Generates a plain-text excerpt of the specified length, breaking at a word boundary.
21+
public static func excerpt(from text: String, length: Int) -> String {
22+
guard text.count > length else { return text }
23+
let truncated = String(text.prefix(length))
24+
if let lastSpace = truncated.lastIndex(of: " ") {
25+
return String(truncated[truncated.startIndex..<lastSpace]) + "..."
26+
}
27+
return truncated + "..."
28+
}
29+
30+
/// Tokenizes text into individual searchable terms.
31+
public static func tokenize(
32+
_ text: String,
33+
language: SearchLanguage,
34+
stem: Bool,
35+
removeStopWords: Bool
36+
) -> [String] {
37+
let words = text
38+
.lowercased()
39+
.components(separatedBy: .alphanumerics.inverted)
40+
.filter { !$0.isEmpty }
41+
42+
var tokens = words
43+
44+
if removeStopWords {
45+
let stopWords = language.stopWords
46+
tokens = tokens.filter { !stopWords.contains($0) }
47+
}
48+
49+
if stem {
50+
tokens = tokens.map { stemWord($0, language: language) }
51+
}
52+
53+
var seen = Set<String>()
54+
tokens = tokens.filter { seen.insert($0).inserted }
55+
56+
return tokens
57+
}
58+
59+
/// Applies basic Porter stemming rules for English.
60+
public static func stemWord(_ word: String, language: SearchLanguage) -> String {
61+
guard language == .english else { return word }
62+
guard word.count > 3 else { return word }
63+
64+
var stem = word
65+
66+
let suffixes: [(String, String)] = [
67+
("ousness", "ous"), ("iveness", "ive"), ("fulness", "ful"),
68+
("ization", "ize"), ("ational", "ate"),
69+
("tional", "tion"), ("biliti", "ble"), ("iviti", "ive"),
70+
("aliti", "al"), ("ousli", "ous"), ("entli", "ent"),
71+
("alism", "al"), ("ation", "ate"), ("ator", "ate"),
72+
("anci", "ance"), ("enci", "ence"), ("izer", "ize"),
73+
("alli", "al"), ("eli", "e")
74+
]
75+
76+
for (suffix, replacement) in suffixes {
77+
if stem.hasSuffix(suffix) {
78+
return String(stem.dropLast(suffix.count)) + replacement
79+
}
80+
}
81+
82+
let simpleSuffixes = ["ing", "tion", "ness", "ment", "able", "ible",
83+
"ful", "less", "ous", "ive", "ly", "ed", "er", "es", "s"]
84+
for suffix in simpleSuffixes {
85+
if stem.hasSuffix(suffix) && stem.count - suffix.count >= 3 {
86+
stem = String(stem.dropLast(suffix.count))
87+
break
88+
}
89+
}
90+
91+
return stem
92+
}
93+
}

0 commit comments

Comments
 (0)