Add initial support for Swift Concurrency (#117)

This change adds initial support for running certain publishing operations concurrently
using Swift's new built-in concurrency system. This doesn't involve any breaking changes
to the public API, apart from now requiring all Publish users to ensure that their HTML theme
and multi-item mutation code can be executed in a concurrent manner.

The following key changes were made as part of this:

- Bump the project's Swift version to `5.5`, and update the CLI's project template accordingly.
- Add CollectionConcurrencyKit as a new package dependency, which adds concurrent
   versions of the various sequence mapping functions that Publish makes heavy use of.
- Make `PublishingStep` use an `async` closure type as its body, which in turn enables
   asynchronous calls to be made within any step, both built-in and custom ones.
- Each publishing pipeline is still performed in sequence, just like before. So it's only the
   internal operations within each step that may now be parallelized.
- The top-level `Website.publish` APIs are still completely synchronous, and uses an internal
   `DispatchSemaphore` to await the completion of the website's publishing pipeline before
   returning.
- A new built-in `mutateAllItems(in sections:)` API has been added, to make it convenient
   to iterate over all items within a certain set of sections in a concurrent manner.
- `HTMLGenerator` now executes concurrently, meaning that it can output a large number
   of HTML files in parallel, rather than writing all files one-by-one.
- `MarkdownFileHandler` now also runs concurrently, and processes all of its Markdown files
   in parallel, rather than doing so sequentially.
- `RSSFeedGenerator` and `PodcastFeedGenerator` now generates their respective feeds
   concurrently.

Overall, this set of changes results in a 4x speed improvement when generating Swift by Sundell
using the M1 Max chip.

As part of this, Publish now requires macOS 12, to be able to use Swift Concurrency in a
predictable way. macOS version 12.0 has been out for quite a while now, so it's safe to require
it at this point.

Author: JohnSundell

Package.resolved

@@ -1,4 +1,4 @@
-// swift-tools-version:5.4
+// swift-tools-version:5.5
 
 /**
 *  Publish
@@ -10,23 +10,54 @@ import PackageDescription
 
 let package = Package(
     name: "Publish",
+    platforms: [.macOS(.v12)],
     products: [
         .library(name: "Publish", targets: ["Publish"]),
         .executable(name: "publish-cli", targets: ["PublishCLI"])
     ],
     dependencies: [
-        .package(name: "Ink", url: "https://github.com/johnsundell/ink.git", from: "0.2.0"),
-        .package(name: "Plot", url: "https://github.com/johnsundell/plot.git", from: "0.9.0"),
-        .package(name: "Files", url: "https://github.com/johnsundell/files.git", from: "4.0.0"),
-        .package(name: "Codextended", url: "https://github.com/johnsundell/codextended.git", from: "0.1.0"),
-        .package(name: "ShellOut", url: "https://github.com/johnsundell/shellout.git", from: "2.3.0"),
-        .package(name: "Sweep", url: "https://github.com/johnsundell/sweep.git", from: "0.4.0")
+        .package(
+            name: "Ink",
+            url: "https://github.com/johnsundell/ink.git",
+            from: "0.2.0"
+        ),
+        .package(
+            name: "Plot",
+            url: "https://github.com/johnsundell/plot.git",
+            from: "0.9.0"
+        ),
+        .package(
+            name: "Files",
+            url: "https://github.com/johnsundell/files.git",
+            from: "4.0.0"
+        ),
+        .package(
+            name: "Codextended",
+            url: "https://github.com/johnsundell/codextended.git",
+            from: "0.1.0"
+        ),
+        .package(
+            name: "ShellOut",
+            url: "https://github.com/johnsundell/shellout.git",
+            from: "2.3.0"
+        ),
+        .package(
+            name: "Sweep",
+            url: "https://github.com/johnsundell/sweep.git",
+            from: "0.4.0"
+        ),
+        .package(
+            name: "CollectionConcurrencyKit",
+            url: "https://github.com/johnsundell/collectionConcurrencyKit.git",
+            from: "0.1.0"
+        )
     ],
     targets: [
         .target(
             name: "Publish",
             dependencies: [
-                "Ink", "Plot", "Files", "Codextended", "ShellOut", "Sweep"
+                "Ink", "Plot", "Files", "Codextended",
+                "ShellOut", "Sweep", "CollectionConcurrencyKit"
             ]
         ),
         .executableTarget(

Package.swift

@@ -15,7 +15,7 @@ import Plot
 /// Steps are added when calling `Website.publish`.
 public struct PublishingStep<Site: Website> {
     /// Closure type used to define the main body of a publishing step.
-    public typealias Closure = (inout PublishingContext<Site>) throws -> Void
+    public typealias Closure = (inout PublishingContext<Site>) async throws -> Void
 
     internal let kind: Kind
     internal let body: Body
@@ -60,7 +60,7 @@ public extension PublishingStep {
             return step
         case .operation(let name, let closure):
             return .step(named: name, kind: step.kind) { context in
-                do { try closure(&context) }
+                do { try await closure(&context) }
                 catch {}
             }
         }
@@ -131,7 +131,7 @@ public extension PublishingStep {
     static func addMarkdownFiles(at path: Path = "Content") -> Self {
         step(named: "Add Markdown files from '\(path)' folder") { context in
             let folder = try context.folder(at: path)
-            try MarkdownFileHandler().addMarkdownFiles(in: folder, to: &context)
+            try await MarkdownFileHandler().addMarkdownFiles(in: folder, to: &context)
         }
     }
 
@@ -164,21 +164,52 @@ public extension PublishingStep {
         matching predicate: Predicate<Item<Site>> = .any,
         using mutations: @escaping Mutations<Item<Site>>
     ) -> Self {
-        let nameSuffix = section.map { " in '\($0)'" } ?? ""
+        mutateAllItems(
+            in: section.map { [$0] } ?? Set(Site.SectionID.allCases),
+            matching: predicate,
+            using: mutations
+        )
+    }
 
-        return step(named: "Mutate items" + nameSuffix) { context in
-            if let section = section {
-                try context.sections[section].mutateItems(
-                    matching: predicate,
-                    using: mutations
+    /// Mutate all items matching a predicate within a certain set of sections.
+    /// - parameter sections: The sections to mutate all items within.
+    /// - parameter predicate: Any predicate to filter the items using.
+    /// - parameter mutations: The mutations to apply to each item.
+    static func mutateAllItems(
+        in sections: Set<Site.SectionID>,
+        matching predicate: Predicate<Item<Site>> = .any,
+        using mutations: @escaping Mutations<Item<Site>>
+    ) -> Self {
+        var stepName = "Mutate all items"
+        
+        if sections.count != Site.SectionID.allCases.count {
+            let sectionDescription = sections
+                .map(\.rawValue)
+                .joined(separator: ", ")
+            
+            stepName.append(" in \(sectionDescription)")
+        }
+        
+        return step(named: stepName) { context in
+            for section in sections {
+                try await context.sections[section].replaceItems(
+                    with: context.sections[section].items.concurrentMap { item in
+                        guard predicate.matches(item) else {
+                            return item
+                        }
+
+                        do {
+                            var item = item
+                            try mutations(&item)
+                            return item
+                        } catch {
+                            throw ContentError(
+                                path: item.path,
+                                reason: .itemMutationFailed(error)
+                            )
+                        }
+                    }
                 )
-            } else {
-                for section in context.sections.ids {
-                    try context.sections[section].mutateItems(
-                        matching: predicate,
-                        using: mutations
-                    )
-                }
             }
         }
     }
@@ -343,7 +374,7 @@ public extension PublishingStep {
                 context: context
             )
 
-            try generator.generate()
+            try await generator.generate()
         }
     }
 
@@ -372,7 +403,7 @@ public extension PublishingStep {
                 date: date
             )
 
-            try generator.generate()
+            try await generator.generate()
         }
     }
 
@@ -420,7 +451,7 @@ public extension PublishingStep where Site.ItemMetadata: PodcastCompatibleWebsit
                 date: date
             )
 
-            try generator.generate()
+            try await generator.generate()
         }
     }
 }

Sources/Publish/API/PublishingStep.swift

@@ -122,6 +122,11 @@ internal extension Section {
         updateLastItemModificationDateIfNeeded(to: item.date)
         items.append(item)
     }
+    
+    mutating func replaceItems(with newItems: [Item<Site>]) {
+        items = newItems
+        rebuildIndexes()
+    }
 }
 
 private extension Section {

Sources/Publish/API/Section.swift

@@ -6,6 +6,7 @@
 
 import Foundation
 import Plot
+import Dispatch
 
 /// Protocol that all `Website.SectionID` implementations must conform to.
 public protocol WebsiteSectionID: Decodable, Hashable, CaseIterable, RawRepresentable where RawValue == String {}
@@ -111,7 +112,23 @@ public extension Website {
             originFilePath: Path("\(file)")
         )
 
-        return try pipeline.execute(for: self, at: path)
+        let semaphore = DispatchSemaphore(value: 0)
+        var result: Result<PublishedWebsite<Self>, Error>?
+        let completionHandler = { result = $0 }
+        
+        Task {
+            do {
+                let website = try await pipeline.execute(for: self, at: path)
+                completionHandler(.success(website))
+            } catch {
+                completionHandler(.failure(error))
+            }
+            
+            semaphore.signal()
+        }
+        
+        semaphore.wait()
+        return try result!.get()
     }
 }
 

Sources/Publish/API/Website.swift

@@ -6,32 +6,38 @@
 
 import Plot
 import Files
+import CollectionConcurrencyKit
 
 internal struct HTMLGenerator<Site: Website> {
     let theme: Theme<Site>
     let indentation: Indentation.Kind?
     let fileMode: HTMLFileMode
     let context: PublishingContext<Site>
 
-    func generate() throws {
-        try copyThemeResources()
-        try generateIndexHTML()
-        try generateSectionHTML()
-        try generatePageHTML()
-        try generateTagHTMLIfNeeded()
+    func generate() async throws {
+        try await withThrowingTaskGroup(of: Void.self) { group in
+            group.addTask { try await copyThemeResources() }
+            group.addTask { try generateIndexHTML() }
+            group.addTask { try await generateSectionHTML() }
+            group.addTask { try await generatePageHTML() }
+            group.addTask { try await generateTagHTMLIfNeeded() }
+
+            // Throw any errors generated by the above set of operations:
+            for try await _ in group {}
+        }
     }
 }
 
 private extension HTMLGenerator {
-    func copyThemeResources() throws {
+    func copyThemeResources() async throws {
         guard !theme.resourcePaths.isEmpty else {
             return
         }
 
         let creationFile = try File(path: theme.creationPath.string)
         let packageFolder = try creationFile.resolveSwiftPackageFolder()
 
-        for path in theme.resourcePaths {
+        try await theme.resourcePaths.concurrentForEach { path in
             do {
                 let file = try packageFolder.file(at: path.string)
                 try context.copyFileToOutput(file, targetFolderPath: nil)
@@ -51,16 +57,16 @@ private extension HTMLGenerator {
         try indexFile.write(html.render(indentedBy: indentation))
     }
 
-    func generateSectionHTML() throws {
-        for section in context.sections {
+    func generateSectionHTML() async throws {
+        try await context.sections.concurrentForEach { section in
             try outputHTML(
                 for: section,
                 indentedBy: indentation,
                 using: theme.makeSectionHTML,
                 fileMode: .foldersAndIndexFiles
             )
-
-            for item in section.items {
+            
+            try await section.items.concurrentForEach { item in
                 try outputHTML(
                     for: item,
                     indentedBy: indentation,
@@ -71,8 +77,8 @@ private extension HTMLGenerator {
         }
     }
 
-    func generatePageHTML() throws {
-        for page in context.pages.values {
+    func generatePageHTML() async throws {
+        try await context.pages.values.concurrentForEach { page in
             try outputHTML(
                 for: page,
                 indentedBy: indentation,
@@ -82,7 +88,7 @@ private extension HTMLGenerator {
         }
     }
 
-    func generateTagHTMLIfNeeded() throws {
+    func generateTagHTMLIfNeeded() async throws {
         guard let config = context.site.tagHTMLConfig else {
             return
         }
@@ -99,7 +105,7 @@ private extension HTMLGenerator {
             try listFile.write(listHTML.render(indentedBy: indentation))
         }
 
-        for tag in context.allTags {
+        try await context.allTags.concurrentForEach { tag in
             let detailsPath = context.site.path(for: tag)
             let detailsContent = config.detailsContentResolver(tag)
 
@@ -110,7 +116,7 @@ private extension HTMLGenerator {
             )
 
             guard let detailsHTML = try theme.makeTagDetailsHTML(detailsPage, context) else {
-                continue
+                return
             }
 
             try outputHTML(