Generate files concurrently (#54)

Author: jasonvarga

README.md

@@ -38,6 +38,15 @@ php please ssg:generate
 
 Your site will be generated into a directory which you can deploy however you like. See [Deployment Examples](#deployment-examples) below for inspiration.
 
+### Multiple Workers
+
+For improved performance, you may spread the page generation across multiple workers. This requires Spatie's [Fork](https://github.com/spatie/fork) package. Then you may specify how many workers are to be used. You can use as many workers as you have CPU cores.
+
+```
+composer require spatie/fork
+php please ssg:generate --workers=4
+```
+
 
 ## Routes
 

composer.json

@@ -29,5 +29,8 @@
     "require-dev": {
         "orchestra/testbench": "^4.0"
     },
+    "suggest": {
+        "spatie/fork": "Required to generate pages concurrently (^0.0.4)."
+    },
     "minimum-stability": "dev"
 }

src/Commands/StaticSiteGenerate.php

@@ -21,7 +21,7 @@ class StaticSiteGenerate extends Command
      *
      * @var string
      */
-    protected $signature = 'statamic:ssg:generate';
+    protected $signature = 'statamic:ssg:generate {--workers=1}';
 
     /**
      * The console command description.
@@ -51,6 +51,8 @@ public function handle()
     {
         Partyline::bind($this);
 
-        $this->generator->generate();
+        $this->generator
+            ->workers($this->option('workers'))
+            ->generate();
     }
 }

src/ConcurrentTasks.php

@@ -0,0 +1,20 @@
+<?php
+
+namespace Statamic\StaticSite;
+
+use Spatie\Fork\Fork;
+
+class ConcurrentTasks implements Tasks
+{
+    protected $fork;
+
+    public function __construct(Fork $fork)
+    {
+        $this->fork = $fork;
+    }
+
+    public function run(...$closures)
+    {
+        return $this->fork->run(...$closures);
+    }
+}

src/ConsecutiveTasks.php

@@ -0,0 +1,17 @@
+<?php
+
+namespace Statamic\StaticSite;
+
+class ConsecutiveTasks implements Tasks
+{
+    public function run(...$closures)
+    {
+        $results = [];
+
+        foreach ($closures as $closure) {
+            $results[] = $closure();
+        }
+
+        return $results;
+    }
+}

src/Generator.php

@@ -2,6 +2,7 @@
 
 namespace Statamic\StaticSite;
 
+use Spatie\Fork\Fork;
 use Facades\Statamic\View\Cascade;
 use Statamic\Facades\URL;
 use Statamic\Support\Str;
@@ -26,6 +27,7 @@ class Generator
     protected $app;
     protected $files;
     protected $router;
+    protected $tasks;
     protected $config;
     protected $request;
     protected $after;
@@ -34,12 +36,14 @@ class Generator
     protected $warnings = 0;
     protected $viewPaths;
     protected $extraUrls;
+    protected $workers = 1;
 
-    public function __construct(Application $app, Filesystem $files, Router $router)
+    public function __construct(Application $app, Filesystem $files, Router $router, Tasks $tasks)
     {
         $this->app = $app;
         $this->files = $files;
         $this->router = $router;
+        $this->tasks = $tasks;
         $this->extraUrls = collect();
         $this->config = $this->initializeConfig();
     }
@@ -55,6 +59,13 @@ private function initializeConfig()
         return $config;
     }
 
+    public function workers(int $workers)
+    {
+        $this->workers = $workers;
+
+        return $this;
+    }
+
     public function after($after)
     {
         $this->after = $after;
@@ -69,6 +80,8 @@ public function addUrls($closure)
 
     public function generate()
     {
+        $this->checkConcurrencySupport();
+
         Site::setCurrent(Site::default()->handle());
 
         $this
@@ -134,7 +147,7 @@ public function createSymlinks()
                 Partyline::line("Symlink not created. $dest already exists.");
             } else {
                 $this->files->link($source, $dest);
-                Partyline::line("$source symlinked to $dest");
+                Partyline::line("<info>[✔]</info> $source symlinked to $dest");
             }
         }
 
@@ -152,7 +165,7 @@ public function copyFiles()
                 $this->files->copyDirectory($source, $dest);
             }
 
-            Partyline::line("$source copied to to $dest");
+            Partyline::line("<info>[✔]</info> $source copied to $dest");
         }
     }
 
@@ -163,33 +176,28 @@ protected function createContentFiles()
             $this->app->instance('request', $request);
         });
 
-        $this->pages()->each(function ($page) use ($request) {
-            $this->updateCurrentSite($page->site());
+        $pages = $this->gatherContent();
 
-            view()->getFinder()->setPaths($this->viewPaths);
+        Partyline::line("Generating {$pages->count()} content files...");
 
-            $this->count++;
+        $closures = $this->makeContentGenerationClosures($pages, $request);
 
-            $request->setPage($page);
+        $results = $this->tasks->run(...$closures);
 
-            Partyline::comment("Generating {$page->url()}...");
+        $this->outputResults($results);
 
-            try {
-                $generated = $page->generate($request);
-            } catch (NotGeneratedException $e) {
-                $this->skips++;
-                Partyline::line($e->consoleMessage());
-                return;
-            }
+        return $this;
+    }
 
-            if ($generated->hasWarning()) {
-                $this->warnings++;
-            }
+    protected function gatherContent()
+    {
+        Partyline::line('Gathering content to be generated...');
 
-            Partyline::line($generated->consoleMessage());
-        });
+        $pages = $this->pages();
 
-        return $this;
+        Partyline::line("\x1B[1A\x1B[2K<info>[✔]</info> Gathered content to be generated");
+
+        return $pages;
     }
 
     protected function pages()
@@ -215,6 +223,57 @@ protected function pages()
             });
     }
 
+    protected function makeContentGenerationClosures($pages, $request)
+    {
+        return $pages->split($this->workers)->map(function ($pages) use ($request) {
+            return function () use ($pages, $request) {
+                $count = $skips = $warnings = 0;
+                $errors = [];
+
+                foreach ($pages as $page) {
+                    $this->updateCurrentSite($page->site());
+
+                    view()->getFinder()->setPaths($this->viewPaths);
+
+                    $count++;
+
+                    $request->setPage($page);
+
+                    Partyline::line("\x1B[1A\x1B[2KGenerating ".$page->url());
+
+                    try {
+                        $generated = $page->generate($request);
+                    } catch (NotGeneratedException $e) {
+                        $skips++;
+                        $errors[] = $e->consoleMessage();
+                        continue;
+                    }
+
+                    if ($generated->hasWarning()) {
+                        $warnings++;
+                    }
+                }
+
+                return compact('count', 'skips', 'warnings', 'errors');
+            };
+        })->all();
+    }
+
+    protected function outputResults($results)
+    {
+        $results = collect($results);
+
+        Partyline::line("\x1B[1A\x1B[2K<info>[✔]</info> Generated {$results->sum('count')} content files");
+
+        if ($results->sum('skips')) {
+            $results->reduce(function ($carry, $item) {
+                return $carry->merge($item['errors']);
+            }, collect())->each(function ($error) {
+                Partyline::line($error);
+            });
+        }
+    }
+
     protected function entries()
     {
         return Entry::all()
@@ -301,4 +360,13 @@ protected function updateCurrentSite($site)
         setlocale(LC_TIME, $site->locale());
         app()->setLocale($site->shortLocale());
     }
+
+    protected function checkConcurrencySupport()
+    {
+        if ($this->workers === 1 || class_exists(Fork::class)) {
+            return;
+        }
+
+        throw new \RuntimeException('To use multiple workers, you must install PHP 8 and spatie/fork.');
+    }
 }

src/NotGeneratedException.php

@@ -43,6 +43,6 @@ public function consoleMessage()
                 $message = $this->getMessage();
         }
 
-        return sprintf('%s %s (%s)', "\x1B[1A\x1B[2K<fg=red>[✘]</>", $this->getPage()->url(), $message);
+        return sprintf('%s %s (%s)', "<fg=red>[✘]</>", $this->getPage()->url(), $message);
     }
 }

src/ServiceProvider.php

@@ -2,15 +2,22 @@
 
 namespace Statamic\StaticSite;
 
+use Spatie\Fork\Fork;
 use Statamic\StaticSite\Generator;
 use Illuminate\Support\ServiceProvider as LaravelServiceProvider;
 
 class ServiceProvider extends LaravelServiceProvider
 {
     public function register()
     {
+        $this->app->bind(Tasks::class, function () {
+            return class_exists(Fork::class)
+                ? new ConcurrentTasks(new Fork)
+                : new ConsecutiveTasks;
+        });
+
         $this->app->singleton(Generator::class, function ($app) {
-            return new Generator($app, $app['files'], $app['router']);
+            return new Generator($app, $app['files'], $app['router'], $app[Tasks::class]);
         });
     }
 

src/Tasks.php

@@ -0,0 +1,8 @@
+<?php
+
+namespace Statamic\StaticSite;
+
+interface Tasks
+{
+    public function run(...$closures);
+}

tests/ConcurrentTasksTest.php

@@ -0,0 +1,28 @@
+<?php
+
+namespace Tests;
+
+use Spatie\Fork\Fork;
+use Statamic\StaticSite\ConcurrentTasks;
+
+class ConcurrentTasksTest extends TestCase
+{
+    /** @test */
+    public function it_runs_callbacks()
+    {
+        $one = function () {
+            return 'one';
+        };
+
+        $two = function () {
+            return 'two';
+        };
+
+        $fork = $this->mock(Fork::class);
+        $fork->shouldReceive('run')->once()->with($one, $two)->andReturn([$one(), $two()]);
+
+        $results = (new ConcurrentTasks($fork))->run($one, $two);
+
+        $this->assertEquals(['one', 'two'], $results);
+    }
+}

tests/ConsecutiveTasksTest.php

@@ -0,0 +1,29 @@
+<?php
+
+namespace Tests;
+
+use Statamic\StaticSite\ConsecutiveTasks;
+
+class ConsecutiveTasksTest extends TestCase
+{
+    /** @test */
+    public function it_runs_callbacks()
+    {
+        $callbacksRan = 0;
+
+        $one = function () use (&$callbacksRan) {
+            $callbacksRan++;
+            return 'one';
+        };
+
+        $two = function () use (&$callbacksRan) {
+            $callbacksRan++;
+            return 'two';
+        };
+
+        $results = (new ConsecutiveTasks)->run($one, $two);
+
+        $this->assertEquals(['one', 'two'], $results);
+        $this->assertEquals(2, $callbacksRan);
+    }
+}