Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions src/Crawler/ContentProcessor/AstroProcessor.php
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ class AstroProcessor extends BaseProcessor implements ContentProcessor
Crawler::CONTENT_TYPE_ID_SCRIPT,
];

private bool $disableInlineModules = false;

/**
* @inheritDoc
*/
Expand All @@ -47,6 +49,15 @@ public function findUrls(string $content, ParsedUrl $sourceUrl): ?FoundUrls
return null;
}

/**
* Set whether to disable inlining of Astro modules for offline export
* @param bool $disable
*/
public function setDisableInlineModules(bool $disable): void
{
$this->disableInlineModules = $disable;
}

/**
* Astro will need to replace all modules with inline content due to CORS is blocking modules with file:// protocol
*
Expand All @@ -59,6 +70,11 @@ public function applyContentChangesForOfflineVersion(string &$content, int $cont
return;
}

// Skip inlining if disabled via option
if ($this->disableInlineModules) {
return;
}

// stack of already included modules to prevent duplicates
$alreadyIncludedModules = [];

Expand Down
17 changes: 17 additions & 0 deletions src/Crawler/Export/OfflineWebsiteExporter.php
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,12 @@ class OfflineWebsiteExporter extends BaseExporter implements Exporter

protected bool $ignoreStoreFileError = false;

/**
* Disable inlining of Astro module scripts for offline export
* @var bool
*/
protected bool $disableAstroInlineModules = false;

/**
* Replace HTML/JS/CSS content with `xxx -> bbb` or regexp in PREG format: `/card[0-9]/ -> card`
*
Expand Down Expand Up @@ -103,6 +109,16 @@ public function export(): void
// user-defined replaceQueryString will deactivate replacing query string with hash and use custom replacement
OfflineUrlConverter::setReplaceQueryString($this->replaceQueryString);

// configure AstroProcessor to disable inline modules if requested
if ($this->disableAstroInlineModules) {
foreach ($this->crawler->getContentProcessorManager()->getProcessors() as $processor) {
if ($processor instanceof \Crawler\ContentProcessor\AstroProcessor) {
$processor->setDisableInlineModules(true);
break;
}
}
}

// filter only relevant URLs with OK status codes
$exportedUrls = array_filter($visitedUrls, function (VisitedUrl $visitedUrl) {
return in_array($visitedUrl->statusCode, [200, 201, 301, 302, 303, 308]);
Expand Down Expand Up @@ -336,6 +352,7 @@ public static function getOptions(): Options
new Option('--replace-content', null, 'replaceContent', Type::REPLACE_CONTENT, true, "Replace HTML/JS/CSS content with `foo -> bar` or regexp in PREG format: `/card[0-9]/i -> card`", null, true, true),
new Option('--replace-query-string', null, 'replaceQueryString', Type::REPLACE_CONTENT, true, "Instead of using a short hash instead of a query string in the filename, just replace some characters. You can use simple format 'foo -> bar' or regexp in PREG format, e.g. '/([a-z]+)=([^&]*)(&|$)/i -> $1__$2'", null, true, true),
new Option('--ignore-store-file-error', null, 'ignoreStoreFileError', Type::BOOL, false, 'Ignores any file storing errors. The export process will continue.', false, false),
new Option('--disable-astro-inline-modules', null, 'disableAstroInlineModules', Type::BOOL, false, 'Disables inlining of Astro module scripts for offline export. Scripts will remain as external files with corrected relative paths.', false, false),
]));
return $options;
}
Expand Down