Generate sitemap.xml when building.

Author: adamghill

coltrane/__init__.py

@@ -140,7 +140,7 @@ def _merge_installed_apps(
 
 def _get_from_env(env_name: str) -> List[str]:
     """
-    Retrieves environment value that could potentially be an list of strings.
+    Retrieves environment value that could potentially be a list of strings.
     """
 
     env_values = []
@@ -233,6 +233,11 @@ def _merge_settings(base_dir: Path, django_settings: Dict[str, Any]) -> Dict[str
         "SETTINGS_MODULE": "coltrane",
     }
 
+    coltrane_site = getenv("COLTRANE_SITE")
+
+    if coltrane_site:
+        default_settings["COLTRANE"]["SITE"] = coltrane_site
+
     if is_whitenoise_installed:
         default_settings["WHITENOISE_MANIFEST_STRICT"] = False
         default_settings[

coltrane/config/settings.py

@@ -19,9 +19,7 @@
     "toc",
 ]
 
-DEFAULT_COLTRANE_SETTINGS = {
-    "MARKDOWN_EXTRAS": DEFAULT_MARKDOWN_EXTRAS,
-}
+DEFAULT_COLTRANE_SETTINGS = {"MARKDOWN_EXTRAS": DEFAULT_MARKDOWN_EXTRAS}
 
 
 def get_coltrane_settings() -> Dict:

coltrane/management/commands/build.py

@@ -6,9 +6,10 @@
 from multiprocessing import cpu_count
 from pathlib import Path
 from types import SimpleNamespace
-from typing import Dict
+from typing import Dict, List
 
 from django.conf import settings
+from django.contrib.sitemaps.views import sitemap
 from django.core import management
 from django.core.management.base import BaseCommand
 
@@ -22,7 +23,9 @@
     get_output_static_directory,
 )
 from coltrane.manifest import Manifest, ManifestItem
+from coltrane.renderer import StaticRequest
 from coltrane.retriever import get_content_paths
+from coltrane.urls import sitemaps
 from coltrane.utils import threadpool
 
 
@@ -65,6 +68,15 @@ def add_arguments(self, parser):
     def _load_manifest(self) -> Manifest:
         return Manifest(manifest_file=get_output_json())
 
+    def _generate_sitemap(self) -> None:
+        assert self.output_directory
+
+        sitemap_response = sitemap(
+            request=StaticRequest(path="/", META={}), sitemaps=sitemaps
+        )
+        sitemap_xml = sitemap_response.render().content.decode()
+        (self.output_directory / "sitemap.xml").write_text(sitemap_xml)
+
     @threadpool
     def _call_collectstatic(self) -> str:
         stdout = StringIO()
@@ -163,12 +175,13 @@ def _set_output_directory(self, options: Dict) -> None:
                 pass
 
     def handle(self, *args, **options):
-        self.is_force = False
-        self.manifest = None
+        self.is_force: bool = False
+        self.manifest: Manifest = None
         self.output_result_counts.create_count = 0
         self.output_result_counts.update_count = 0
         self.output_result_counts.skip_count = 0
-        self.errors = []
+        self.output_directory: Path = None
+        self.errors: List[str] = []
 
         start_time = time.time()
 
@@ -180,10 +193,10 @@ def handle(self, *args, **options):
 
         collectstatic_future = self._call_collectstatic()
 
-        output_directory = get_output_directory()
+        self.output_directory = get_output_directory()
         self._success("Use output directory of ", ending="")
-        self.stdout.write(self.style.WARNING(str(output_directory)))
-        output_directory.mkdir(exist_ok=True)
+        self.stdout.write(self.style.WARNING(str(self.output_directory)))
+        self.output_directory.mkdir(exist_ok=True)
 
         spinner.start("Load manifest")
         self.manifest = self._load_manifest()
@@ -250,6 +263,10 @@ def handle(self, *args, **options):
         spinner.succeed(result_msg)
 
         if self.manifest.is_dirty:
+            spinner.start("Update sitemap.xml")
+            self._generate_sitemap()
+            spinner.succeed()
+
             spinner.start("Update manifest")
             self.manifest.write_data()
             spinner.succeed()

coltrane/manifest.py

@@ -113,7 +113,6 @@ def render_html(self):
 
         (template, context) = render_markdown(self.slug, request)
         rendered_html = render_to_string(template, context)
-        print("rendered_htmlrendered_htmlrendered_html", rendered_html)
 
         return rendered_html
 

coltrane/renderer.py

@@ -1,7 +1,8 @@
 import logging
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Dict, Optional, Tuple, Union
 
+from django.conf import settings
 from django.http import HttpRequest
 from django.template import engines
 from django.utils.html import mark_safe  # type: ignore
@@ -29,7 +30,22 @@ class StaticRequest:
     """
 
     path: str
-    META: Dict
+    META: Dict = field(default_factory=dict)
+    GET: Dict = field(default_factory=dict)
+
+    @property
+    def site(self):
+        _site = settings.COLTRANE.get("SITE")
+        assert _site, "COLTRANE_SITE is required in .env"
+
+        return _site
+
+    @property
+    def scheme(self) -> str:
+        return self.site.split("://")[0]
+
+    def get_host(self) -> str:
+        return self.site.split("://")[1]
 
 
 def render_markdown_path(path) -> Dict[str, Optional[Dict]]:
@@ -48,6 +64,17 @@ def render_markdown_path(path) -> Dict[str, Optional[Dict]]:
 
     metadata = content.metadata
 
+    if metadata is None:
+        metadata = {}
+
+    if "date" in metadata:
+        metadata["date"] = dateparser.parse(metadata["date"])
+
+    if "draft" in metadata:
+        metadata["draft"] = metadata["draft"] == "true"
+
+    metadata["now"] = now()
+
     return (str(content), metadata)
 
 
@@ -86,14 +113,6 @@ def get_html_and_markdown(slug: str) -> Tuple[str, Dict]:
 
     metadata["slug"] = slug
 
-    if "date" in metadata:
-        metadata["date"] = dateparser.parse(metadata["date"])
-
-    if "draft" in metadata:
-        metadata["draft"] = metadata["draft"] == "true"
-
-    metadata["now"] = now
-
     return (html, metadata)
 
 

coltrane/sitemaps.py

@@ -2,6 +2,7 @@
 
 from django.contrib.sitemaps import Sitemap
 
+from coltrane.renderer import render_markdown_path
 from coltrane.retriever import get_content_paths
 
 
@@ -11,7 +12,19 @@ class ContentSitemap(Sitemap):
 
     def items(self):
         # TODO: This could get cached so it isn't re-generated every time
-        return list(get_content_paths())
+        paths = get_content_paths()
+        _items = []
+
+        for path in paths:
+            (_, metadata) = render_markdown_path(path)
+
+            if "draft" in metadata:
+                if metadata["draft"]:
+                    continue
+
+            _items.append(path)
+
+        return _items
 
     def location(self, path: Path):
         relative_url = str(path)[7:-3]

conftest.py

@@ -4,19 +4,23 @@
 from django.conf import settings
 
 from coltrane import _get_default_template_settings
+from coltrane.config.settings import DEFAULT_COLTRANE_SETTINGS
 
 
 def pytest_configure():
     base_dir = Path(".")
 
     templates = deepcopy(_get_default_template_settings(base_dir))
+    coltrane_settings = deepcopy(DEFAULT_COLTRANE_SETTINGS)
+    coltrane_settings["SITE"] = "http://localhost"
 
     settings.configure(
         BASE_DIR=base_dir,
         SECRET_KEY="this-is-a-secret",
         TEMPLATES=templates,
         ROOT_URLCONF="coltrane.urls",
         INSTALLED_APPS=[
+            "django.contrib.sitemaps",
             "django_fastdev.django_app.FastDevConfig",
             "coltrane",
         ],
@@ -27,4 +31,5 @@ def pytest_configure():
         },
         STATIC_ROOT=base_dir / "output" / "static",
         SETTINGS_MODULE="coltrane",
+        COLTRANE=coltrane_settings,
     )

tests/renderer/test_get_markdown_content_as_html.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from unittest.mock import ANY
 
 from coltrane.config.settings import DEFAULT_MARKDOWN_EXTRAS
 from coltrane.renderer import _get_markdown_content_as_html
@@ -18,7 +19,7 @@ def test_get_markdown_content_as_html_with_frontmatter(settings, tmp_path: Path)
     )
 
     rendered_html = "<p>test data</p>\n"
-    context = {"template": "test-template.html"}
+    context = {"template": "test-template.html", "now": ANY}
     expected = (rendered_html, context)
     actual = _get_markdown_content_as_html("test-1")
 
@@ -43,7 +44,7 @@ def test_get_markdown_content_as_html_extras_settings(settings, tmp_path: Path):
     )
 
     rendered_html = "<p>test data</p>\n"
-    context = {"template": "test-template.html"}
+    context = {"template": "test-template.html", "now": ANY}
     expected = (rendered_html, context)
     actual = _get_markdown_content_as_html("test-1")
 

tests/renderer/test_render_markdown.py

@@ -1,4 +1,7 @@
+from dataclasses import dataclass
 from pathlib import Path
+from typing import Dict
+from unittest.mock import patch
 
 from coltrane.renderer import StaticRequest, render_markdown
 
@@ -28,3 +31,19 @@ def test_render_markdown(settings, tmp_path: Path):
     assert actual_context.get("content") == expected_content
     assert actual_context.get("data") == expected_data
     assert actual_context.get("template") == expected_template
+
+
+@dataclass
+class MarkdownContent:
+    metadata: Dict
+
+    def __str__(self):
+        return "test-content"
+
+
+@patch("coltrane.renderer.markdown_path", return_value=MarkdownContent(metadata=None))
+def test_render_markdown_metadata(settings, tmp_path: Path):
+    static_request = StaticRequest(path="/", META={})
+
+    (_, metadata) = render_markdown("test-2", static_request)
+    assert metadata.get("content") == "test-content"

tests/renderer/test_static_request.py

@@ -0,0 +1,33 @@
+import pytest
+
+from coltrane.renderer import StaticRequest
+
+
+def test_static_request_scheme(settings):
+    settings.COLTRANE = {"SITE": "http://localhost"}
+    request = StaticRequest(path="/")
+
+    assert request.scheme == "http"
+
+
+def test_static_request_get_host(settings):
+    settings.COLTRANE = {"SITE": "http://localhost"}
+    request = StaticRequest(path="/")
+
+    assert request.get_host() == "localhost"
+
+
+def test_static_request_missing_site(settings):
+    settings.COLTRANE = {}
+    request = StaticRequest(path="/")
+
+    with pytest.raises(AssertionError):
+        request.scheme
+
+
+def test_static_request_missing_coltrane(settings):
+    delattr(settings, "COLTRANE")
+    request = StaticRequest(path="/")
+
+    with pytest.raises(AttributeError):
+        request.scheme