Added code references to search results.

djangoproject/scss/_style.scss

@@ -2635,6 +2635,38 @@ table.docutils th {
             color: var(--search-mark-text);
         }
     }
+
+    .code-links {
+        margin-top: 15px;
+        margin-left: 10px;
+
+        a {
+            &:active,
+            &:focus,
+            &:hover {
+                code {
+                    color: var(--primary);
+                }
+                .meta {
+                    color: var(--text-light);
+                }
+            }
+        }
+
+        code {
+            color: var(--primary-accent);
+            font-weight: 700;
+        }
+
+        div {
+            margin: 10px 0;
+
+            .meta {
+                margin: 5px 0 0;
+                color: var(--body-fg);
+            }
+        }
+    }
 }
 
 .list-links-small {

docs/models.py

@@ -187,6 +187,13 @@ def sync_to_db(self, decoded_documents):
             document_path = _clean_document_path(document["current_page_name"])
             document["slug"] = Path(document_path).parts[-1]
             document["parents"] = " ".join(Path(document_path).parts[:-1])
+            code_references = utils.generate_code_references(document["body"])
+            document["code_references"] = code_references
+            document["code_references_search"] = " ".join(
+                # Keeps the code suffix to improve the search results for terms such as
+                # "select" for QuerySet.select_related.
+                [key.split(".")[-1] for key in code_references.keys()]
+            )
             Document.objects.create(
                 release=self,
                 path=document_path,
@@ -278,7 +285,16 @@ def search(self, query_text, release):
                         stop_sel=STOP_SEL,
                         config=models.F("config"),
                     ),
+                    code_matched=SearchHeadline(
+                        KeyTextTransform("code_references_search", "metadata"),
+                        search_query,
+                        start_sel=START_SEL,
+                        stop_sel=STOP_SEL,
+                        config=models.F("config"),
+                        highlight_all=True,
+                    ),
                     breadcrumbs=models.F("metadata__breadcrumbs"),
+                    code_references=models.F("metadata__code_references"),
                 )
                 .only(
                     "path",

docs/templates/docs/search_results.html

@@ -48,6 +48,19 @@ <h2 class="result-title">
             {% if result.highlight %}
               …&nbsp;{{ result.highlight|cut:"¶"|safe }}&nbsp;…
             {% endif %}
+            {% code_links result.code_matched result.code_references as result_code_links %}
+            {% if result_code_links %}
+              <div class="code-links">
+                {% for name, value in result_code_links.items %}
+                  <a href="{% url 'document-detail' lang=result.release.lang version=result.release.version url=result.path host 'docs' %}#{{ value.full_path }}">
+                    <div>
+                      <code>{% if value.full_path|slice:":5" == "term-" %}{{ name|dash_to_space }}{% else %}{{ name }}{% endif %}</code>
+                      {% if value.module_path %}<div class="meta">{{ value.module_path }}</div>{% endif %}
+                    </div>
+                  </a>
+                {% endfor %}
+              </div>
+            {% endif %}
           </dd>
         {% endfor %}
       </dl>

docs/templatetags/docs.py

@@ -121,3 +121,40 @@ def generate_scroll_to_text_fragment(highlighted_text):
     # Due to Python code such as timezone.now(), remove the space after a bracket.
     single_spaced = re.sub(r"([(\[])\s", r"\1", single_spaced)
     return f"#:~:text={quote(single_spaced)}"
+
+
+def get_module_path(name, full_path):
+    if full_path.startswith(("term-", "envvar-")) or len(name) >= len(full_path):
+        return None
+    return full_path[: -len(name) - 1]
+
+
+@register.filter
+def dash_to_space(value):
+    return value.replace("-", " ")
+
+
+@register.simple_tag(name="code_links")
+def code_links(code_matched, code_references):
+    if not code_matched or START_SEL not in code_matched:
+        return {}
+    code_matches = [
+        word.replace(START_SEL, "").replace(STOP_SEL, "")
+        for word in code_matched.split(" ")
+        if START_SEL in word
+    ]
+    matched_reference = {}
+    # Map "select_related" to "QuerySet.select_related" in code_references.
+    reference_map = {key.split(".")[-1]: key for key in code_references.keys()}
+    for match in code_matches:
+        if full_path := code_references.get(match):
+            matched_reference[match] = {
+                "full_path": full_path,
+                "module_path": get_module_path(match, full_path),
+            }
+        elif name := reference_map.get(match):
+            matched_reference[name] = {
+                "full_path": code_references[name],
+                "module_path": get_module_path(name, code_references[name]),
+            }
+    return dict(sorted(matched_reference.items()))

docs/tests.py

@@ -13,14 +13,19 @@
 from django.template.loader import render_to_string
 from django.test import RequestFactory, TestCase
 from django.urls import reverse, set_urlconf
+from django_hosts.resolvers import reverse as reverse_with_host
 
 from djangoproject.urls import www as www_urls
 from releases.models import Release
 
 from .models import DOCUMENT_SEARCH_VECTOR, Document, DocumentRelease
 from .sitemaps import DocsSitemap
-from .templatetags.docs import generate_scroll_to_text_fragment, get_all_doc_versions
-from .utils import get_doc_path, sanitize_for_trigram
+from .templatetags.docs import (
+    code_links,
+    generate_scroll_to_text_fragment,
+    get_all_doc_versions,
+)
+from .utils import generate_code_references, get_doc_path, sanitize_for_trigram
 
 
 class ModelsTests(TestCase):
@@ -179,9 +184,12 @@ def test_internals_team(self):
 class SearchFormTestCase(TestCase):
     fixtures = ["doc_test_fixtures"]
 
-    def setUp(self):
+    @classmethod
+    def setUpTestData(cls):
         # We need to create an extra Site because docs have SITE_ID=2
         Site.objects.create(name="Django test", domain="example2.com")
+        cls.release = Release.objects.create(version="5.1")
+        cls.doc_release = DocumentRelease.objects.create(release=cls.release)
 
     @classmethod
     def tearDownClass(cls):
@@ -195,6 +203,70 @@ def test_empty_get(self):
         )
         self.assertEqual(response.status_code, 200)
 
+    def test_code_links(self):
+        Document.objects.create(
+            **{
+                "metadata": {
+                    "body": (
+                        "QuerySet API Reference QuerySet select_related selects related things "
+                        "select_for_update selects things for update. Plus details on a Feature release."
+                    ),
+                    "code_references": {
+                        "QuerySet": "django.db.models.query.QuerySet",
+                        "QuerySet.select_related": "django.db.models.query.QuerySet.select_related",
+                        "QuerySet.select_for_update": "django.db.models.query.QuerySet.select_for_update",
+                        "Feature-release": "term-Feature-release",
+                    },
+                    "code_references_search": "QuerySet select_related select_for_update Feature-release",
+                    "breadcrumbs": [{"path": "refs", "title": "API Reference"}],
+                    "parents": "API Reference",
+                    "slug": "query",
+                    "title": "QuerySet API Reference",
+                    "toc": '<ul>\n<li><a class="reference internal" href="#">QuerySet API Reference</a></li>\n</ul>\n',
+                },
+                "path": "refs/query",
+                "release": self.doc_release,
+                "title": "QuerySet",
+            }
+        )
+        Document.objects.search_update()
+        base_url = reverse_with_host(
+            "document-detail",
+            host="docs",
+            kwargs={"lang": "en", "version": "5.1", "url": "refs/query"},
+        )
+        for query, expected_code_links in [
+            (
+                "queryset",
+                f'<div class="code-links"><a href="{base_url}#django.db.models.query.QuerySet"><div><code>QuerySet'
+                '</code><div class="meta">django.db.models.query</div></div></a></div>',
+            ),
+            (
+                "select",
+                f'<div class="code-links"><a href="{base_url}#django.db.models.query.QuerySet.select_for_update"><div>'
+                '<code>QuerySet.select_for_update</code><div class="meta">django.db.models.query</div></div></a>'
+                f'<a href="{base_url}#django.db.models.query.QuerySet.select_related"><div><code>QuerySet.select_related'
+                '</code><div class="meta">django.db.models.query</div></div></a></div>',
+            ),
+            (
+                "release",
+                f'<div class="code-links"><a href="{base_url}#term-Feature-release"><div><code>Feature release</code>'
+                "</div></a></div>",
+            ),
+        ]:
+            with self.subTest(query=query):
+                response = self.client.get(
+                    f"/en/5.1/search/?q={query}",
+                    headers={"host": "docs.djangoproject.localhost:8000"},
+                )
+                self.assertEqual(response.status_code, 200)
+                self.assertContains(
+                    response,
+                    f"Only 1 result for <em>{query}</em> in version 5.1",
+                    html=True,
+                )
+                self.assertContains(response, expected_code_links, html=True)
+
 
 class TemplateTagTests(TestCase):
     fixtures = ["doc_test_fixtures"]
@@ -326,6 +398,41 @@ def test_generate_scroll_to_text_fragment(self):
                     url_text_fragment,
                 )
 
+    def test_code_links(self):
+        code_references = {
+            "Layer": "django.contrib.gis.gdal.Layer",
+            "Migration.initial": "django.db.migrations.Migration.initial",
+            "db_for_write": "db_for_write",
+        }
+        for code_matched, expected in [
+            (None, {}),
+            ("", {}),
+            ("Layer initial db_for_write", {}),
+            (
+                "Layer initial <mark>db_for</mark>_write",
+                {"db_for_write": {"full_path": "db_for_write", "module_path": None}},
+            ),
+            (
+                "<mark>Layer</mark> <mark>initial</mark> <mark>db_for</mark>_write",
+                {
+                    "db_for_write": {"full_path": "db_for_write", "module_path": None},
+                    "Layer": {
+                        "full_path": "django.contrib.gis.gdal.Layer",
+                        "module_path": "django.contrib.gis.gdal",
+                    },
+                    "Migration.initial": {
+                        "full_path": "django.db.migrations.Migration.initial",
+                        "module_path": "django.db.migrations",
+                    },
+                },
+            ),
+        ]:
+            with self.subTest(code_matched=code_matched):
+                self.assertEqual(
+                    code_links(code_matched, code_references),
+                    expected,
+                )
+
 
 class TestUtils(TestCase):
     def test_get_doc_path(self):
@@ -360,6 +467,33 @@ def test_sanitize_for_trigram(self):
             with self.subTest(query=query):
                 self.assertEqual(sanitize_for_trigram(query), sanitized_query)
 
+    def test_generate_code_references(self):
+        test_cases = [
+            ("", {}),
+            (
+                '<dl class="py attribute"><dt class="sig sig-object py" id="django.db.migrations.Migration.initial">',
+                {"Migration.initial": "django.db.migrations.Migration.initial"},
+            ),
+            (
+                '<dl class="py class"><dt class="sig sig-object py" id="django.contrib.gis.gdal.Layer">',
+                {"Layer": "django.contrib.gis.gdal.Layer"},
+            ),
+            (
+                '<dl class="py method"><dt class="sig sig-object py" id="db_for_write">',
+                {"db_for_write": "db_for_write"},
+            ),
+            (
+                '<dl class="glossary"><dt id="term-Feature-release">',
+                {"Feature-release": "term-Feature-release"},
+            ),
+            (
+                '<dl class="std envvar"><dt class="sig sig-object std" id="envvar-DJANGO_ALLOW_ASYNC_UNSAFE">',
+                {"DJANGO_ALLOW_ASYNC_UNSAFE": "envvar-DJANGO_ALLOW_ASYNC_UNSAFE"},
+            ),
+        ]
+        for body, expected in test_cases:
+            self.assertEqual(generate_code_references(body), expected)
+
 
 class UpdateDocTests(TestCase):
     @classmethod
@@ -424,6 +558,38 @@ def test_title_entities(self):
             transform=attrgetter("title"),
         )
 
+    def test_code_entities(self):
+        self.release.sync_to_db(
+            [
+                {
+                    "body": (
+                        '<dl class="py class"><dt class="sig sig-object py" id="django.contrib.gis.gdal.Layer">'
+                        '<dl class="py attribute"><dt class="sig sig-object py" id="django.db.migrations.Migration.initial">'
+                        '<dl class="py method"><dt class="sig sig-object py" id="db_for_write">'
+                    ),
+                    "title": "This is the title",
+                    "current_page_name": "foo/bar",
+                }
+            ]
+        )
+        self.assertQuerySetEqual(
+            self.release.documents.all(),
+            [
+                (
+                    {
+                        "Layer": "django.contrib.gis.gdal.Layer",
+                        "Migration.initial": "django.db.migrations.Migration.initial",
+                        "db_for_write": "db_for_write",
+                    },
+                    "Layer initial db_for_write",
+                )
+            ],
+            transform=lambda doc: (
+                doc.metadata["code_references"],
+                doc.metadata["code_references_search"],
+            ),
+        )
+
     def test_empty_documents(self):
         self.release.sync_to_db(
             [
@@ -477,10 +643,10 @@ def test_sitemap_index(self):
             "/sitemap.xml", headers={"host": "docs.djangoproject.localhost:8000"}
         )
         self.assertContains(response, "<sitemap>", count=2)
-        self.assertContains(
-            response,
-            "<loc>http://docs.djangoproject.localhost:8000/sitemap-en.xml</loc>",
+        en_sitemap_url = reverse_with_host(
+            "document-sitemap", host="docs", kwargs={"section": "en"}
         )
+        self.assertContains(response, f"<loc>{en_sitemap_url}</loc>")
 
     def test_sitemap(self):
         doc_release = DocumentRelease.objects.create()