Surface documented crates on docs.forc.pub and warm docs loads

Author: JoshuaBatty

app/src/features/detail/components/PackageSidebar.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import React from "react";
+import React, { useEffect, useRef } from "react";
 import { useRouter } from "next/navigation";
 import {
   Box,
@@ -32,6 +32,7 @@ interface PackageSidebarProps {
 }
 
 const PackageSidebar = ({ data, loading, error }: PackageSidebarProps) => {
+  const warmedDocsRef = useRef<Set<string>>(new Set());
   const router = useRouter();
   const docsRelativeUrl =
     data?.docsIpfsUrl && data.name && data.version
@@ -49,6 +50,74 @@ const PackageSidebar = ({ data, loading, error }: PackageSidebarProps) => {
       : docsRelativeUrl
     : "";
 
+  useEffect(() => {
+    const docKey =
+      data?.docsIpfsUrl && data?.name && data?.version
+        ? `${data.name}@${data.version}`
+        : null;
+
+    if (!docKey || !docsRelativeUrl || warmedDocsRef.current.has(docKey)) {
+      return;
+    }
+
+    const controller = new AbortController();
+    let isMounted = true;
+    const baseDocsUrl = docsRelativeUrl.endsWith("/")
+      ? docsRelativeUrl.slice(0, -1)
+      : docsRelativeUrl;
+
+    const warmDocs = async () => {
+      try {
+        const response = await fetch(baseDocsUrl, {
+          signal: controller.signal,
+          credentials: "same-origin",
+          cache: "force-cache",
+          headers: {
+            "x-docs-prefetch": "1",
+          },
+        });
+
+        if (!response.ok) {
+          throw new Error(`Docs prefetch failed with status ${response.status}`);
+        }
+
+        // Warm the search index in the background; ignore any errors.
+        fetch(`${baseDocsUrl}/search.js`, {
+          signal: controller.signal,
+          credentials: "same-origin",
+          cache: "force-cache",
+          headers: {
+            "x-docs-prefetch": "1",
+          },
+        }).catch(() => undefined);
+
+        if (isMounted) {
+          warmedDocsRef.current.add(docKey);
+          if (process.env.NODE_ENV !== "production") {
+            console.debug(`Prefetched docs for ${docKey}`);
+          }
+        }
+      } catch (prefetchError) {
+        if (
+          controller.signal.aborted ||
+          (prefetchError as Error).name === "AbortError"
+        ) {
+          return;
+        }
+        if (process.env.NODE_ENV !== "production") {
+          console.debug("Docs prefetch failed", prefetchError);
+        }
+      }
+    };
+
+    warmDocs();
+
+    return () => {
+      isMounted = false;
+      controller.abort();
+    };
+  }, [data?.docsIpfsUrl, data?.name, data?.version, docsRelativeUrl]);
+
   if (loading) {
     return (
       <Box

src/api/search.rs

@@ -8,8 +8,17 @@ use url::Url;
 #[derive(Serialize, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct RecentPackagesResponse {
-    pub recently_created: Vec<PackagePreview>,
-    pub recently_updated: Vec<PackagePreview>,
+    pub recently_created: Vec<RecentPackage>,
+    pub recently_updated: Vec<RecentPackage>,
+}
+
+#[derive(Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct RecentPackage {
+    #[serde(flatten)]
+    pub package: PackagePreview,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub docs_ipfs_url: Option<String>,
 }
 
 #[derive(Serialize, Debug)]

src/db/package_version.rs

@@ -4,7 +4,7 @@ use crate::api::pagination::{PaginatedResponse, Pagination};
 use crate::handlers::publish::PublishInfo;
 use crate::models::{
     ApiToken, AuthorInfo, CountResult, FullPackage, FullPackageWithCategories, PackagePreview,
-    PackagePreviewWithCategories, PackageVersionInfo,
+    PackagePreviewWithCategories, PackagePreviewWithDocsHash, PackageVersionInfo,
 };
 use chrono::{DateTime, Utc};
 use diesel::prelude::*;
@@ -136,7 +136,9 @@ impl DbConn<'_> {
     }
 
     /// Fetch the most recently updated packages.
-    pub fn get_recently_updated(&mut self) -> Result<Vec<PackagePreview>, DatabaseError> {
+    pub fn get_recently_updated(
+        &mut self,
+    ) -> Result<Vec<PackagePreviewWithDocsHash>, DatabaseError> {
         let packages = diesel::sql_query(
             r#"WITH ranked_versions AS (
                 SELECT 
@@ -146,30 +148,35 @@ impl DbConn<'_> {
                     pv.package_description AS description, 
                     p.created_at AS created_at, 
                     pv.created_at AS updated_at,
+                    u.docs_ipfs_hash AS docs_ipfs_hash,
                     ROW_NUMBER() OVER (PARTITION BY p.id ORDER BY pv.created_at DESC) AS rank
                 FROM package_versions pv
                 JOIN packages p ON pv.package_id = p.id
+                JOIN uploads u ON pv.upload_id = u.id
             )
             SELECT 
                 name, 
                 version, 
                 description, 
                 created_at, 
-                updated_at
+                updated_at,
+                docs_ipfs_hash
             FROM ranked_versions
             WHERE rank = 1
             ORDER BY updated_at DESC
             LIMIT 10;
             "#,
         )
-        .load::<PackagePreview>(self.inner())
+        .load::<PackagePreviewWithDocsHash>(self.inner())
         .map_err(|err| DatabaseError::QueryFailed("recently updated".to_string(), err))?;
 
         Ok(packages)
     }
 
     /// Fetch the [PackagePreview]s of the most recently created packages.
-    pub fn get_recently_created(&mut self) -> Result<Vec<PackagePreview>, DatabaseError> {
+    pub fn get_recently_created(
+        &mut self,
+    ) -> Result<Vec<PackagePreviewWithDocsHash>, DatabaseError> {
         let packages = diesel::sql_query(
             r#"WITH ranked_versions AS (
                 SELECT 
@@ -179,23 +186,26 @@ impl DbConn<'_> {
                     pv.package_description AS description, 
                     p.created_at AS created_at, 
                     pv.created_at AS updated_at,
+                    u.docs_ipfs_hash AS docs_ipfs_hash,
                     ROW_NUMBER() OVER (PARTITION BY p.id ORDER BY pv.created_at DESC) AS rank
                 FROM package_versions pv
                 JOIN packages p ON pv.package_id = p.id
+                JOIN uploads u ON pv.upload_id = u.id
             )
             SELECT 
                 name, 
                 version, 
                 description, 
                 created_at, 
-                updated_at
+                updated_at,
+                docs_ipfs_hash
             FROM ranked_versions
             WHERE rank = 1
             ORDER BY created_at DESC
             LIMIT 10;
             "#,
         )
-        .load::<PackagePreview>(self.inner())
+        .load::<PackagePreviewWithDocsHash>(self.inner())
         .map_err(|err| DatabaseError::QueryFailed("recently created".to_string(), err))?;
 
         Ok(packages)

src/main.rs

@@ -7,7 +7,9 @@ use chrono::{DateTime, Utc};
 use forc_pub::api::api_token::{CreateTokenRequest, CreateTokenResponse, Token, TokensResponse};
 use forc_pub::api::pagination::{PaginatedResponse, Pagination};
 use forc_pub::api::publish::{PublishRequest, PublishResponse, UploadResponse};
-use forc_pub::api::search::{DownloadLinksResponse, FullPackage, RecentPackagesResponse};
+use forc_pub::api::search::{
+    DownloadLinksResponse, FullPackage, RecentPackage, RecentPackagesResponse,
+};
 use forc_pub::api::ApiError;
 use forc_pub::api::{
     auth::{LoginRequest, LoginResponse, UserResponse},
@@ -17,7 +19,7 @@ use forc_pub::db::error::DatabaseError;
 use forc_pub::db::Database;
 use forc_pub::file_uploader::s3::{ipfs_hash_to_s3_url, S3Client, S3ClientImpl};
 use forc_pub::file_uploader::{
-    pinata::{PinataClient, PinataClientImpl},
+    pinata::{ipfs_hash_to_docs_url, PinataClient, PinataClientImpl},
     FileUploader,
 };
 use forc_pub::github::handle_login;
@@ -27,7 +29,8 @@ use forc_pub::middleware::cors::Cors;
 use forc_pub::middleware::session_auth::{SessionAuth, SESSION_COOKIE_NAME};
 use forc_pub::middleware::token_auth::TokenAuth;
 use forc_pub::models::{
-    FullPackageWithCategories, PackagePreviewWithCategories, PackageVersionInfo,
+    FullPackageWithCategories, PackagePreviewWithCategories, PackagePreviewWithDocsHash,
+    PackageVersionInfo,
 };
 use forc_pub::util::{load_env, validate_or_format_semver};
 use rocket::http::Status;
@@ -400,11 +403,28 @@ fn recent_packages(db: &State<Database>) -> ApiResult<RecentPackagesResponse> {
         Ok::<_, DatabaseError>((recently_created, recently_updated))
     })?;
     Ok(Json(RecentPackagesResponse {
-        recently_created,
-        recently_updated,
+        recently_created: map_recent_packages(recently_created),
+        recently_updated: map_recent_packages(recently_updated),
     }))
 }
 
+fn map_recent_packages(packages: Vec<PackagePreviewWithDocsHash>) -> Vec<RecentPackage> {
+    packages
+        .into_iter()
+        .map(|pkg| {
+            let docs_ipfs_url = pkg
+                .docs_ipfs_hash
+                .filter(|hash| !hash.is_empty())
+                .map(|hash| ipfs_hash_to_docs_url(&hash));
+
+            RecentPackage {
+                package: pkg.package,
+                docs_ipfs_url,
+            }
+        })
+        .collect()
+}
+
 /// Catches all OPTION requests in order to get the CORS related Fairing triggered.
 #[options("/<_..>")]
 fn all_options() {
@@ -484,8 +504,6 @@ async fn get_package_docs(
     name: String,
     version: String,
 ) -> Result<Redirect, Status> {
-    use forc_pub::file_uploader::pinata::ipfs_hash_to_docs_url;
-
     let package_result =
         db.transaction(|conn| conn.get_full_package_with_categories(name.clone(), version.clone()));
 

src/models.rs

@@ -218,6 +218,14 @@ pub struct PackagePreview {
     pub updated_at: DateTime<Utc>,
 }
 
+#[derive(QueryableByName, Debug, Clone)]
+pub struct PackagePreviewWithDocsHash {
+    #[diesel(embed)]
+    pub package: PackagePreview,
+    #[diesel(sql_type = Nullable<Text>)]
+    pub docs_ipfs_hash: Option<String>,
+}
+
 #[derive(Serialize, Debug, Clone)]
 #[serde(rename_all = "camelCase")]
 pub struct PackagePreviewWithCategories {