Fix superadmin running crawls views (#846)

- Updates superadmin "Running Crawls" to show active crawls (starting, waiting, running, stopping) and sort by start by default
- Navigates to crawl workflow watch view on clicking crawl item
- Adds "Copy Crawl ID" to crawl actions for easy paste into "Jump to crawl"
- Navigates to crawl workflow watch when jumping to crawl

Author: SuaYoo

frontend/src/components/crawl-list.ts

@@ -27,6 +27,7 @@ import { RelativeDuration } from "./relative-duration";
 import type { Crawl } from "../types/crawler";
 import { srOnly, truncate, dropdown } from "../utils/css";
 import type { NavigateEvent } from "../utils/LiteElement";
+import { isActive } from "../utils/crawler";
 
 const mediumBreakpointCss = css`30rem`;
 const largeBreakpointCss = css`60rem`;
@@ -219,12 +220,13 @@ export class CrawlListItem extends LitElement {
   }
 
   renderRow() {
+    const hash = this.crawl && isActive(this.crawl.state) ? "#watch" : "";
     return html`<a
       class="item row"
       role="button"
       href=${`${this.baseUrl || `/orgs/${this.crawl?.oid}/artifacts/crawl`}/${
         this.crawl?.id
-      }`}
+      }${hash}`}
       @click=${async (e: MouseEvent) => {
         e.preventDefault();
         await this.updateComplete;

frontend/src/index.ts

@@ -704,8 +704,8 @@ export class App extends LiteElement {
           <form
             @submit=${(e: any) => {
               e.preventDefault();
-              const id = new FormData(e.target).get("crawlId");
-              this.navigate(`/crawls/crawl/${id}`);
+              const id = new FormData(e.target).get("crawlId") as string;
+              this.navigate(`/crawls/crawl/${id}#watch`);
               e.target.closest("sl-dropdown").hide();
             }}
           >

frontend/src/pages/crawls.ts

@@ -4,19 +4,29 @@ import { msg, localized, str } from "@lit/localize";
 import type { AuthState } from "../utils/AuthService";
 import LiteElement, { html } from "../utils/LiteElement";
 import { needLogin } from "../utils/auth";
+import type { Crawl } from "../types/crawler";
 import { ROUTES } from "../routes";
-import "./org/crawl-detail";
+import "./org/workflow-detail";
 import "./org/crawls-list";
 
 @needLogin
 @localized()
 export class Crawls extends LiteElement {
   @property({ type: Object })
-  authState?: AuthState;
+  authState!: AuthState;
 
   @property({ type: String })
   crawlId?: string;
 
+  @state()
+  private crawl?: Crawl;
+
+  willUpdate(changedProperties: Map<string, any>) {
+    if (changedProperties.has("crawlId") && this.crawlId) {
+      this.fetchWorkflowId();
+    }
+  }
+
   render() {
     return html` <div
       class="w-full max-w-screen-lg mx-auto px-3 py-4 box-border"
@@ -26,23 +36,44 @@ export class Crawls extends LiteElement {
   }
 
   private renderDetail() {
+    if (!this.crawl) return;
+
     return html`
-      <btrix-crawl-detail
+      <btrix-workflow-detail
         .authState=${this.authState!}
-        crawlId=${this.crawlId!}
-        crawlsBaseUrl=${ROUTES.crawls}
-        crawlsAPIBaseUrl="/orgs/all/crawls"
-        showOrgLink
-      ></btrix-crawl-detail>
+        orgId=${this.crawl.oid}
+        workflowId=${this.crawl.cid}
+        initialActivePanel="watch"
+        isCrawler
+      ></btrix-workflow-detail>
     `;
   }
 
   private renderList() {
     return html`<btrix-crawls-list
-      .authState=${this.authState!}
+      .authState=${this.authState}
       crawlsBaseUrl=${ROUTES.crawls}
       crawlsAPIBaseUrl="/orgs/all/crawls"
+      isCrawler
+      isAdminView
       shouldFetch
     ></btrix-crawls-list>`;
   }
+
+  private async fetchWorkflowId() {
+    try {
+      this.crawl = await this.getCrawl();
+    } catch (e) {
+      console.error(e);
+    }
+  }
+
+  private async getCrawl(): Promise<Crawl> {
+    const data: Crawl = await this.apiFetch(
+      `/orgs/all/crawls/${this.crawlId}/replay.json`,
+      this.authState!
+    );
+
+    return data;
+  }
 }

frontend/src/pages/org/crawls-list.ts

@@ -18,7 +18,7 @@ import type { AuthState } from "../../utils/AuthService";
 import LiteElement, { html } from "../../utils/LiteElement";
 import type { Crawl, CrawlState, Workflow, WorkflowParams } from "./types";
 import type { APIPaginatedList, APIPaginationQuery } from "../../types/api";
-import { isActive } from "../../utils/crawler";
+import { isActive, activeCrawlStates } from "../../utils/crawler";
 
 type Crawls = APIPaginatedList & {
   items: Crawl[];
@@ -78,6 +78,7 @@ export class CrawlsList extends LiteElement {
     firstSeed: msg("Crawl Start URL"),
     cid: msg("Workflow ID"),
   };
+
   @property({ type: Object })
   authState!: AuthState;
 
@@ -87,6 +88,11 @@ export class CrawlsList extends LiteElement {
   @property({ type: Boolean })
   isCrawler!: boolean;
 
+  // TODO better handling of using same crawls-list
+  // component between superadmin view and regular view
+  @property({ type: Boolean })
+  isAdminView = false;
+
   // e.g. `/org/${this.orgId}/crawls`
   @property({ type: String })
   crawlsBaseUrl!: string;
@@ -103,7 +109,7 @@ export class CrawlsList extends LiteElement {
   shouldFetch?: boolean;
 
   @state()
-  private lastFetched?: number;
+  private crawlStates: CrawlState[] = finishedCrawlStates;
 
   @state()
   private crawls?: Crawls;
@@ -166,6 +172,15 @@ export class CrawlsList extends LiteElement {
   }
 
   protected willUpdate(changedProperties: Map<string, any>) {
+    if (changedProperties.has("isAdminView") && this.isAdminView === true) {
+      // TODO better handling of using same crawls-list
+      // component between superadmin view and regular view
+      this.crawlStates = activeCrawlStates;
+      this.orderBy = {
+        field: "started",
+        direction: sortableFields["started"].defaultDirection!,
+      };
+    }
     if (
       changedProperties.has("shouldFetch") ||
       changedProperties.get("crawlsBaseUrl") ||
@@ -199,7 +214,10 @@ export class CrawlsList extends LiteElement {
       changedProperties.has("crawlsBaseUrl") ||
       changedProperties.has("crawlsAPIBaseUrl")
     ) {
-      this.fetchConfigSearchValues();
+      // TODO add back when API supports `orgs/all/crawlconfigs`
+      if (!this.isAdminView) {
+        this.fetchConfigSearchValues();
+      }
     }
   }
 
@@ -223,7 +241,11 @@ export class CrawlsList extends LiteElement {
       <main>
         <header class="contents">
           <div class="flex w-full h-8 mb-4">
-            <h1 class="text-xl font-semibold">${msg("Finished Crawls")}</h1>
+            <h1 class="text-xl font-semibold">
+              ${this.isAdminView
+                ? msg("Running Crawls")
+                : msg("Finished Crawls")}
+            </h1>
           </div>
           <div
             class="sticky z-10 mb-3 top-2 p-4 bg-neutral-50 border rounded-lg"
@@ -266,7 +288,7 @@ export class CrawlsList extends LiteElement {
         class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-[minmax(0,100%)_fit-content(100%)_fit-content(100%)] gap-x-2 gap-y-2 items-center"
       >
         <div class="col-span-1 md:col-span-2 lg:col-span-1">
-          ${this.renderSearch()}
+          ${when(!this.isAdminView, () => this.renderSearch())}
         </div>
         <div class="flex items-center">
           <div class="text-neutral-500 mx-2">${msg("View:")}</div>
@@ -277,7 +299,9 @@ export class CrawlsList extends LiteElement {
             pill
             multiple
             max-tags-visible="1"
-            placeholder=${msg("Finished Crawls")}
+            placeholder=${this.isAdminView
+              ? msg("All Active Crawls")
+              : msg("Finished Crawls")}
             @sl-change=${async (e: CustomEvent) => {
               const value = (e.target as SlSelect).value as CrawlState[];
               await this.updateComplete;
@@ -287,7 +311,7 @@ export class CrawlsList extends LiteElement {
               };
             }}
           >
-            ${finishedCrawlStates.map(this.renderStatusMenuItem)}
+            ${this.crawlStates.map(this.renderStatusMenuItem)}
           </sl-select>
         </div>
 
@@ -448,7 +472,7 @@ export class CrawlsList extends LiteElement {
     if (!this.crawls) return;
 
     return html`
-      <btrix-crawl-list>
+      <btrix-crawl-list baseUrl=${this.isAdminView ? "/crawls/crawl" : ""}>
         ${this.crawls.items.map(this.renderCrawlItem)}
       </btrix-crawl-list>
 
@@ -515,6 +539,10 @@ export class CrawlsList extends LiteElement {
         <sl-icon name="copy-code" library="app" slot="prefix"></sl-icon>
         ${msg("Copy Workflow ID")}
       </sl-menu-item>
+      <sl-menu-item @click=${() => CopyButton.copyToClipboard(crawl.id)}>
+        <sl-icon name="copy-code" library="app" slot="prefix"></sl-icon>
+        ${msg("Copy Crawl ID")}
+      </sl-menu-item>
       <sl-menu-item
         @click=${() => CopyButton.copyToClipboard(crawl.tags.join(","))}
         ?disabled=${!crawl.tags.length}
@@ -639,7 +667,7 @@ export class CrawlsList extends LiteElement {
   }
 
   private async getCrawls(queryParams?: APIPaginationQuery): Promise<Crawls> {
-    const state = this.filterBy.state || finishedCrawlStates;
+    const state = this.filterBy.state || this.crawlStates;
     const query = queryString.stringify(
       {
         ...this.filterBy,
@@ -666,7 +694,6 @@ export class CrawlsList extends LiteElement {
     );
 
     this.getCrawlsController = null;
-    this.lastFetched = Date.now();
 
     return data;
   }

frontend/src/pages/org/workflow-detail.ts

@@ -1,6 +1,7 @@
 import type { HTMLTemplateResult, TemplateResult } from "lit";
 import { state, property } from "lit/decorators.js";
 import { when } from "lit/directives/when.js";
+import { until } from "lit/directives/until.js";
 import { ifDefined } from "lit/directives/if-defined.js";
 import { msg, localized, str } from "@lit/localize";
 import queryString from "query-string";
@@ -424,16 +425,22 @@ export class WorkflowDetail extends LiteElement {
       <btrix-tab-panel name="artifacts"
         >${this.renderArtifacts()}</btrix-tab-panel
       >
-      <btrix-tab-panel name="watch"
-        >${when(this.activePanel === "watch", () =>
-          this.currentCrawlId
-            ? html` <div class="border rounded-lg py-2 mb-5 h-14">
-                  ${this.renderCurrentCrawl()}
-                </div>
-                ${this.renderWatchCrawl()}`
-            : this.renderInactiveWatchCrawl()
-        )}</btrix-tab-panel
-      >
+      <btrix-tab-panel name="watch">
+        ${until(
+          this.getWorkflowPromise?.then(
+            () => html`
+              ${when(this.activePanel === "watch", () =>
+                this.currentCrawlId
+                  ? html` <div class="border rounded-lg py-2 mb-5 h-14">
+                        ${this.renderCurrentCrawl()}
+                      </div>
+                      ${this.renderWatchCrawl()}`
+                  : this.renderInactiveWatchCrawl()
+              )}
+            `
+          )
+        )}
+      </btrix-tab-panel>
       <btrix-tab-panel name="settings">
         ${this.renderSettings()}
       </btrix-tab-panel>
@@ -496,7 +503,7 @@ export class WorkflowDetail extends LiteElement {
     return html`
       <a
         slot="nav"
-        href=${`/orgs/${this.orgId}/workflows/crawl/${this.workflow?.id}#${tabName}`}
+        href=${`${window.location.pathname}#${tabName}`}
         class="block font-medium rounded-sm mb-2 mr-2 p-2 transition-all ${className}"
         aria-selected=${isActive}
         aria-disabled=${disabled}
@@ -806,7 +813,7 @@ export class WorkflowDetail extends LiteElement {
               ${msg(
                 html`Crawl is currently running.
                   <a
-                    href="${`/orgs/${this.orgId}/workflows/crawl/${this.workflow?.id}#watch`}"
+                    href="${`${window.location.pathname}#watch`}"
                     class="underline hover:no-underline"
                     >Watch Crawl Progress</a
                   >`