Add initial download_urls table with links checked on the fly

idr_gallery/templates/idr_gallery/download_urls.html

@@ -0,0 +1,134 @@
+{% extends "idr_gallery/base.html" %}
+
+{% block content %}
+
+
+<style>
+  body {
+    background-color: #f6f6f9;
+  }
+
+  table {
+    width: 100%;
+    border-collapse: collapse;
+    font-size: 12px;
+  }
+
+</style>
+
+<h1>Download URLs</h1>
+
+<div id="table-container">
+  <p>Loading download URLs...</p>
+</div>
+
+<script>
+
+  function checkLinks() {
+    let links = Array.from(document.querySelectorAll('a'));
+    // links = links.slice(0,50)
+    links.forEach(link => {
+      // no corss-origin check for now, just check if the link is valid
+      fetch(link.href, { method: 'HEAD'})
+        .then(response => {
+          // console.log("response.ok", response.ok, link.href);
+          link.style.color =  response.ok ? 'green' : 'orange';
+          link.title = response.ok ? 'Link is valid' : 'Link is broken';
+
+          if (!response.ok) {
+            // Use our custom link check endpoint to get more details on why the link is broken
+            // fetch(`/link_check/?url=${encodeURIComponent(link.href)}`)
+            //   .then(res => res.json())
+            //   .then(data => {
+            //     console.log('Link check data:', data);
+            //     if (data.is_valid) {
+            //       link.style.color = 'blue';
+            //       link.title = 'Link is valid (checked with server)';
+            //     } else {
+            //       link.style.color = 'yellow';
+            //       link.title = `Link is broken: ${data.error}`;
+            //     }
+            //   });
+          }
+        })
+        .catch(error => {
+          console.error('Error checking link:', error);
+          link.style.color = 'pink';
+          link.title = 'Error checking link';
+
+          fetch(`/link_check/?url=${encodeURIComponent(link.href)}`)
+              .then(res => res.json())
+              .then(data => {
+                console.log('Link check data:', data);
+                if (data.is_valid) {
+                  link.style.color = 'green';
+                  link.title = 'Link is valid (checked with server)';
+                } else {
+                  link.style.color = 'red';
+                  link.title = `Link is broken: ${data.error}`;
+                }
+              });
+        });
+    });
+  }
+
+  // Fetch the TSV file and display it in a table
+  fetch('https://raw.githubusercontent.com/will-moore/idr.openmicroscopy.org/refs/heads/download_urls/_data/download_urls.tsv')
+    .then(response => response.text())
+    .then(data => {
+      const lines = data.split('\n');
+      const table = document.createElement('table');
+      table.style.width = '100%';
+      table.style.borderCollapse = 'collapse';
+
+        // Create table header
+        const header = table.createTHead();
+        const headerRow = header.insertRow();
+        const headers = lines[0].split('\t');
+        headers.forEach(headerText => {
+          const th = document.createElement('th');
+          th.textContent = headerText;
+          th.style.border = '1px solid #ddd';
+          th.style.padding = '2px';
+          th.style.backgroundColor = '#f2f2f2';
+          headerRow.appendChild(th);
+        });
+
+        // Create table body
+        const tbody = table.createTBody();
+        lines.slice(1).forEach(line => {
+          if (line.trim() === '') return; // Skip empty lines
+          const row = tbody.insertRow();
+          const cells = line.split('\t');
+          cells.forEach(cellText => {
+            const td = document.createElement('td');
+            td.style.border = '1px solid #ddd';
+            td.style.padding = '2px';
+            if (cellText.startsWith('http')) {
+              const link = document.createElement('a');
+              link.href = cellText;
+              link.textContent = cellText;
+              link.target = '_blank';
+              td.textContent = '';
+              td.appendChild(link);
+            } else {
+              td.textContent = cellText;
+            }
+            row.appendChild(td);
+          });
+        });
+        const tableContainer = document.getElementById('table-container');
+        tableContainer.innerHTML = '';
+        tableContainer.appendChild(table);
+
+        checkLinks();
+    })
+    .catch(error => {
+      console.error('Error fetching TSV file:', error);
+      const tableContainer = document.getElementById('table-container');
+      tableContainer.innerHTML = '<p>Error loading download URLs.</p>';
+    });
+
+</script>
+
+{% endblock %}

idr_gallery/templates/idr_gallery/idr_study.html

@@ -181,7 +181,8 @@ <h3>Download</h3>
             {% if download_url %}
             <p>
                 To download original image files in your browser, you can
-                <a class="downld-link" target="_blank" href="{{ download_url }}">access original data</a>.
+                <!-- NB: This URL is updated dynamically by JavaScript below, using download_urls.tsv -->
+                <a id="download_url" target="_blank" href="{{ download_url }}">access original data</a>.
             </p>
             {% endif %}
 
@@ -224,14 +225,17 @@ <h3>Download</h3>
 </div>
 
 <script>
+
+    const BASE_URL = "{{ base_url }}";
+    const IDR_ID = "{{ idr_id }}";
+    const MAX_IMAGES = 1000000;
+    const SLOW_WARNING_THRESHOLD = 100000;
+    let url = BASE_URL + "searchengine/api/v1/resources/container_images/?data_source=idr";
+    let fmt = new Intl.NumberFormat();
+
     document.addEventListener("DOMContentLoaded", function() {
         // load image counts...
         // https://idr.openmicroscopy.org/searchengine//api/v1/resources/container_images/?data_source=idr
-        const BASE_URL = "{{ base_url }}";
-        const MAX_IMAGES = 1000000;
-        const SLOW_WARNING_THRESHOLD = 100000;
-        let url = BASE_URL + "searchengine/api/v1/resources/container_images/?data_source=idr";
-        let fmt = new Intl.NumberFormat();
 
         // for each table-download-link (csv download link OR hidden parquet link), HEAD request to get content-length...
         // Then add size in MB after link
@@ -255,6 +259,7 @@ <h3>Download</h3>
                 });
         });
 
+        // Load image counts for each container from searchengine and add to page
         fetch(url)
             .then(response => response.json())
             .then(data => {
@@ -312,6 +317,35 @@ <h3>Download</h3>
             .catch(error => {
                 console.error('Error fetching image counts:', error);
             });
-        });
+
+        // Load download URLs from TSV file in GitHub...
+        fetch('https://raw.githubusercontent.com/will-moore/idr.openmicroscopy.org/refs/heads/download_urls/_data/download_urls.tsv')
+            .then(response => response.text())
+            .then(data => {
+                const lines = data.split('\n');
+
+                // Use first line as header to find column names...
+                const headers = lines[0].split('\t');
+
+                const idrIndex = headers.indexOf("idrid");
+
+                // Find the line for this IDR ID
+                const studyLine = lines.find(line => line.split('\t')[idrIndex] === IDR_ID);
+                if (studyLine) {
+                    const cells = studyLine.split('\t');
+                    const downloadUrlIndex = headers.indexOf("download_url");
+                    const downloadUrl = cells[downloadUrlIndex];
+                    if (downloadUrl) {
+                        const downloadLink = document.getElementById("download_url");
+                        if (downloadLink) {
+                            downloadLink.href = downloadUrl;
+                        }
+                    }
+                } else {
+                    console.warn(`No download URL found for ${IDR_ID} in TSV file`);
+                }
+            });
+    });
+
 </script>
 {% endblock %}

idr_gallery/urls.py

@@ -18,6 +18,13 @@
     re_path(r'^search/$', views.index, {'super_category': None},
             name="idr_gallery_search"),
 
+    re_path(r'^download_urls/$', views.download_urls,
+            name="idr_gallery_download_urls"),
+
+    # Use ?url=... to check if a URL is valid, used by download_urls page
+    re_path(r'^link_check/$', views.link_check,
+            name="idr_gallery_link_check"),
+
     # Supports e.g. ?project=1&project=2&screen=3
     re_path(r'^gallery-api/thumbnails/$', views.api_thumbnails,
             name='idr_gallery_api_thumbnails'),

idr_gallery/views.py

@@ -209,7 +209,7 @@ def study_page(request, idrid, format="html", conn=None, **kwargs):
                                            f"{obj.name}.parquet",
                                            ext="parquet"),
             "empty_study_container": ("experiment" not in obj.name
-                                      and "screen" not in obj.name,)
+                                      and "screen" not in obj.name)
         })
 
     img_objects = []
@@ -533,3 +533,34 @@ def get_bff_url(request, data_url, fname, ext="csv"):
     s = urllib.parse.quote(json.dumps(source))
     bff_url = f"{BFF_URL}?source={s}"
     return bff_url
+
+
+@render_response()
+def download_urls(request, conn=None, **kwargs):
+    """
+    Return a page with download URLs for all studies.
+    """
+    context = {
+        "template": "idr_gallery/download_urls.html",
+        "VERSION": VERSION,
+    }
+    # settings_ctx = get_settings_as_context()
+    # context = {**context, **settings_ctx}
+    return context
+
+
+def link_check(request):
+    """
+    API endpoint to check if a URL is valid, used by download_urls page
+    """
+    url = request.GET.get("url")
+    if url is None:
+        return HttpResponseBadRequest("Missing 'url' parameter")
+    try:
+        response = requests.head(url, allow_redirects=True, timeout=5)
+        is_valid = response.ok
+    except requests.RequestException:
+        is_valid = False
+    status_code = response.status_code if is_valid else None
+    return JsonResponse({"url": url, "is_valid": is_valid,
+                         "status_code": status_code})