galaxyproject · jnussbaum · Jun 27, 2026
diff --git a/.gitattributes b/.gitattributes
@@ -1,2 +1,3 @@
 * text=auto
 lib/galaxy/datatypes/test/dosimzml eol=crlf
+lib/galaxy/datatypes/test/example.warc eol=crlf
diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample
@@ -630,6 +630,8 @@
     <datatype extension="bed_tabix.gz" type="galaxy.datatypes.interval:BedTabix" mimetype="application/octet-stream" display_in_upload="true"/>
     <datatype extension="gff_tabix.gz" type="galaxy.datatypes.interval:GffTabix" mimetype="application/octet-stream" display_in_upload="true"/>
     <datatype extension="bgzip" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true"/>
+    <datatype extension="warc" type="galaxy.datatypes.warc:Warc" display_in_upload="true" description="Web ARChive (WARC) file." description_url="https://iipc.github.io/warc-specifications/"/>
+    <datatype extension="warc.gz" type="galaxy.datatypes.warc:WarcGz" display_in_upload="true" description="gzip-compressed Web ARChive (WARC) file." description_url="https://iipc.github.io/warc-specifications/"/>
     <!-- GWAS results datatypes to be used with locuszoom visualization plugin -->
     <datatype extension="gwas_bgzip" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true"/>
     <datatype extension="gwas_tabix.gz" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true"/>
@@ -1522,5 +1524,7 @@
     <sniffer type="galaxy.datatypes.text:StormSample"/>
     <sniffer type="galaxy.datatypes.text:StormCheck"/>
     <sniffer type="galaxy.datatypes.text:SourmashSignature"/>
+    <sniffer type="galaxy.datatypes.warc:WarcGz"/>
+    <sniffer type="galaxy.datatypes.warc:Warc"/>
   </sniffers>
 </datatypes>
diff --git a/lib/galaxy/datatypes/test/example.warc b/lib/galaxy/datatypes/test/example.warc
@@ -0,0 +1,187 @@
+WARC/1.0
+WARC-Type: warcinfo
+WARC-Record-ID: <urn:uuid:fbd6cf0a-6160-4550-b343-12188dc05234>
+WARC-Date: 2014-01-03T03:03:22Z
+Content-Length: 196
+Content-Type: application/warc-fields
+WARC-Filename: live-20140103030321-wwwb-app5.us.archive.org.warc.gz
+
+software: LiveWeb Warc Writer 1.0
+host: wwwb-app5.us.archive.org
+isPartOf: liveweb
+format: WARC file version 1.0
+conformsTo: http://bibnum.bnf.fr/WARC/WARC_ISO_28500_version1_latestdraft.pdf
+
+
+WARC/1.0
+WARC-Type: response
+WARC-Record-ID: <urn:uuid:6d058047-ede2-4a13-be79-90c17c631dd4>
+WARC-Date: 2014-01-03T03:03:21Z
+Content-Length: 1610
+Content-Type: application/http; msgtype=response
+WARC-Payload-Digest: sha1:B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A
+WARC-Target-URI: http://example.com?example=1
+WARC-Warcinfo-ID: <urn:uuid:fbd6cf0a-6160-4550-b343-12188dc05234>
+
+HTTP/1.1 200 OK
+Accept-Ranges: bytes
+Cache-Control: max-age=604800
+Content-Type: text/html
+Date: Fri, 03 Jan 2014 03:03:21 GMT
+Etag: "359670651"
+Expires: Fri, 10 Jan 2014 03:03:21 GMT
+Last-Modified: Fri, 09 Aug 2013 23:54:35 GMT
+Server: ECS (sjc/4FCE)
+X-Cache: HIT
+x-ec-custom-error: 1
+Content-Length: 1270
+Connection: close
+
+<!doctype html>
+<html>
+<head>
+    <title>Example Domain</title>
+
+    <meta charset="utf-8" />
+    <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <style type="text/css">
+    body {
+        background-color: #f0f0f2;
+        margin: 0;
+        padding: 0;
+        font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
+
+    }
+    div {
+        width: 600px;
+        margin: 5em auto;
+        padding: 50px;
+        background-color: #fff;
+        border-radius: 1em;
+    }
+    a:link, a:visited {
+        color: #38488f;
+        text-decoration: none;
+    }
+    @media (max-width: 700px) {
+        body {
+            background-color: #fff;
+        }
+        div {
+            width: auto;
+            margin: 0 auto;
+            border-radius: 0;
+            padding: 1em;
+        }
+    }
+    </style>    
+</head>
+
+<body>
+<div>
+    <h1>Example Domain</h1>
+    <p>This domain is established to be used for illustrative examples in documents. You may use this
+    domain in examples without prior coordination or asking for permission.</p>
+    <p><a href="http://www.iana.org/domains/example">More information...</a></p>
+</div>
+</body>
+</html>
+
+
+WARC/1.0
+WARC-Type: request
+WARC-Record-ID: <urn:uuid:9a3ffea5-9556-4790-a6bf-c15231fd6b97>
+WARC-Date: 2014-01-03T03:03:21Z
+Content-Length: 323
+Content-Type: application/http; msgtype=request
+WARC-Concurrent-To: <urn:uuid:6d058047-ede2-4a13-be79-90c17c631dd4>
+WARC-Target-URI: http://example.com?example=1
+WARC-Warcinfo-ID: <urn:uuid:fbd6cf0a-6160-4550-b343-12188dc05234>
+
+GET /?example=1 HTTP/1.1
+Connection: close
+Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
+Accept-Language: en-US,en;q=0.8
+User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36 (via Wayback Save Page)
+Host: example.com
+
+
+WARC/1.0
+WARC-Type: revisit
+WARC-Record-ID: <urn:uuid:3619f5b0-d967-44be-8f24-762098d427c4>
+WARC-Date: 2014-01-03T03:03:41Z
+Content-Length: 340
+Content-Type: application/http; msgtype=response
+WARC-Payload-Digest: sha1:B2LTWWPUOYAH7UIPQ7ZUPQ4VMBSVC36A
+WARC-Target-URI: http://example.com?example=1
+WARC-Warcinfo-ID: <urn:uuid:fbd6cf0a-6160-4550-b343-12188dc05234>
+WARC-Profile: http://netpreserve.org/warc/0.18/revisit/identical-payload-digest
+WARC-Refers-To-Target-URI: http://example.com?example=1
+WARC-Refers-To-Date: 2014-01-03T03:03:21Z
+
+HTTP/1.1 200 OK
+Accept-Ranges: bytes
+Cache-Control: max-age=604800
+Content-Type: text/html
+Date: Fri, 03 Jan 2014 03:03:41 GMT
+Etag: "359670651"
+Expires: Fri, 10 Jan 2014 03:03:41 GMT
+Last-Modified: Fri, 09 Aug 2013 23:54:35 GMT
+Server: ECS (sjc/4FCE)
+X-Cache: HIT
+x-ec-custom-error: 1
+Content-Length: 1270
+Connection: close
+
+
+
+WARC/1.0
+WARC-Type: request
+WARC-Record-ID: <urn:uuid:c59f3330-b241-4fca-8513-d687cd85bcfb>
+WARC-Date: 2014-01-03T03:03:41Z
+Content-Length: 323
+Content-Type: application/http; msgtype=request
+WARC-Concurrent-To: <urn:uuid:3619f5b0-d967-44be-8f24-762098d427c4>
+WARC-Target-URI: http://example.com?example=1
+WARC-Warcinfo-ID: <urn:uuid:fbd6cf0a-6160-4550-b343-12188dc05234>
+
+GET /?example=1 HTTP/1.1
+Connection: close
+Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
+Accept-Language: en-US,en;q=0.8
+User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36 (via Wayback Save Page)
+Host: example.com
+
+
+WARC/1.0
+WARC-Type: response
+WARC-Record-ID: <urn:uuid:1d673b2a-c593-402e-8973-3950d0bc6163>
+WARC-Date: 2014-01-28T05:15:39Z
+Content-Length: 471
+Content-Type: application/http; msgtype=response
+WARC-Payload-Digest: sha1:JZ622UA23G5ZU6Y3XAKH4LINONUEICEG
+WARC-Target-URI: http://www.iana.org/domains/example
+WARC-Warcinfo-ID: <urn:uuid:e9f7f74b-0280-47fd-99bc-f00f1a570a46>
+
+HTTP/1.1 302 Found
+Server: Apache
+Location: /domains/reserved
+Content-Type: text/html; charset=iso-8859-1
+Content-Length: 201
+Accept-Ranges: bytes
+Date: Tue, 28 Jan 2014 05:15:39 GMT
+X-Varnish: 774901408 774900872
+Age: 80
+Via: 1.1 varnish
+Connection: close
+
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head>
+<title>302 Found</title>
+</head><body>
+<h1>Found</h1>
+<p>The document has moved <a href="/domains/reserved">here</a>.</p>
+</body></html>
+
+
diff --git a/lib/galaxy/datatypes/test/example.warc.gz b/lib/galaxy/datatypes/test/example.warc.gz
diff --git a/lib/galaxy/datatypes/warc.py b/lib/galaxy/datatypes/warc.py
@@ -0,0 +1,63 @@
+"""
+Datatypes for the Web ARChive (WARC) format.
+
+The WARC format stores archived web content together with the metadata that
+describes how it was captured. See the format specification at
+https://iipc.github.io/warc-specifications/ for details.
+
+WARC files are frequently distributed gzip-compressed (``.warc.gz``), where
+each record is written as a separate gzip member. Such files are kept
+compressed on upload: decompressing them would change the per-record byte
+offsets that tools such as ``warcio index`` report, and the records carry
+exact ``Content-Length`` byte counts that must not be altered by line-ending
+normalisation.
+"""
+
+from galaxy.datatypes.data import Text
+from galaxy.datatypes.sniff import (
+    build_sniff_from_prefix,
+    FilePrefix,
+)
+
+
+@build_sniff_from_prefix
+class Warc(Text):
+    """
+    Uncompressed Web ARChive (WARC) file.
+
+    Every WARC record begins with the version signature ``WARC/`` followed by
+    the version number (for example ``WARC/1.0``).
+    """
+
+    file_ext = "warc"
+
+    def sniff_prefix(self, file_prefix: FilePrefix) -> bool:
+        """
+        Every WARC record starts with the version signature ``WARC/``.
+
+        >>> from galaxy.datatypes.sniff import get_test_fname
+        >>> Warc().sniff(get_test_fname('example.warc'))
+        True
+        >>> Warc().sniff(get_test_fname('Si.cif'))
+        False
+        """
+        return file_prefix.startswith("WARC/")
+
+
+@build_sniff_from_prefix
+class WarcGz(Warc):
+    """
+    gzip-compressed Web ARChive (WARC) file.
+
+    The datatype is kept compressed on upload (``compressed = True``) so that
+    record byte offsets and ``Content-Length`` values remain valid. The
+    inherited sniffer matches against the decompressed content.
+
+    >>> from galaxy.datatypes.sniff import get_test_fname
+    >>> WarcGz().sniff(get_test_fname('example.warc.gz'))
+    True
+    """
+
+    file_ext = "warc.gz"
+    compressed = True
+    compressed_format = "gzip"
diff --git a/test/unit/data/datatypes/test_warc.py b/test/unit/data/datatypes/test_warc.py
@@ -0,0 +1,31 @@
+import pytest
+
+from galaxy.datatypes.warc import (
+    Warc,
+    WarcGz,
+)
+from .util import get_input_files
+
+
+@pytest.mark.parametrize(
+    "loader, input_file",
+    [
+        [Warc, "example.warc"],
+        [WarcGz, "example.warc.gz"],
+    ],
+)
+def test_warc_sniff(loader, input_file):
+    with get_input_files(input_file) as input_files:
+        assert loader().sniff(input_files[0]) is True
+
+
+@pytest.mark.parametrize(
+    "loader, input_file",
+    [
+        [Warc, "Si.cif"],
+        [WarcGz, "Si.cif"],
+    ],
+)
+def test_warc_sniff_negative(loader, input_file):
+    with get_input_files(input_file) as input_files:
+        assert loader().sniff(input_files[0]) is False