Merge pull request #121 from internetarchive/issue-51

Adding Spectrogram and Waveform

Author: glenrobson

iiify/loadtest/checkWaveformSizes.py

@@ -0,0 +1,50 @@
+import requests
+from iiify import resolver
+
+def getAudioItems():
+    """
+        Used to find audio items that have a waveform to check their size. 
+        It uses the advanced search to find items and then uses the image api
+        to find the width and height and check it against the expected size of 800,200
+    """
+    item_count = 0
+    waveform_item_count = 0
+    waveform_file_count = 0
+    for page in range(5):
+        audio = resolver.search("-collection:(stream_only) AND mediatype:(audio)".replace(":", "%3A"), page=page)
+        for item in audio["response"]["docs"]:
+            item_count += 1
+            identifier = item["identifier"]
+            metadata = requests.get(f"https://archive.org/metadata/{item["identifier"]}").json()
+            # sort the files into originals and derivatives, splitting the derivatives into buckets based on the original
+            (originals, derivatives) = resolver.sortDerivatives(metadata)
+            hasWaveform=False
+            for file in [f for f in originals if f['format'] in resolver.AUDIO_FORMATS]:
+                if file['name'] in derivatives and "PNG" in derivatives[file['name']]:   
+                    waveform_file_count += 1
+                    filename = derivatives[file['name']]["PNG"]["name"]
+                    imgId = f"{identifier}/{filename}".replace("/", "%2f")
+                    imgUrl = f"{resolver.IMG_SRV}/3/{imgId}/info.json"
+                    try:
+                        infojson = requests.get(imgUrl).json()
+                        if infojson['width'] == 800 and infojson['height'] == 200:
+                            print (f"EXPECTED {identifier} waveform: {filename}")
+                        else:    
+                            print (f"DIFFERENT {identifier} waveform: {filename} different size: {infojson['width']}, {infojson['height']} from {imgUrl}")
+                    except Exception as error:
+                        print (f"Failed to get {imgUrl}")    
+                        print (error)
+
+                    hasWaveform=True
+            if hasWaveform:
+                waveform_item_count += 1        
+
+    print (f"Number of items checked: {item_count}")
+    print (f"Number of waveforms {waveform_file_count} from {waveform_item_count} items")
+
+if __name__ == "__main__":
+    """
+        Run this as:
+            python -m iiify.loadtest.checkWaveformSizes   
+    """
+    getAudioItems()

iiify/resolver.py

@@ -3,7 +3,7 @@
 import os
 import requests
 from .configs import options, cors, approot, cache_root, media_root, apiurl, LINKS
-from iiif_prezi3 import Manifest, config, Annotation, AnnotationPage, AnnotationPageRef, Canvas, Manifest, ResourceItem, ServiceItem, Choice, Collection, ManifestRef, CollectionRef, ResourceItem1, CanvasRef
+from iiif_prezi3 import Manifest, config, Annotation, AnnotationPage, AnnotationPageRef, Canvas, Manifest, ResourceItem, ServiceItem, Choice, Collection, ManifestRef, CollectionRef, ResourceItem1, AccompanyingCanvas, CanvasRef
 from urllib.parse import urlparse, parse_qs, quote
 import json
 import math
@@ -477,7 +477,7 @@ def addSeeAlso(manifest, identifier, files):
         "Djvu XML": "OCR Data",
         "Scandata": "OCR Data",
         "Archive BitTorrent": "Torrent",
-        "Metadata": "Metadata",
+        "Metadata": "Metadata"
     }
 
     for file in files:
@@ -490,6 +490,60 @@ def addSeeAlso(manifest, identifier, files):
                  "format": seeAlso['format']
                  })
 
+def addWaveform(identifier, slugged_id, filename, hard_code_size=True):
+    """
+        Create an IIIF AccompanyingCanvas representing a waveform image.
+
+        This function generates an IIIF AccompanyingCanvas containing a waveform image, 
+        associated with an audio file. By default, the image dimensions are hardcoded, 
+        but if `hard_code_size` is False, the image's width and height will be retrieved 
+        dynamically from a IIIF image server.
+
+        Parameters:
+            identifier (str): The archive.org identifier for the resource (e.g. item ID).
+            slugged_id (str): A slugified version of the identifier used in the canvas ID.
+            filename (str): The filename of the waveform image (PNG).
+            hard_code_size (bool): If True, sets the image size to 800x200; if False, fetches dimensions from IIIF image server.
+
+        Returns:
+            AccompanyingCanvas: An IIIF-compliant AccompanyingCanvas object representing the waveform image.
+    """
+
+    # This should be the Wave form
+    accompanying_canvas = AccompanyingCanvas(
+        id=f"{URI_PRIFIX}/{identifier}/{slugged_id}/canvas/accompanying",
+        label={ "en": ["Waveform"]}
+    )
+    if hard_code_size:
+        width = 800
+        height = 200
+        body = ResourceItem(id=f"https://archive.org/download/{identifier}/{filename.replace(' ', '%20')}", type="Image", width=width, height=height)
+        body.format = "image/jpeg"
+    else:
+        imgId = f"{identifier}/{filename}".replace('/','%2f')
+        imgURL = f"{IMG_SRV}/3/{imgId}".replace(' ', '%20')
+        # Find the width and height from the image server
+        body = ResourceItem(id="http://example.com", type="Image")
+        infoJson = body.set_hwd_from_iiif(imgURL)
+
+        service = ServiceItem(id=infoJson['id'], profile=infoJson['profile'], type=infoJson['type'])
+        body.service = [service]
+        body.id = f'{infoJson["id"]}/full/max/0/default.jpg'
+        body.format = "image/jpeg"
+
+        width = infoJson['width']
+        height = infoJson['height']
+
+    annotation = Annotation(id=f"{accompanying_canvas.id}/anno", motivation='painting', body=body, target=accompanying_canvas.id)
+
+    annotationPage = AnnotationPage(id=f"{accompanying_canvas.id}/annoPage")
+    annotationPage.add_item(annotation)
+
+    accompanying_canvas.add_item(annotationPage)
+    accompanying_canvas.height = height
+    accompanying_canvas.width = width
+
+    return accompanying_canvas
 
 def addRendering(manifest, identifier, files):
     manifest.rendering = []
@@ -784,6 +838,18 @@ def create_manifest3(identifier, domain=None, page=None):
                             label={"none": [format]},
                             duration=float(file['length']))
                         body.items.append(r)
+
+                if "Spectrogram" in derivatives[file['name']]:        
+                    c.seeAlso = [{
+                        "id": f"https://archive.org/download/{identifier}/{normalised_id.replace(' ', '%20')}_spectrogram.png",
+                        "type": "Image",
+                        "label": {"en": ["Spectrogram"]},
+                        "format": "image/png"
+                    }]
+
+                if "PNG" in derivatives[file['name']]:   
+                    # This should be the Wave form
+                    c.accompanyingCanvas = addWaveform(identifier, slugged_id, derivatives[file['name']]["PNG"]["name"])
             else:
                 # todo: deal with instances where there are no derivatives for whatever reason
                 body = ResourceItem(

tests/test_audio.py

@@ -18,6 +18,24 @@ def test_audio_no_derivatives(self):
 
         self.assertEqual(len(manifest['items']),114,f"Expected 114 canvases but got: {len(manifest['items'])}") 
 
+    def test_spectrogram_waveforms(self):
+        resp = self.test_app.get("/iiif/3/hhfbc-cyl26/manifest.json?recache=True")
+        self.assertEqual(resp.status_code, 200)
+        manifest = resp.json        
+
+        for canvas in manifest['items']:
+            self.assertTrue('seeAlso' in canvas)
+            spectrogram = canvas['seeAlso'][0]
+            self.assertEqual(spectrogram["format"], "image/png")
+            self.assertEqual(spectrogram["label"]["en"][0], "Spectrogram")
+
+            self.assertTrue('accompanyingCanvas' in canvas)
+            accCanvas = canvas['accompanyingCanvas']
+            self.assertEqual(accCanvas["type"], "Canvas")
+            self.assertEqual(accCanvas["label"]["en"][0], "Waveform")
+            self.assertTrue("height" in accCanvas and accCanvas["height"] == 200)
+            self.assertTrue("width" in accCanvas and accCanvas["width"] == 800)
+
     def test_multi_track_audio_gets_ranges(self):
         resp = self.test_app.get("/iiif/Weirdos_demo-1978/manifest.json")
         self.assertEqual(resp.status_code, 200)
@@ -33,3 +51,4 @@ def test_single_track_audio_gets_no_ranges(self):
         manifest = resp.json
 
         self.assertNotIn("structures", manifest, "Expected single file audio to have no structures or ranges.")
+