MMLLM Experiments with ollama and OpenAI hybrid.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e675e841-ee0b-45a1-be69-e797cc5469b2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-13T10:58:07.920418Z",
     "iopub.status.busy": "2024-12-13T10:58:07.920250Z",
     "iopub.status.idle": "2024-12-13T10:58:08.245951Z",
     "shell.execute_reply": "2024-12-13T10:58:08.245606Z",
     "shell.execute_reply.started": "2024-12-13T10:58:07.920403Z"
    }
   },
   "source": [
    "Processing pipeline adapted from Taylor Arnold and Lauren Tilton's \"Explainable Search and Discovery of Visual Cultural Heritage Collections with Multimodal Large Language Models\" https://2024.computational-humanities-research.org/papers/paper28/ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9addec5f-cda4-4c91-98a7-ee94a068d4f6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-17T17:33:36.954869Z",
     "iopub.status.busy": "2024-12-17T17:33:36.954496Z",
     "iopub.status.idle": "2024-12-17T17:33:37.328123Z",
     "shell.execute_reply": "2024-12-17T17:33:37.327616Z",
     "shell.execute_reply.started": "2024-12-17T17:33:36.954844Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import tropy\n",
    "from dotenv import load_dotenv\n",
    "import numpy as np\n",
    "import faiss\n",
    "import base64\n",
    "from PIL import Image\n",
    "from typing import List, Tuple\n",
    "from openai import OpenAI\n",
    "from ollama import embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3b27076b-e71c-4e1b-b7b6-073c47a8db87",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-17T17:33:37.329214Z",
     "iopub.status.busy": "2024-12-17T17:33:37.329043Z",
     "iopub.status.idle": "2024-12-17T17:33:37.337534Z",
     "shell.execute_reply": "2024-12-17T17:33:37.337082Z",
     "shell.execute_reply.started": "2024-12-17T17:33:37.329204Z"
    }
   },
   "outputs": [],
   "source": [
    "# Configuration\n",
    "\n",
    "# Set OpenAI API key\n",
    "load_dotenv()\n",
    "client = OpenAI(\n",
    "    api_key=os.environ.get(\"OPENAI_API_KEY\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9113da5a-7fa5-438f-b785-ca61c5069211",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-17T17:33:37.338115Z",
     "iopub.status.busy": "2024-12-17T17:33:37.338003Z",
     "iopub.status.idle": "2024-12-17T17:33:37.340710Z",
     "shell.execute_reply": "2024-12-17T17:33:37.340274Z",
     "shell.execute_reply.started": "2024-12-17T17:33:37.338105Z"
    }
   },
   "outputs": [],
   "source": [
    "# Initialize a vector database (using FAISS for simplicity)\n",
    "DIMENSION = 1024  # Embedding dimension (ensure this matches your embedding model) 3072 for text-embedding-3-large; 1024 for mxbai-embed-large\n",
    "index = faiss.IndexFlatL2(DIMENSION)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7387d256-417a-42ee-b1e7-dea8dfe8cb34",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-17T17:33:37.341235Z",
     "iopub.status.busy": "2024-12-17T17:33:37.341116Z",
     "iopub.status.idle": "2024-12-17T17:33:37.343505Z",
     "shell.execute_reply": "2024-12-17T17:33:37.342903Z",
     "shell.execute_reply.started": "2024-12-17T17:33:37.341226Z"
    }
   },
   "outputs": [],
   "source": [
    "# Mapping of image embeddings to Tropy metadata\n",
    "metadata_store = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e165f7eb-40ae-4879-b66b-4aa9adf36f79",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-17T17:33:37.345623Z",
     "iopub.status.busy": "2024-12-17T17:33:37.345458Z",
     "iopub.status.idle": "2024-12-17T17:33:37.349153Z",
     "shell.execute_reply": "2024-12-17T17:33:37.348734Z",
     "shell.execute_reply.started": "2024-12-17T17:33:37.345611Z"
    }
   },
   "outputs": [],
   "source": [
    "# Function to process an image and generate a caption\n",
    "def generate_caption(image_path: str) -> str:\n",
    "    \"\"\"\n",
    "    Generate a caption for an image using OpenAI's API.\n",
    "    \"\"\"\n",
    "    # Ensure the image file is not too large\n",
    "    max_size = 500 * 1024  # 500KB\n",
    "    if os.path.getsize(image_path) > max_size:\n",
    "\n",
    "        # Resize the image to reduce file size\n",
    "        with Image.open(image_path) as img:\n",
    "            img.thumbnail((800, 800))  # Resize to fit within 800x800 pixels\n",
    "            resized_path = \"temp_resized_image.jpg\"\n",
    "            img.save(resized_path, \"JPEG\")\n",
    "            image_path = resized_path\n",
    "\n",
    "    with open(image_path, \"rb\") as image_file:\n",
    "        img_b64_str = base64.b64encode(image_file.read()).decode(\"utf-8\")\n",
    "\n",
    "    # Delete temp file if it exists\n",
    "    try:\n",
    "        os.remove(\"temp_resized_image.jpg\")\n",
    "    except OSError:\n",
    "        pass\n",
    "\n",
    "    prompt = \"Please provide a detailed plain text description of what this photograph portrays. Also provide a description of its composition as a photograph.\"\n",
    "    \n",
    "    # Construct the payload\n",
    "    response = client.chat.completions.create(\n",
    "        model=\"gpt-4o\",\n",
    "        messages=[\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": [\n",
    "                    {\"type\": \"text\", \"text\": prompt},\n",
    "                    {\n",
    "                        \"type\": \"image_url\",\n",
    "                        \"image_url\": {\n",
    "                            \"url\": f\"data:image/jpeg;base64,{img_b64_str}\"\n",
    "                        },\n",
    "                    },\n",
    "                ],\n",
    "            }\n",
    "        ],\n",
    "        max_tokens=150  # Adjust this based on desired caption length\n",
    "    )\n",
    "\n",
    "    return response.choices[0].message.content.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f0a170de-4f57-4cf9-b512-d20755c96b55",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-17T17:33:37.349703Z",
     "iopub.status.busy": "2024-12-17T17:33:37.349611Z",
     "iopub.status.idle": "2024-12-17T17:33:37.352288Z",
     "shell.execute_reply": "2024-12-17T17:33:37.351811Z",
     "shell.execute_reply.started": "2024-12-17T17:33:37.349694Z"
    }
   },
   "outputs": [],
   "source": [
    "# Function to generate embeddings for a text with ollama\n",
    "def generate_embedding(text: str) -> np.ndarray:\n",
    "    \"\"\"\n",
    "    Generate a text embedding using ollama\n",
    "    \"\"\"\n",
    "\n",
    "    response = embeddings(\n",
    "        model='mxbai-embed-large',\n",
    "        prompt=text,\n",
    "    )\n",
    "\n",
    "    return np.array(response[\"embedding\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "345d5514-b16a-44cc-a445-7bf158f75d5e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-17T17:33:37.352741Z",
     "iopub.status.busy": "2024-12-17T17:33:37.352653Z",
     "iopub.status.idle": "2024-12-17T17:33:37.366903Z",
     "shell.execute_reply": "2024-12-17T17:33:37.366658Z",
     "shell.execute_reply.started": "2024-12-17T17:33:37.352732Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1018, 1020, 1022, 1024, 1026, 1028, 1030, 1032, 1034, 1036, 1038, 1040, 1042, 1044, 1046, 1048, 1050, 1052, 1054, 1056, 1058, 1060, 1062, 1064, 1066, 1068, 1070, 1072, 1074, 1076, 1078, 1080, 1082, 1084, 1086, 1088, 1090, 1092, 1094, 1097]\n"
     ]
    }
   ],
   "source": [
    "# Retrieve all items from Tropy\n",
    "items = tropy.get_all_items()\n",
    "print(items)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8178bbc8-36e7-47af-aecf-634c00e53e01",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-17T17:33:37.367575Z",
     "iopub.status.busy": "2024-12-17T17:33:37.367387Z",
     "iopub.status.idle": "2024-12-17T17:33:37.371134Z",
     "shell.execute_reply": "2024-12-17T17:33:37.370824Z",
     "shell.execute_reply.started": "2024-12-17T17:33:37.367565Z"
    }
   },
   "outputs": [],
   "source": [
    "def process_images():\n",
    "    \"\"\"\n",
    "    Process images, generate captions and embeddings, and store them in the vector database.\n",
    "    \"\"\"\n",
    "    global metadata_store\n",
    "\n",
    "    for item in items:\n",
    "        try:\n",
    "            photos = tropy.get_item_photos(item)\n",
    "            if not photos:\n",
    "                print(f\"No photos found for item {item}\")\n",
    "                continue\n",
    "            \n",
    "            # Ensure photos is a list or iterable\n",
    "            if not isinstance(photos, list):\n",
    "                print(f\"Unexpected format for photos in item {item}: {photos}\")\n",
    "                continue\n",
    "\n",
    "            # Get the first photo -- TODO: handle multi-image / multi-page items\n",
    "            image_path = photos[0]\n",
    "\n",
    "            if not os.path.isfile(image_path):\n",
    "                print(f\"File not found: {image_path}\")\n",
    "                continue\n",
    "\n",
    "            # Generate a caption\n",
    "            caption = generate_caption(image_path)\n",
    "            caption = caption.replace(\"\\n\", \" \") \n",
    "                        \n",
    "            # Generate an embedding\n",
    "            embedding = generate_embedding(caption)\n",
    "            \n",
    "            # Add the embedding to the vector database\n",
    "            embedding_array = np.array([embedding], dtype=np.float32)\n",
    "\n",
    "            # Validate embedding before adding\n",
    "            if embedding_array.shape[1] != DIMENSION:\n",
    "                raise ValueError(f\"Embedding dimension mismatch. Expected {DIMENSION}, got {embedding_array.shape[1]}\")\n",
    "\n",
    "            index.add(embedding_array)\n",
    "                \n",
    "            # Store metadata\n",
    "            metadata_store[len(metadata_store)] = {\n",
    "                \"file_path\": image_path, # we probably don't need this\n",
    "                \"caption\": caption,\n",
    "                \"item_id\": item  # Adding Tropy item ID for traceability back to the application\n",
    "            }\n",
    "\n",
    "            \n",
    "#            print(f\"Processed {image_path}: {caption}\")\n",
    "        except ValueError as ve:\n",
    "            print(f\"ValueError while processing item {item}: {ve}\")\n",
    "        except Exception as e:\n",
    "            print(f\"Error processing item {item}: {e}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4c2b9037-2538-4030-a149-5604d80ba91d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-17T17:33:37.372013Z",
     "iopub.status.busy": "2024-12-17T17:33:37.371810Z",
     "iopub.status.idle": "2024-12-17T17:33:37.375215Z",
     "shell.execute_reply": "2024-12-17T17:33:37.374783Z",
     "shell.execute_reply.started": "2024-12-17T17:33:37.372001Z"
    }
   },
   "outputs": [],
   "source": [
    "# Function to search for images\n",
    "def search_images(query: str, top_k: int = 5) -> List[Tuple[str, str, str]]:\n",
    "    \"\"\"\n",
    "    Search for images using a natural language query.\n",
    "    Returns a list of tuples containing file_path, caption, and item_id.\n",
    "    \"\"\"\n",
    "    query_embedding = generate_embedding(query)\n",
    "\n",
    "    # Perform a similarity search\n",
    "    distances, indices = index.search(np.array([query_embedding], dtype=np.float32), top_k)\n",
    "\n",
    "    results = []\n",
    "    for i in range(len(indices[0])):\n",
    "        idx = indices[0][i]\n",
    "        if idx in metadata_store:\n",
    "            results.append((\n",
    "                metadata_store[idx][\"file_path\"],\n",
    "                metadata_store[idx][\"caption\"],\n",
    "                metadata_store[idx][\"item_id\"]\n",
    "            ))\n",
    "\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cd90a464-fd2f-44d3-a1f0-6d77257153ab",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-17T17:33:37.375722Z",
     "iopub.status.busy": "2024-12-17T17:33:37.375627Z",
     "iopub.status.idle": "2024-12-17T17:37:13.513079Z",
     "shell.execute_reply": "2024-12-17T17:37:13.512744Z",
     "shell.execute_reply.started": "2024-12-17T17:33:37.375712Z"
    }
   },
   "outputs": [],
   "source": [
    "# Let's go!\n",
    "process_images()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6fad401a-885c-4dcc-a1b5-352401b8f436",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-17T17:37:13.513579Z",
     "iopub.status.busy": "2024-12-17T17:37:13.513494Z",
     "iopub.status.idle": "2024-12-17T17:37:13.528616Z",
     "shell.execute_reply": "2024-12-17T17:37:13.528285Z",
     "shell.execute_reply.started": "2024-12-17T17:37:13.513570Z"
    }
   },
   "outputs": [],
   "source": [
    "# Perform a search\n",
    "query = \"Aerospace\"\n",
    "results = search_images(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9e817b13-656b-4d10-a573-42768c1b3e6e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-17T17:37:13.529090Z",
     "iopub.status.busy": "2024-12-17T17:37:13.529014Z",
     "iopub.status.idle": "2024-12-17T17:37:13.530963Z",
     "shell.execute_reply": "2024-12-17T17:37:13.530716Z",
     "shell.execute_reply.started": "2024-12-17T17:37:13.529081Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image: /Users/stakats/Documents/Mason/CHNM/Tropy/Training/Tropy-HAEU/ELDO-4592_08.jpg, Caption: This photograph captures the launch of a rocket, ascending vertically from a launch pad. The rocket is prominently centered in the image, depicted mid-launch with a trail of exhaust behind it, indicating motion and power. The background is a cloudless sky, serving as a stark contrast to the detailed machinery of the rocket.  The composition of the photograph emphasizes both the scale and technological prowess of the launch. The rocket is positioned slightly off-center, creating a dynamic sense of movement towards the upper part of the frame. The plume of smoke and gas extends downward, providing a visual anchor that guides the eye from the rocket to the launch pad below.  The launch pad infrastructure is visible at the bottom of the image. Its intricate framework and details highlight the industrial setting and add, Item ID: 1038\n",
      "Image: /Users/stakats/Documents/Mason/CHNM/Tropy/Training/Tropy-HAEU/ELDO-4592_01.jpg, Caption: This black-and-white photograph depicts a rocket ready for launch at a launch pad. The rocket is vertically positioned, partially obscured by a tall service tower on its right side, which is used for support and possibly fueling. The rocket itself has a distinctive design, with alternating dark and light sections and some checkered patterns near the bottom.   In the foreground, the launch pad structure is visible, with platforms, railings, and various technical apparatuses and cables connected to the rocket. This infrastructure emphasizes the complexity and technicality involved in rocket launches.  The sky in the background is mostly clear, with some soft, wispy clouds, underneath a vast expanse of open sky. The composition of the photograph is balanced with the rocket positioned centrally, drawing, Item ID: 1052\n",
      "Image: /Users/stakats/Documents/Mason/CHNM/Tropy/Training/Tropy-HAEU/ELDO-4592_03.jpg, Caption: The photograph depicts a rocket positioned vertically on a launch pad, ready for lift-off. The rocket is prominently centered in the image, flanked by a supporting structure. It is a tall, cylindrical object with distinct banding patterns that are typical of mid-20th century rocket designs. The launch pad is equipped with various scaffolding and machinery necessary for pre-launch activities.  The composition of the photograph is balanced, with the rocket and the gantry providing a strong vertical element against the broad horizontal expanse of the launch site. The sky is vast and extends across the upper portion of the image, providing contrast against the solid form of the rocket. The photo is in black and white, highlighting contrasts and giving it a historical, documentary feel. The framing, Item ID: 1048\n",
      "Image: /Users/stakats/Documents/Mason/CHNM/Tropy/Training/Tropy-HAEU/ELDO-4592_04.jpg, Caption: This black-and-white photograph captures the launch of a rocket from a ground-based platform. The rocket is vertical, with flames and exhaust visibly billowing from its engines, indicating liftoff. The rocket has a cylindrical body with distinctive patterns or markings. Beside it stands a tall launch tower, equipped with various structures and mechanical parts used to support the rocket before launch.  The background shows a clear sky, contributing to the contrast with the ascending rocket. The ground level has a series of platforms and railings, with some equipment and antennas visible, suggesting a typical launch complex setup.   In terms of composition, the photograph is centered on the rocket, which dominates the vertical space and conveys a sense of ascent and power. The tower is positioned to the, Item ID: 1046\n",
      "Image: /Users/stakats/Documents/Mason/CHNM/Tropy/Training/Tropy-HAEU/ELDO-4592_02.jpg, Caption: This black-and-white photograph depicts a rocket on its launch pad. The rocket is positioned vertically and appears to be prepped for a launch with an accompanying tower structure. Numerous cables and external structures are evident, which likely serve as support and fueling conduits.   The foreground consists of the launch platform, which includes various equipment, scaffolding, and steps, indicating the infrastructure needed for such an operation. There are poles and small structures evident, possibly for communications or surveillance.  As for the composition of the photograph, it has a balanced and centered arrangement, with the rocket as the focal point standing tall against the sky. The use of black-and-white enhances the stark contrast between the rocket and the background, emphasizing the geometric lines of the launch structure. The, Item ID: 1050\n"
     ]
    }
   ],
   "source": [
    "# Display results\n",
    "for file_path, caption, item_id in results:\n",
    "    print(f\"Image: {file_path}, Caption: {caption}, Item ID: {item_id}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d8c6845e-84b2-4c39-8499-c80b31b89bcd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-17T17:37:13.531333Z",
     "iopub.status.busy": "2024-12-17T17:37:13.531246Z",
     "iopub.status.idle": "2024-12-17T17:37:13.533037Z",
     "shell.execute_reply": "2024-12-17T17:37:13.532782Z",
     "shell.execute_reply.started": "2024-12-17T17:37:13.531324Z"
    }
   },
   "outputs": [],
   "source": [
    "matching_item_ids = [result[2] for result in results]  # Assuming item_id is the 3rd element in the tuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e4016aa8-0e47-45f6-bc9b-caf7511979ae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-17T17:37:13.535130Z",
     "iopub.status.busy": "2024-12-17T17:37:13.534979Z",
     "iopub.status.idle": "2024-12-17T17:37:13.576871Z",
     "shell.execute_reply": "2024-12-17T17:37:13.576473Z",
     "shell.execute_reply.started": "2024-12-17T17:37:13.535120Z"
    }
   },
   "outputs": [],
   "source": [
    "# Tag matching items with the search query\n",
    "for item in matching_item_ids:\n",
    "    tropy.tag_item_by_tag_name(item, [query])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}