feat: add activity recognition (#435)

* Add activity recognition and Gemini image generation officially to tool list

Author: cmaloney111

tests/integ/test_tools.py

@@ -3,7 +3,7 @@
 from PIL import Image
 
 from vision_agent.tools import (
-    activity_recognition,
+    agentic_activity_recognition,
     agentic_document_extraction,
     agentic_object_detection,
     agentic_sam2_instance_segmentation,
@@ -21,7 +21,6 @@
     florence2_ocr,
     florence2_sam2_instance_segmentation,
     florence2_sam2_video_tracking,
-    flux_image_inpainting,
     gemini_image_generation,
     generate_pose_image,
     ocr,
@@ -263,16 +262,45 @@ def test_qwen2_vl_video_vqa():
     assert "cat" in result.strip()
 
 
-def test_activity_recognition():
+def test_agentic_activity_recognition_no_audio():
+    frames = [
+        np.array(Image.fromarray(ski.data.cat()).convert("RGB")) for _ in range(10)
+    ]
+    result = agentic_activity_recognition(
+        prompt="cat",
+        frames=frames,
+        with_audio=False
+    )
+    assert len(result) == 1
+    assert isinstance(result[0]["start_time"], int)
+    assert isinstance(result[0]["end_time"], int)
+    assert result[0]["location"] is not None and len(result[0]["location"]) > 0
+    assert result[0]["description"] is not None and len(result[0]["description"]) > 0
+    assert result[0]["label"] == 0
+
+
+def test_agentic_activity_recognition_multiple_activities_low_specificity():
     frames = [
         np.array(Image.fromarray(ski.data.cat()).convert("RGB")) for _ in range(5)
     ]
-    result = activity_recognition(
-        prompt="Is it there a cat in this video?",
+    result = agentic_activity_recognition(
+        prompt="cat; animal",
         frames=frames,
-        model="qwen2vl",
+        fps=1,
+        with_audio=False,
+        specificity="low",
     )
-    assert len(result) == 5
+    assert len(result) == 2
+    assert isinstance(result[0]["start_time"], int)
+    assert isinstance(result[0]["end_time"], int)
+    assert result[0]["location"] is not None and len(result[0]["location"]) > 0
+    assert result[0]["description"] is not None and len(result[0]["description"]) > 0
+    assert result[0]["label"] == 0
+    assert isinstance(result[1]["start_time"], int)
+    assert result[1]["end_time"] > 0
+    assert result[1]["location"] is not None and len(result[0]["location"]) > 0
+    assert result[1]["description"] is not None and len(result[0]["description"]) > 0
+    assert result[1]["label"] == 1
 
 
 def test_ocr():
@@ -385,23 +413,6 @@ def test_countgd_visual_object_detection_empty():
     assert result == []
 
 
-def test_flux_image_inpainting():
-    mask_image = np.zeros((32, 32), dtype=np.uint8)
-    mask_image[:4, :4] = 1
-    image = np.zeros((32, 32), dtype=np.uint8)
-
-    result = flux_image_inpainting(
-        prompt="horse",
-        image=image,
-        mask=mask_image,
-    )
-
-    assert result.shape[0] == 32
-    assert result.shape[1] == 32
-    assert result.shape[0] == image.shape[0]
-    assert result.shape[1] == image.shape[1]
-
-
 def test_gemini_image_inpainting():
     image = np.zeros((32, 32), dtype=np.uint8)
 
@@ -476,38 +487,6 @@ def test_siglip_classification():
     assert result["scores"][0] > result["scores"][2]
 
 
-def test_flux_image_inpainting_resizing_not_multiple_8():
-    mask_image = np.zeros((37, 37), dtype=np.uint8)
-    mask_image[:4, :4] = 1
-    image = np.zeros((37, 37), dtype=np.uint8)
-
-    result = flux_image_inpainting(
-        prompt="horse",
-        image=image,
-        mask=mask_image,
-    )
-
-    assert result.shape[0] == 32
-    assert result.shape[1] == 32
-    assert result.shape[0] != image.shape[0]
-    assert result.shape[1] != image.shape[1]
-
-
-def test_flux_image_inpainting_resizing_big_image():
-    mask_image = np.zeros((1200, 500), dtype=np.uint8)
-    mask_image[:100, :100] = 1
-    image = np.zeros((1200, 500), dtype=np.uint8)
-
-    result = flux_image_inpainting(
-        prompt="horse",
-        image=image,
-        mask=mask_image,
-    )
-
-    assert result.shape[0] == 512
-    assert result.shape[1] == 208
-
-
 def test_video_tracking_with_countgd():
     frames = [
         np.array(Image.fromarray(ski.data.coins()).convert("RGB")) for _ in range(5)

uv.lock

@@ -406,6 +406,29 @@ desc,doc,name
         [
             {'label': 'hello world', 'bbox': [0.1, 0.11, 0.35, 0.4], 'score': 0.99},
         ]",ocr
+"'gemini_image_generation' performs either image inpainting given an image and text prompt, or image generation given a prompt. It can be used to edit parts of an image or the entire image according to the prompt given.","gemini_image_generation(prompt: str, image: Optional[numpy.ndarray] = None) -> numpy.ndarray:
+'gemini_image_generation' performs either image inpainting given an image and text prompt, or image generation given a prompt.
+    It can be used to edit parts of an image or the entire image according to the prompt given.
+
+    Parameters:
+        prompt (str): A detailed text description guiding what should be generated
+            in the image. More detailed and specific prompts typically yield
+            better results.
+        image (np.ndarray, optional): The source image to be inpainted. The image will serve as
+            the base context for the inpainting process.
+
+    Returns:
+        np.ndarray: The generated image(s) as a numpy array in RGB format with values
+            ranging from 0 to 255.
+
+    -------
+    Example:
+        >>> # Generate inpainting
+        >>> result = gemini_image_generation(
+        ...     prompt=""a modern black leather sofa with white pillows"",
+        ...     image=image,
+        ... )
+        >>> save_image(result, ""inpainted_room.png"")",gemini_image_generation
 'qwen25_vl_images_vqa' is a tool that can answer any questions about arbitrary images including regular images or images of documents or presentations. It can be very useful for document QA or OCR text extraction. It returns text as an answer to the question.,"qwen25_vl_images_vqa(prompt: str, images: List[numpy.ndarray]) -> str:
 'qwen25_vl_images_vqa' is a tool that can answer any questions about arbitrary
     images including regular images or images of documents or presentations. It can be
@@ -439,27 +462,28 @@ desc,doc,name
     -------
         >>> qwen25_vl_video_vqa('Which football player made the goal?', frames)
         'Lionel Messi'",qwen25_vl_video_vqa
-'activity_recognition' is a tool that can recognize activities in a video given a text prompt. It can be used to identify where specific activities or actions happen in a video and returns a list of 0s and 1s to indicate the activity.,"activity_recognition(prompt: str, frames: List[numpy.ndarray], model: str = 'qwen25vl', chunk_length_frames: int = 10) -> List[float]:
-'activity_recognition' is a tool that can recognize activities in a video given a
-    text prompt. It can be used to identify where specific activities or actions
-    happen in a video and returns a list of 0s and 1s to indicate the activity.
+"'agentic_activity_recognition' is a tool that allows you to detect multiple activities within a video. It can be used to identify when specific activities or actions happen in a video, along with a description of the activity.","agentic_activity_recognition(prompt: str, frames: List[numpy.ndarray], fps: Optional[float] = 5, specificity: str = 'max', with_audio: bool = False) -> List[Dict[str, Any]]:
+'agentic_activity_recognition' is a tool that allows you to detect multiple activities within a video.
+    It can be used to identify when specific activities or actions happen in a video, along with a description of the activity.
 
     Parameters:
-        prompt (str): The event you want to identify, should be phrased as a question,
-            for example, ""Did a goal happen?"".
-        frames (List[np.ndarray]): The reference frames used for the question
-        model (str): The model to use for the inference. Valid values are
-            'claude-35', 'gpt-4o', 'qwen2vl'.
-        chunk_length_frames (int): length of each chunk in frames
+        prompt (str): The prompt for activity recognition. Multiple activieties can be separated by semi-colon.
+        frames (List[np.ndarray]): The list of frames corresponding to the video.
+        fps (float, optional): The frame rate per second to extract the frames at. Defaults to 5.
+        specificity (str, optional): Specificity or precision level for activity recognition - low, medium, high, max. Default is max.
+        with_audio (bool, optional): Whether to include audio processing in activity recognition. Set it to false if there is no audio in the video. Default is false.
 
     Returns:
-        List[float]: A list of floats with a value of 1.0 if the activity is detected in
-            the chunk_length_frames of the video.
+        List[Dict[str, Any]]: A list of dictionaries containing the start time, end time, location, description, and label for each detected activity.
+            The start and end times are in seconds, the location is a string, the description is a string, and the label is an integer.
 
     Example
     -------
-        >>> activity_recognition('Did a goal happened?', frames)
-        [0.0, 0.0, 0.0, 1.0, 1.0, 0.0]",activity_recognition
+        >>> agentic_activity_recognition('Person gets on bike; Person gets off bike', frames)
+        [
+            {'start_time': 2, 'end_time': 4, 'location': 'Outdoor area', 'description': 'A person approaches a white bicycle parked in a row. The person then swings their leg over the bike and gets on it.', 'label': 0},
+            {'start_time': 10, 'end_time': 13, 'location': 'Outdoor area', 'description': 'A person gets off a white bicycle parked in a row. The person swings their leg over the bike and dismounts.', 'label': 1},
+        ]",agentic_activity_recognition
 'depth_anything_v2' is a tool that runs depth anything v2 model to generate a depth image from a given RGB image. The returned depth image is monochrome and represents depth values as pixel intensities with pixel values ranging from 0 to 255.,"depth_anything_v2(image: numpy.ndarray) -> numpy.ndarray:
 'depth_anything_v2' is a tool that runs depth anything v2 model to generate a
     depth image from a given RGB image. The returned depth image is monochrome and
@@ -514,59 +538,6 @@ desc,doc,name
     -------
         >>> vit_nsfw_classification(image)
         {""label"": ""normal"", ""scores"": 0.68},",vit_nsfw_classification
-"'flux_image_inpainting' performs image inpainting to fill the masked regions, given by mask, in the image, given image based on the text prompt and surrounding image context. It can be used to edit regions of an image according to the prompt given.","flux_image_inpainting(prompt: str, image: numpy.ndarray, mask: numpy.ndarray) -> numpy.ndarray:
-'flux_image_inpainting' performs image inpainting to fill the masked regions,
-    given by mask, in the image, given image based on the text prompt and surrounding
-    image context. It can be used to edit regions of an image according to the prompt
-    given.
-
-    Parameters:
-        prompt (str): A detailed text description guiding what should be generated
-            in the masked area. More detailed and specific prompts typically yield
-            better results.
-        image (np.ndarray): The source image to be inpainted. The image will serve as
-            the base context for the inpainting process.
-        mask (np.ndarray): A binary mask image with 0's and 1's, where 1 indicates
-            areas to be inpainted and 0 indicates areas to be preserved.
-
-    Returns:
-        np.ndarray: The generated image(s) as a numpy array in RGB format with values
-            ranging from 0 to 255.
-
-    -------
-    Example:
-        >>> # Generate inpainting
-        >>> result = flux_image_inpainting(
-        ...     prompt=""a modern black leather sofa with white pillows"",
-        ...     image=image,
-        ...     mask=mask,
-        ... )
-        >>> save_image(result, ""inpainted_room.png"")
-    ",flux_image_inpainting
-"'gemini_image_generation' performs image inpainting given an image and text prompt. It can be used to edit parts of an image or the entire image according to the prompt given.","gemini_image_generation(prompt: str, image: numpy.ndarray) -> numpy.ndarray:
-'gemini_image_generation' performs image inpainting given an image and text prompt.
-    It can be used to edit parts of an image or the entire image according to the prompt given.
-
-    Parameters:
-        prompt (str): A detailed text description guiding what should be generated
-            in the image. More detailed and specific prompts typically yield
-            better results.
-        image (np.ndarray): The source image to be inpainted. The image will serve as
-            the base context for the inpainting process.
-
-    Returns:
-        np.ndarray: The generated image(s) as a numpy array in RGB format with values
-            ranging from 0 to 255.
-
-    -------
-    Example:
-        >>> # Generate inpainting
-        >>> result = gemini_image_generation(
-        ...     prompt="a modern black leather sofa with white pillows",
-        ...     image=image,
-        ... )
-        >>> save_image(result, ""inpainted_room.png"")
-    ",gemini_image_generation
 'siglip_classification' is a tool that can classify an image or a cropped detection given a list of input labels or tags. It returns the same list of the input labels along with their probability scores based on image content.,"siglip_classification(image: numpy.ndarray, labels: List[str]) -> Dict[str, Any]:
 'siglip_classification' is a tool that can classify an image or a cropped detection given a list
     of input labels or tags. It returns the same list of the input labels along with
@@ -718,4 +689,4 @@ desc,doc,name
                     [0, 0, 0, ..., 0, 0, 0],
                     [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),
             }],
-        )",overlay_segmentation_masks
+        )",overlay_segmentation_masks

vision_agent/.sim_tools/df.csv

@@ -519,7 +519,7 @@ def countgd_sam2_video_tracking(prompt: str, frames: List[np.ndarray], chunk_len
 - "video object tracking" - tracking objects in a video.
 - "depth and pose estimation" - estimating the depth or pose of objects in an image.
 - "activity recognition" - identifying time period(s) an event occurs in a video.
-- "inpainting" - filling in masked parts of an image.
+- "image generation" - generating images from a text prompt.
 
 Return the category or categories (comma separated) inside tags <category># your categories here</category>. If you are unsure about a task, it is better to include more categories than less.
 """

vision_agent/.sim_tools/embs.npy

@@ -55,7 +55,7 @@ def test_detect_dogs():
 - Pose estimation
 - Visual question answering for both images and videos
 - Activity recognition in videos
-- Image inpainting
+- Image generation
 
 How can I help you?</response>
 --- END EXAMPLE2 ---

vision_agent/agent/vision_agent_planner_prompts_v2.py

@@ -7,7 +7,7 @@
 from .planner_tools import judge_od_results
 from .prompts import CHOOSE_PARAMS, SYSTEM_PROMPT
 from .tools import (
-    activity_recognition,
+    agentic_activity_recognition,
     agentic_document_extraction,
     agentic_object_detection,
     agentic_sam2_instance_segmentation,
@@ -30,7 +30,6 @@
     florence2_ocr,
     florence2_sam2_instance_segmentation,
     florence2_sam2_video_tracking,
-    flux_image_inpainting,
     gemini_image_generation,
     generate_pose_image,
     get_tools,