Merge pull request #6 from UTAustin-SwarmLab/0.0.2

0.0.2

Author: minkyu-choi07

Makefile

@@ -8,7 +8,7 @@ BASE_IMG=nvidia/cuda:11.8.0-devel-ubuntu20.04
 CODE_PATH := /home/mc76728/repos/Video-to-Automaton/
 
 # Custom Image
-MY_DOCKER_IMG := ${user}video_to_automaton_test
+MY_DOCKER_IMG := ${user}ns_vfs
 TAG := latest
 
 pull_docker_image:
@@ -24,7 +24,7 @@ run_docker_container:
 			   --name ${MY_DOCKER_IMG} \
 			   --cap-add=SYS_PTRACE \
 			   --ulimit core=0:0 \
-			   --volume ${CODE_PATH}:/opt/Video-to-Automoton \
+			   --volume ${CODE_PATH}:/opt/Neuro-Symbolic-Video-Frame-Search \
 			   ${MY_DOCKER_IMG}:${TAG} \
 			   /bin/bash
 
@@ -37,11 +37,11 @@ run_docker_container_gpu:
 			   --runtime=nvidia \
 			   --cap-add=SYS_PTRACE \
 			   --ulimit core=0:0 \
-			   --volume ${CODE_PATH}:/opt/Video-to-Automoton \
+			   --volume ${CODE_PATH}:/opt/Neuro-Symbolic-Video-Frame-Search \
 			   ${MY_DOCKER_IMG}:${TAG} \
-			   /bin/bash
+			   /bin/bashd
 
-exec:
+exec_docker_container:
 	docker exec -it ${MY_DOCKER_IMG} /bin/bash
 
 stop_docker_container:

README.md

@@ -1 +1,32 @@
-# Video to Automaton 
+# Neuro Symbolic Video Frame Search
+
+## Installation Guid
+**Prerequisites** <br>
+No need to worry about the prerequisites below if you are using a UT Swarm Lab cluster.
+
+* CUDA Driver: Version 11.8.0 or higher
+* Docker: Nvidia Driver
+D
+**evelopment Environment Setup**
+
+1. Clone this repository.
+2. Navigate to the makefile and modify the user input section.
+
+    For example: `CODE_PATH := /home/repos/Neuro-Symbolic-Video-Frame-Search/`
+
+3. Execute `make pull_docker_image`
+4. Execute `make build_docker_image`
+5. Execute `make run_docker_container_gpu`
+6. Execute `make exec_docker_container`
+7. Inside the container, navigate to `/opt/Neuro-Symbolic-Video-Frame-Search` 
+8. Inside the container, execute `bash install.sh`
+
+**Development Inside the Container** <br>
+Enjoy your development environment inside the container!
+
+Please avoid stopping and removing the container, as you will need to reinstall the dependencies. If the container is stopped or removed, repeat steps 5 to 8.
+
+## Dataset
+[The VIRAT Video Dataset](https://viratdata.org/#getting-data): The VIRAT Video Dataset is designed to be realistic, natural and challenging for video surveillance domains in terms of its resolution, background clutter, diversity in scenes, and human activity/event categories than existing action recognition datasets. It has become a benchmark dataset for the computer vision community.
+
+[NuScenes](https://www.nuscenes.org/nuimages#download): A public large-scale dataset for autonomous driving

docker/Dockerfile

@@ -10,20 +10,52 @@ RUN apt-get update && apt-get install -y \
     ffmpeg git vim libglew-dev libglfw3-dev libglm-dev \
     tmux wget unzip unrar curl \ 
     x11-xserver-utils xvfb \
-    gcc g++ make cmake 
+    make cmake 
 
+# RUN apt-get apt-get install -y \
+#         gcc g++ 
 #----------------------------------------------#
 # Install project dependent system dependencies
 
 ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
+
 RUN apt-get install -y \
     python3-pip python3.8-venv \
-    rustc cargo \
-    && apt-get clean 
+    build-essential libboost-all-dev libcln-dev \
+    libgmp-dev libginac-dev automake libglpk-dev \
+    libhwloc-dev libz3-dev libxerces-c-dev libeigen3-dev \
+    software-properties-common 
 
-# # Python
+# FOR GROUNDING DINO
+RUN apt-get install -y \
+    rustc cargo 
+    
+#---------------------DO NOT CHANGE BELOW---------------------#
+# STORMPY PACKAGE INSTALLATION
+RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test
+RUN apt install -y g++-11 \
+    && apt-get clean
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 60 \
+                         --slave /usr/bin/g++ g++ /usr/bin/g++-11
+# GIT CLONE
+WORKDIR /opt
+RUN git clone -b stable https://github.com/moves-rwth/carl-storm
+RUN git clone -b stable https://github.com/moves-rwth/storm.git
+RUN git clone -b stable https://github.com/moves-rwth/stormpy.git
+# INSTALL CARL STORM
+WORKDIR /opt/carl-storm
+RUN mkdir build && cd build && cmake ../ && make lib_carl -j 2
+# INSTALL STORM
+WORKDIR /opt/storm
+ENV STORM_HOME=/opt/storm
+RUN mkdir build && cd build && cmake ../ && make
+RUN export PATH=$PATH:$STORM_DIR/build/bin
+# INSTALL STORMPY
+WORKDIR /opt/stormpy
+RUN python3 -m pip install -ve .
+#---------------------DO NOT CHANGE ABOVE---------------------#
+# PYTHON TORCH
 RUN python3 -m pip install --upgrade pip build
 RUN pip install torch==2.0.0+cu118 torchvision==0.15.1+cu118 torchaudio==2.0.1 --index-url https://download.pytorch.org/whl/cu118
-
 #----------------------------------------------#
-WORKDIR /opt/Video-to-Automoton
+WORKDIR /opt/Neuro-Symbolic-Video-Frame-Search

install.sh

@@ -15,14 +15,6 @@ elif [[ ! -d $WEIGHT_DIR ]]; then
     echo "$WEIGHT_DIR already exists but is not a directory" 1>&2
 fi
 
-# # -- Installing GroundingDINO -- #
-# echo "Installing GroundingDINO"
-# git submodule init
-# git submodule update
-
-# cd $ROOT_DIR/video_to_automaton/GroundingDINO
-# git checkout -q 57535c5a79791cb76e36fdb64975271354f10251
-
 # -- Installing video_to_automaton -- #
 echo "Installing video_to_automaton"
 cd $ROOT_DIR

main.py

@@ -3,12 +3,12 @@
 from ns_vfs.config.loader import load_config
 from ns_vfs.model.vision.grounding_dino import GroundingDino
 from ns_vfs.processor.video_processor import (
-    VideoFrameProcessor,
+    VideoFrameWindowProcessor,
 )
 from ns_vfs.video_to_automaton import VideotoAutomaton
 
 if __name__ == "__main__":
-    sample_video_path = "/opt/Video-to-Automoton/artifacts/data/hmdb51/clap/Applauding_Abby_clap_u_nm_np1_fr_med_1.avi"
+    sample_video_path = "/opt/Neuro-Symbolic-Video-Frame-Search/artifacts/data/summer_event/bali_beach_club_pt1_720p.mp4"
 
     config = load_config()
 
@@ -20,10 +20,14 @@
             weight_path=config.GROUNDING_DINO.GROUNDING_DINO_CHECKPOINT_PATH,
             config_path=config.GROUNDING_DINO.GROUNDING_DINO_CONFIG_PATH,
         ),
-        video_processor=VideoFrameProcessor(video_path=sample_video_path),
-        proposition_set=["clap", "face", "baby"],
-    )
-    states, transitions, prev_states = frame2automaton.build_automaton(
-        is_annotation=False
+        video_processor=VideoFrameWindowProcessor(
+            video_path=sample_video_path,
+            artifact_dir=config.VERSION_AND_PATH.ARTIFACTS_PATH,
+        ),
+        proposition_set=["person", "drink"],
+        artifact_dir=config.VERSION_AND_PATH.ARTIFACTS_PATH,
     )
+
+    frame_window_automata = frame2automaton.run()
+
     print("Development is in progress.")

ns_vfs/common/utility.py

@@ -0,0 +1,6 @@
+from datetime import datetime
+
+
+def get_filename_with_datetime(base_name):
+    current_time = datetime.now().strftime("%Y%m%d_%H%M%S")
+    return f"{base_name}_{current_time}.png"

ns_vfs/config/__init__.py

@@ -10,5 +10,5 @@
 
 config = omegaconf.load_config_from_yaml(config_path)
 
-config.VERSION_AND_PATH.ROOT_PATH = ROOT_DIR
+config.VERSION_AND_PATH.ROOT_PATH = os.path.join(ROOT_DIR)
 config.VERSION_AND_PATH.ARTIFACTS_PATH = os.path.join(ROOT_DIR, "artifacts")

ns_vfs/data/__init__.py

@@ -0,0 +1,41 @@
+import dataclasses
+
+import numpy as np
+
+
+@dataclasses.dataclass
+class Frame:
+    """Frame class."""
+
+    frame_index: int
+    frame_image: np.ndarray
+    propositional_probability: dict = dataclasses.field(default_factory=dict)
+
+
+@dataclasses.dataclass
+class FrameWindow:
+    """Frame window class."""
+
+    frame_window_idx: int
+    frame_image_set: list = dataclasses.field(default_factory=list)
+    states: list = dataclasses.field(default_factory=list)
+    transitions: list = dataclasses.field(default_factory=list)
+    verification_result: bool = False
+
+    def get_propositional_confidence(self):
+        propositional_confidence = [
+            [] for i in range(len(self.frame_image_set[0].propositional_probability.keys()))
+        ]
+        propositional_confidence
+        for frame in self.frame_image_set:
+            frame: Frame
+            idx = 0
+            for prop in frame.propositional_probability.keys():
+                propositional_confidence[idx].append(frame.propositional_probability[prop])
+                idx += 1
+        self.propositional_confidence = propositional_confidence
+        return propositional_confidence
+
+    def update_states(self, states):
+        self.states = states
+        return states