Merge branch 'master' of https://github.com/strands-project/strands_qsr_lib

Conflicts:
	qsr_lib/scripts/mwe.py

Author: cburbridge

docs/rsts/handwritten/images/qstag.png

@@ -0,0 +1,141 @@
+QSTAG
+======
+**Qualitative Spatio-Temporal Activity Graph**
+
+A QSTAG provides a compact and efficient graph structure to represent both qualitative spatial
+and temporal information about entities, allowing the use of standard graph comparison techniques.
+
+Structure
+---------
+The structure of a QSTAG consists of three layers:
+
+• The objects layer consists of nodes representing the objects in the scene.
+
+• The spatial relations layer consists of nodes that represent QSR intervals between the object layer nodes.
+These QSR intervals are generated by merging together repeated QSR values over the time interval they hold.
+
+• The temporal relations layer consists of nodes that encode Allen interval algebra (IA) relations
+between the QSR intervals.
+
+.. image:: images/qstag.png
+   :height: 100px
+   :width: 200 px
+
+Episodes
+---------
+To begin, the QSR_World_Trace is converted into a QSR Episode representation. A QSR Episode can be thought of as an
+interval of time where a QSR holds between some objects.
+
+**The structure of an Episode:**
+
+.. code:: bash
+
+        [ [objects], {spatial relations}, (duration) ]
+
+where:
+
+ [objects] is a list of objects,
+
+ {spatial relations} is a dictionary of spatial relations which hold where the Key is the Type of QSR Relation,
+ and the value is QSR relation, i.e. {"rcc3" : "Dc"}
+
+ (duration) is a tuple containing the interval of discrete time that the spatial relations hold for,
+ comprising of (start_frame, end_frame).
+
+
+.. _usage:
+
+Usage
+-----
+
+To use, first create a QSR_World_Trace (qsrlib_response_message) in the normal way.
+
+**Standard Steps for Creating QSR_World_Trace:**
+
+• Create a QSRlib object
+
+• Convert your data in to QSRlib standard input format
+
+• Make a request to QSRlib (or a request to QSRlib using ROS)
+
+
+**Steps for Creating a QSTAG:**
+
+.. code:: python
+
+    qstag = qsrlib_response_message.qstag
+
+Note: If you want your object nodes to contain *object types* make sure to include this in the dynamic_args:
+
+Also, if you want the spatial relations only created for certain objects, use the `qsr_for` dictionary.
+
+.. code:: python
+
+    object_types = {"o1": "Human",
+                    "o2": "Chair"}
+
+    which_qsr = ["qtcbs", "argd", "mos"]
+
+    dynamic_args = {"qtcbs": {"quantisation_factor": args.quantisation_factor,
+                              "validate": args.validate,
+                              "no_collapse": args.no_collapse,
+                              "qsrs_for": [("o1", "o2"),("o1", "o3")]},
+
+                    "argd": {"qsr_relations_and_values": args.distance_threshold,
+                              "qsrs_for":  [("o1", "o2")]},
+
+                    "mos": {"qsrs_for": [("o1"), ("o2")]},
+
+                    "qstag": {"object_types" : object_types} }
+
+
+Visualize the QSTAG
+------------------------------
+
+A method to save the QSTAG as a dot file, and convert it to a png image.
+
+.. code:: python
+
+    qstag = qsrlib_response_message.qstag
+
+    qstag.graph2dot('/tmp/act_gr.dot')
+    os.system('dot -Tpng /tmp/act_gr.dot -o /tmp/act_gr.png')
+
+
+
+Parse the Episodes and QSTAG
+------------------------------
+
+.. code:: python
+
+    qstag = qsrlib_response_message.qstag
+
+    print("All the Episodes...")
+    for episode in qstag.episodes:
+        print(episode)
+
+    print("The QSTAG Graph: \n", qstag.graph)
+
+    print("All the Graph NODES:")
+    for node in qstag.graph.vs():
+        print(node)
+
+    print("All the Graph EDGES:")
+    for edge in qstag.graph.es():
+        print(edge, " from: ", edge.source, " to: ", edge.target)
+
+
+Example QSTAG code
+------------------------------
+
+An example script for generating a simple QSTAG is available in `/strands_qsr_lib/qsr_lib/scripts/`:
+
+.. code:: bash
+
+    ./qstag_example.py <qsr_name>
+
+e.g.
+
+.. code:: bash
+
+    ./qstag_example.py qtcbs

docs/rsts/handwritten/qstag.rst

@@ -17,6 +17,8 @@
   <build_depend>std_msgs</build_depend>
   <build_depend>rostest</build_depend>
   <build_depend>roslib</build_depend>
+
+  <run_depend>igraph</run_depend>
   <run_depend>message_runtime</run_depend>
   <run_depend>rospy</run_depend>
   <run_depend>std_msgs</run_depend>

qsr_lib/package.xml

@@ -26,6 +26,7 @@ def pretty_print_world_qsr_trace(which_qsr, qsrlib_response_message):
     # ****************************************************************************************************
     # parse command line arguments
     options = sorted(qsrlib.qsrs_registry.keys())
+    print(options)
     parser = argparse.ArgumentParser()
     parser.add_argument("qsr", help="choose qsr: %s" % options, type=str)
     args = parser.parse_args()
@@ -37,22 +38,30 @@ def pretty_print_world_qsr_trace(which_qsr, qsrlib_response_message):
     # ****************************************************************************************************
     # make some input data
     world = World_Trace()
-    o1 = [Object_State(name="o1", timestamp=0, x=1., y=1., xsize=5., ysize=8.),
-          Object_State(name="o1", timestamp=1, x=1., y=2., xsize=5., ysize=8.)]
-
-    o2 = [Object_State(name="o2", timestamp=0, x=11., y=1., xsize=5., ysize=8.),
-          Object_State(name="o2", timestamp=1, x=1., y=2., xsize=5., ysize=8.)]
-
+    o1 = [Object_State(name="o1", timestamp=0, x=1., y=10., xsize=5., ysize=8.),
+          Object_State(name="o1", timestamp=1, x=1., y=10., xsize=5., ysize=8.),
+          Object_State(name="o1", timestamp=2, x=3., y=1., xsize=5., ysize=8.),
+          Object_State(name="o1", timestamp=3, x=6., y=1., xsize=5., ysize=8.),
+          Object_State(name="o1", timestamp=4, x=6., y=1., xsize=5., ysize=8.)]
+    o2 = [Object_State(name="o2", timestamp=0, x=1., y=1., xsize=5., ysize=8.),
+          Object_State(name="o2", timestamp=1, x=2., y=1., xsize=5., ysize=8.),
+          Object_State(name="o2", timestamp=2, x=1., y=1., xsize=5., ysize=8.),
+          Object_State(name="o2", timestamp=3, x=1., y=1., xsize=5., ysize=8.),
+          Object_State(name="o2", timestamp=4, x=1., y=2., xsize=5., ysize=8.)]
     o3 = [Object_State(name="o3", timestamp=0, x=5., y=2., xsize=5.2, ysize=8.5),
-          Object_State(name="o3", timestamp=1, x=6., y=4., xsize=5.2, ysize=8.5)]
+          Object_State(name="o3", timestamp=1, x=6., y=4., xsize=5.2, ysize=8.5),
+          Object_State(name="o3", timestamp=2, x=2., y=4., xsize=5.2, ysize=8.5),
+          Object_State(name="o3", timestamp=3, x=1., y=4., xsize=5.2, ysize=8.5),
+          Object_State(name="o3", timestamp=4, x=0., y=4., xsize=5.2, ysize=8.5)]
 
     world.add_object_state_series(o1)
     world.add_object_state_series(o2)
     world.add_object_state_series(o3)
 
     # ****************************************************************************************************
+    dynammic_args = {"qsr_relations_and_values" : {"Touch": 0.5, "Near": 1, "Far": 3}}
     # make a QSRlib request message
-    qsrlib_request_message = QSRlib_Request_Message(which_qsr=which_qsr, input_data=world)
+    qsrlib_request_message = QSRlib_Request_Message(which_qsr, world, dynammic_args["qsr_relations_and_values"])
     # request your QSRs
     qsrlib_response_message = qsrlib.request_qsrs(req_msg=qsrlib_request_message)
 

qsr_lib/scripts/mwe.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from __future__ import print_function, division
+import argparse
+import os, sys
+import cPickle as pickle
+from qsrlib.qsrlib import QSRlib, QSRlib_Request_Message
+from qsrlib_io.world_trace import Object_State, World_Trace
+
+
+
+def pretty_print_world_qsr_trace(which_qsr, qsrlib_response_message):
+    print(which_qsr, "request was made at ", str(qsrlib_response_message.req_made_at)
+          + " and received at " + str(qsrlib_response_message.req_received_at)
+          + " and finished at " + str(qsrlib_response_message.req_finished_at))
+    print("---")
+    print(qsrlib_response_message.qsrs.get_sorted_timestamps())
+    print("Response is:")
+    for t in qsrlib_response_message.qsrs.get_sorted_timestamps():
+        foo = str(t) + ": "
+        for k, v in zip(qsrlib_response_message.qsrs.trace[t].qsrs.keys(),
+                        qsrlib_response_message.qsrs.trace[t].qsrs.values()):
+            foo += str(k) + ":" + str(v.qsr) + "; "
+        print(foo)
+
+
+
+if __name__ == "__main__":
+
+    options = sorted(QSRlib().qsrs_registry.keys()) + ["multiple"]
+    multiple = options[:]; multiple.remove("multiple"); multiple.remove("argd"); multiple.remove("argprobd")
+    multiple.remove("ra"); multiple.remove("mwe");
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("qsr", help="choose qsr: %s" % options, type=str, default='qtcbs')
+    parser.add_argument("--ros", action="store_true", default=False, help="Use ROS eco-system")
+
+    parser.add_argument("--validate", help="validate state chain. Only QTC", action="store_true", default=False)
+    parser.add_argument("--quantisation_factor", help="quantisation factor for 0-states in qtc, or 's'-states in mos", type=float, default=0.01)
+    parser.add_argument("--no_collapse", help="does not collapse similar adjacent states. Only QTC", action="store_true", default=True)
+    #parser.add_argument("--distance_threshold", help="distance threshold for qtcb <-> qtcc transition. Only QTCBC", type=float)
+
+    args = parser.parse_args()
+    args.distance_threshold = {"touch":1, "near":3, "medium":5, "far":10}
+
+    qtcbs_qsrs_for = [("o1", "o2"),("o1", "o3")]
+    argd_qsrs_for = [("o1", "o2")]
+    mos_qsrs_for = ["o1", "o2"]
+
+    object_types = {"o1": "Human",
+                    "o2": "Chair"}
+
+    if args.qsr in options:
+        if args.qsr != "multiple":
+            which_qsr = args.qsr
+        else:
+            which_qsr = multiple
+    elif args.qsr == "hardcoded":
+        which_qsr = ["qtcbs", "argd", "mos"]
+    else:
+        raise ValueError("qsr not found, keywords: %s" % options)
+
+    world = World_Trace()
+
+    dynamic_args = {"qtcbs": {"quantisation_factor": args.quantisation_factor,
+                              "validate": args.validate,
+                              "no_collapse": args.no_collapse,
+                              "qsrs_for": qtcbs_qsrs_for},
+
+                    "argd": {"qsr_relations_and_values": args.distance_threshold,
+                              "qsrs_for": argd_qsrs_for},
+
+                    "mos": {"qsrs_for": mos_qsrs_for},
+
+                    "qstag": {"object_types" : object_types} }
+
+
+    o1 = [Object_State(name="o1", timestamp=0, x=2., y=2., object_type="Person"),  #accessed first using try: kwargs["object_type"] except:
+          Object_State(name="o1", timestamp=1, x=1., y=1., object_type="Person")]
+          #Object_State(name="o1", timestamp=2, x=2., y=2., object_type="Human"),
+          #Object_State(name="o1", timestamp=3, x=4., y=1., object_type="Human"),
+          #Object_State(name="o1", timestamp=4, x=4., y=1., object_type="Human"),
+          #Object_State(name="o1", timestamp=5, x=4., y=2., object_type="Human")]
+
+    o2 = [Object_State(name="o2", timestamp=0, x=1., y=1., object_type="Chair"),
+          Object_State(name="o2", timestamp=1, x=2., y=2., object_type="Chair")]
+          #Object_State(name="o2", timestamp=2, x=1., y=5., object_type="Chair"),
+          #Object_State(name="o2", timestamp=3, x=1., y=5., object_type="Chair"),
+          #Object_State(name="o2", timestamp=4, x=2., y=5., object_type="Chair"),
+          #Object_State(name="o2", timestamp=5, x=2., y=5., object_type="Chair")]
+
+    o3 = [Object_State(name="o3", timestamp=0, x=0., y=0., object_type="Desk"),
+          Object_State(name="o3", timestamp=1, x=0., y=0., object_type="Desk")]
+          #Object_State(name="o3", timestamp=2, x=0., y=0., object_type="Desk"),
+          #Object_State(name="o3", timestamp=3, x=0., y=0., object_type="Desk"),
+          #Object_State(name="o3", timestamp=4, x=0., y=0., object_type="Desk"),
+          #Object_State(name="o3", timestamp=5, x=0., y=0., object_type="Desk")]
+
+
+    world.add_object_state_series(o1)
+    world.add_object_state_series(o2)
+    world.add_object_state_series(o3)
+    qsrlib_request_message = QSRlib_Request_Message(which_qsr=which_qsr, input_data=world, dynamic_args=dynamic_args)
+
+
+    if args.ros:
+        try:
+            import rospy
+            from qsrlib_ros.qsrlib_ros_client import QSRlib_ROS_Client
+        except ImportError:
+            raise ImportError("ROS not found")
+        client_node = rospy.init_node("qsr_lib_ros_client_example")
+        cln = QSRlib_ROS_Client()
+        req = cln.make_ros_request_message(qsrlib_request_message)
+        res = cln.request_qsrs(req)
+        qsrlib_response_message = pickle.loads(res.data)
+    else:
+        qsrlib = QSRlib()
+        qsrlib_response_message = qsrlib.request_qsrs(req_msg=qsrlib_request_message)
+
+    pretty_print_world_qsr_trace(which_qsr, qsrlib_response_message)
+
+
+    #params={'MAX_ROWS':1, 'MIN_ROWS':None, 'MAX_EPI':4, 'num_cores':8}
+    qstag = qsrlib_response_message.qstag
+
+    print("Episodes...")
+    for i in qstag.episodes:
+        print(i)
+
+    qstag.graph2dot('/tmp/act_gr.dot')
+    os.system('dot -Tpng /tmp/act_gr.dot -o /tmp/act_gr.png')
+
+    ########   PRINT THE GRAPH   #########
+    print("QSTAG Graph:\n", qstag.graph)
+    print("NODES:")
+    for node in qstag.graph.vs():
+        print(node)
+    print("EDGES:")
+    for edge in qstag.graph.es():
+        print(edge, " from: ", edge.source, " to: ", edge.target)

qsr_lib/scripts/qstag_example.py

@@ -6,7 +6,7 @@
 d = generate_distutils_setup(
     # #  don't do this unless you want a globally visible script
     # scripts=['bin/myscript'],
-    packages=['qsrlib', 'qsrlib_qsrs', 'qsrlib_ros', 'qsrlib_io', 'qsrlib_utils'],
+    packages=['qsrlib', 'qsrlib_qsrs', 'qsrlib_ros', 'qsrlib_io', 'qsrlib_utils', 'qsrlib_qstag'],
     package_dir={'': 'src'}
 )
 

qsr_lib/setup.py

@@ -5,12 +5,12 @@
 from qsrlib_io.world_trace import World_Trace
 from qsrlib_utils.utils import merge_world_qsr_traces
 from qsrlib_qsrs import *
+from qsrlib_qstag.qstag import Activity_Graph
 
 
 class QSRlib_Response_Message(object):
     """The response message of QSRlib containing the QSRs and time processing information."""
-
-    def __init__(self, qsrs, req_made_at, req_received_at, req_finished_at):
+    def __init__(self, qsrs, qstag, req_made_at, req_received_at, req_finished_at):
         """Constructor.
 
         :param qsrs: Computed QSRs in World_QSR_Trace format.
@@ -25,6 +25,9 @@ def __init__(self, qsrs, req_made_at, req_received_at, req_finished_at):
         self.qsrs = qsrs
         """:class:`World_QSR_Trace <qsrlib_io.world_qsr_trace.World_QSR_Trace>`: Holds the QSRs."""
 
+        self.qstag = qstag
+        """Activity_Graph: Qualitative Spatio-Temporal Activity Graph"""
+
         self.req_made_at = req_made_at
         """datetime: Time the request was made."""
 
@@ -153,6 +156,7 @@ def request_qsrs(self, req_msg):
             world_qsr_trace = None
 
         qsrlib_response = QSRlib_Response_Message(qsrs=world_qsr_trace,
+                                                  qstag=Activity_Graph(req_msg.input_data, world_qsr_trace,  req_msg.dynamic_args["qstag"]["object_types"] if "object_types" in req_msg.dynamic_args["qstag"] else {}) if "qstag" in req_msg.dynamic_args  else None,
                                                   req_made_at=req_msg.made_at,
                                                   req_received_at=req_received_at,
                                                   req_finished_at=datetime.now())