Merge pull request #9 from TimoK93/clearmot_and_hota

Clearmot and hota

Author: TimoK93

.github/workflows/pylint.yml

@@ -7,11 +7,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

.github/workflows/python-package.yml

@@ -5,7 +5,7 @@ name: Python package
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "clearmot_and_hota" ]
   pull_request:
     branches: [ "main" ]
 
@@ -19,9 +19,9 @@ jobs:
         python-version: ["3.9", "3.10", "3.11"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

README.md

@@ -6,13 +6,20 @@
 from ctc_metrics.metrics.technical.seg import seg
 from ctc_metrics.metrics.technical.tra import tra
 from ctc_metrics.metrics.technical.det import det
+from ctc_metrics.metrics.clearmot.mota import mota
+from ctc_metrics.metrics.hota.hota import hota
+from ctc_metrics.metrics.hota.chota import chota
+from ctc_metrics.metrics.identity_metrics.idf1 import idf1
+from ctc_metrics.metrics.others.mt_ml import mtml
+from ctc_metrics.metrics.others.faf import faf
+
 from ctc_metrics.metrics.technical.op_ctb import op_ctb
 from ctc_metrics.metrics.technical.op_csb import op_csb
 from ctc_metrics.metrics.biological.bio import bio
 from ctc_metrics.metrics.biological.op_clb import op_clb
 from ctc_metrics.metrics.technical.lnk import lnk
 
 ALL_METRICS = [
-    "Valid", "BC", "CT", "CCA", "TF", "SEG", "TRA", "DET", "LNK", "OP_CTB",
-    "OP_CSB", "BIO", "OP_CLB"
-]
+    "Valid", "BC", "CT", "CCA", "TF", "SEG", "TRA", "DET", "MOTA", "HOTA",
+    "CHOTA", "IDF1", "MTML", "FAF", "LNK", "OP_CTB", "OP_CSB", "BIO", "OP_CLB"
+

ctc_metrics/metrics/__init__.py

@@ -8,10 +8,14 @@ def get_ids_that_ends_with_split(
     Extracts the ids of tracks that end with a cell split.
 
     Args:
-        tracks: The tracks to check.
+        tracks: The tracks to check. A numpy nd array with columns:
+            - label
+            - birth frame
+            - end frame
+            - parent
 
     Returns:
-        The ids of tracks that end with a cell split.
+        The ids of tracks that end with a cell split stored in a numpy.ndarray.
     """
     parents, counts = np.unique(tracks[:, 3], return_counts=True)
     counts = counts[parents > 0]
@@ -43,8 +47,8 @@ def calculate_f1_score(
 
 
 def is_matching(
-        comp: int,
-        ref: int,
+        id_comp: int,
+        id_ref: int,
         mapped_ref: list,
         mapped_comp: list,
         ref_children: np.ndarray,
@@ -56,14 +60,14 @@ def is_matching(
     Checks if the reference and the computed track match.
 
     Args:
-        comp: The computed track id.
-        ref: The reference track id.
+        id_comp: The computed track id.
+        id_ref: The reference track id.
         mapped_ref: The matched labels of the ground truth masks.
         mapped_comp: The matched labels of the result masks.
         ref_children: The children ids of the reference track.
         comp_children: The children ids of the computed track.
-        tr: The reference track end.
-        tc: The computed track end.
+        tr: The frame of the reference track end.
+        tc: The frame of the computed track end.
 
     Returns:
         True if the reference and the computed track match, False otherwise.
@@ -74,10 +78,10 @@ def is_matching(
     # Compare parents
     t1, t2 = min(tr, tc), max(tr, tc)
     mr, mc = mapped_ref[t1], mapped_comp[t1]
-    if np.sum(mc == comp) < 1 or np.sum(mr == ref) != 1:
+    if np.sum(mc == id_comp) < 1 or np.sum(mr == id_ref) != 1:
         return False
-    ind = np.argwhere(mr == ref).squeeze()
-    if mc[ind] != comp:
+    ind = np.argwhere(mr == id_ref).squeeze()
+    if mc[ind] != id_comp:
         return False
     # Compare children
     mr, mc = np.asarray(mapped_ref[t2 + 1]), np.asarray(mapped_comp[t2 + 1])
@@ -101,10 +105,26 @@ def bc(
            - Vladimir Ulman et al., Nature methods 2017
 
     Args:
-        comp_tracks: The result tracks.
-        ref_tracks: The ground truth tracks.
-        mapped_ref: The matched labels of the ground truth masks.
-        mapped_comp: The matched labels of the result masks.
+        comp_tracks: The result tracks. A (n,4) numpy ndarray with columns:
+            - label
+            - birth frame
+            - end frame
+            - parent
+        ref_tracks: The ground truth tracks. A (n,4) numpy ndarray with columns:
+            - label
+            - birth frame
+            - end frame
+            - parent
+        mapped_ref: The matched labels of the ground truth masks. A list of
+            length equal to the number of frames. Each element is a list with
+            the matched labels of the ground truth masks in the respective
+            frame. The elements are in the same order as the corresponding
+            elements in mapped_comp.
+        mapped_comp: The matched labels of the result masks. A list of length
+            equal to the number of frames. Each element is a list with the
+            matched labels of the result masks in the respective frame. The
+            elements are in the same order as the corresponding elements in
+            mapped_ref.
         i: The maximal allowed error in frames.
 
     Returns:

ctc_metrics/metrics/biological/bc.py

@@ -8,6 +8,9 @@ def bio(
     """
     Computes the BIO. As described by
         [celltrackingchallenge](http://celltrackingchallenge.net/).
+    It is the average of the CT, TF, BC, and CCA metrics. If a metric is not
+    available, it is not considered in the average.
+
 
     Args:
         ct: The complete tracking metric.
@@ -16,7 +19,8 @@ def bio(
         cca: The cell cycle accuracy metric.
 
     Returns:
-        The bio metric.
+        The BIO metric.
+
     """
     total_metrics = 0
     if ct is not None:
@@ -35,5 +39,7 @@ def bio(
         total_metrics += 1
     else:
         cca = 0
-    metric = (ct + tf + bc + cca) / total_metrics
-    return metric
+
+    bio_score = (ct + tf + bc + cca) / total_metrics
+    return bio_score
+

ctc_metrics/metrics/biological/bio.py

@@ -24,15 +24,26 @@ def is_valid_track(
     return valid
 
 
-def cca(comp_tracks: np.ndarray, ref_tracks: np.ndarray):
+def cca(
+        comp_tracks: np.ndarray,
+        ref_tracks: np.ndarray
+):
     """
     Computes the cell cycle accuracy. As described in the paper,
          "An objective comparison of cell-tracking algorithms."
            - Vladimir Ulman et al., Nature methods 2017
 
     Args:
-        comp_tracks: The result tracks.
-        ref_tracks: The ground truth tracks.
+        comp_tracks: The result tracks. A (n,4) numpy ndarray with columns:
+            - label
+            - birth frame
+            - end frame
+            - parent
+        ref_tracks: The ground truth tracks. A (n,4) numpy ndarray with columns:
+            - label
+            - birth frame
+            - end frame
+            - parent
 
     Returns:
         The cell cycle accuracy metric.

ctc_metrics/metrics/biological/cca.py

@@ -16,11 +16,29 @@ def ct(
            - Vladimir Ulman et al., Nature methods 2017
 
     Args:
-        comp_tracks: The result tracks.
-        ref_tracks: The ground truth tracks.
-        labels_ref: The labels of the ground truth masks.
-        mapped_ref: The matched labels of the ground truth masks.
-        mapped_comp: The matched labels of the result masks.
+        comp_tracks: The result tracks. A (n,4) numpy ndarray with columns:
+            - label
+            - birth frame
+            - end frame
+            - parent
+        ref_tracks: The ground truth tracks. A (n,4) numpy ndarray with columns:
+            - label
+            - birth frame
+            - end frame
+            - parent
+        labels_ref: The labels of the ground truth masks. A list of length
+            equal to the number of frames. Each element is a list with the
+            labels of the ground truth masks in the respective frame.
+        mapped_ref: The matched labels of the ground truth masks. A list of
+            length equal to the number of frames. Each element is a list with
+            the matched labels of the ground truth masks in the respective
+            frame. The elements are in the same order as the corresponding
+            elements in mapped_comp.
+        mapped_comp: The matched labels of the result masks. A list of length
+            equal to the number of frames. Each element is a list with the
+            matched labels of the result masks in the respective frame. The
+            elements are in the same order as the corresponding elements in
+            mapped_ref.
 
     Returns:
         The complete tracks metric.

ctc_metrics/metrics/biological/ct.py

@@ -61,10 +61,22 @@ def tf(
            - Vladimir Ulman et al., Nature methods 2017
 
     Args:
-        ref_tracks: The ground truth tracks.
-        labels_ref: The labels of the ground truth masks.
-        mapped_ref: The matched labels of the ground truth masks.
-        mapped_comp: The matched labels of the result masks.
+        ref_tracks: The ground truth tracks. A (n,4) numpy ndarray with columns:
+            - label
+            - birth frame
+            - end frame
+            - parent
+        labels_ref: The labels of the ground truth masks. A list of length
+        mapped_ref: The matched labels of the ground truth masks. A list of
+            length equal to the number of frames. Each element is a list with
+            the matched labels of the ground truth masks in the respective
+            frame. The elements are in the same order as the corresponding
+            elements in mapped_comp.
+        mapped_comp: The matched labels of the result masks. A list of length
+            equal to the number of frames. Each element is a list with the
+            matched labels of the result masks in the respective frame. The
+            elements are in the same order as the corresponding elements in
+            mapped_ref.
 
     Returns:
         The track fractions metric.
@@ -73,7 +85,7 @@ def tf(
         ref_tracks, labels_ref, mapped_ref, mapped_comp)
     # Calculate the track fractions with respect to the reference tracks
     tfs = {k: 0 for k in ref_tracks[:, 0]}
-    for k, v in sorted(comp_fractions.items()):
+    for _, v in sorted(comp_fractions.items()):
         for k2, v2 in sorted(v.items()):
             if tfs[k2] == 1:
                 continue

ctc_metrics/metrics/biological/tf.py

@@ -0,0 +1,67 @@
+import numpy as np
+
+
+def mota(
+        labels_ref: list,
+        labels_comp: list,
+        mapped_ref: list,
+        mapped_comp: list
+):
+    """
+    Computes the MOTA metric. As described in the paper,
+         "Evaluating Multiple Object Tracking Performance:
+          The CLEAR MOT Metrics."
+           - Keni Bernardin and Rainer Stiefelhagen, EURASIP 2008
+
+    Args:
+        labels_comp: The labels of the computed masks. A list of length equal
+            to the number of frames. Each element is a list with the labels of
+            the computed masks in the respective frame.
+        labels_ref: The labels of the ground truth masks. A list of length
+            equal to the number of frames. Each element is a list with the
+            labels of the ground truth masks in the respective frame.
+        mapped_ref: The matched labels of the ground truth masks. A list of
+            length equal to the number of frames. Each element is a list with
+            the matched labels of the ground truth masks in the respective
+            frame. The elements are in the same order as the corresponding
+            elements in mapped_comp.
+        mapped_comp: The matched labels of the result masks. A list of length
+            equal to the number of frames. Each element is a list with the
+            matched labels of the result masks in the respective frame. The
+            elements are in the same order as the corresponding elements in
+            mapped_ref.
+
+    Returns:
+        The MOTA tracks metric.
+    """
+    tp, fp, fn, idsw, multi_assignments = 0, 0, 0, 0, 0
+
+    max_label_gt = int(np.max(np.concatenate(labels_ref)))
+    matches = np.zeros(max_label_gt + 1)
+    for ref, comp, m_ref, m_comp in zip(
+            labels_ref, labels_comp, mapped_ref, mapped_comp):
+        # Calculate metrics
+        _, counts = np.unique(m_comp, return_counts=True)
+        tp += len(m_ref)
+        fn += len(ref) - len(m_ref)
+        fp += len(comp) - len(m_comp) + np.sum(counts[counts > 1] - 1)
+        multi_assignments += np.sum(counts[counts > 1] - 1)
+        idsw += np.sum((matches[m_ref] != m_comp) & (matches[m_ref] != 0))
+        # Update the match cache
+        matches[m_ref] = m_comp
+
+    mota_score = 1 - (fn + fp + idsw + multi_assignments) / (tp + fn)
+    precision = tp / (tp + fp)
+    recall = tp / (tp + fn)
+
+    res = {
+        "MOTA": mota_score,
+        "TP": tp,
+        "FP": fp,
+        "FN": fn,
+        "IDSW": idsw,
+        "MULTI-ASSIGNMENTS": multi_assignments,
+        "Precision": precision,
+        "Recall": recall
+    }
+    return res