Result: add mutable extra dict for arbitrary attributes; objective: add MinimizeAttribute/MaximizeAttribute (works with built-ins or extra keys); docs+example; bump DB version to 0.1; tests

Author: jakepu

README.md

@@ -98,3 +98,41 @@ OpenTuner is supported in part by the United States Department of Energy
 [xstack]: http://science.energy.gov/ascr/research/computer-science/ascr-x-stack-portfolio/
 [dtec]: http://www.dtec-xstack.org/
 
+## Attaching custom attributes to Result
+
+You can now attach arbitrary metadata to each `Result` via a mutable `extra` dict.
+
+Example:
+
+```python
+from opentuner import Result
+
+# return a result with extra metrics
+return Result(time=elapsed_seconds).update_attributes({
+  'throughput': qps,
+  'build_hash': git_sha,
+})
+
+# or later, after creating a result instance
+result.set_attribute('notes', 'warm cache')
+print(result.get_attribute('throughput'))
+```
+
+The `extra` field is stored in the database using a compressed pickle and is tracked for in-place mutations, so updating keys will be persisted automatically on commit.
+
+### Using custom attributes as metrics
+
+You can drive the search by any built-in `Result` field or a key in `Result.extra` using the new flexible objectives:
+
+```python
+from opentuner.search.objective import MinimizeAttribute, MaximizeAttribute
+
+# Example: minimize a custom latency value stored in Result.extra['p95_ms']
+objective = MinimizeAttribute('p95_ms', missing_value=float('inf'))
+
+# Or maximize a custom throughput stored in Result.extra['qps']
+objective = MaximizeAttribute('qps', missing_value=float('-inf'))
+```
+
+If the attribute name matches a concrete column (e.g., `time`, `accuracy`), ordering is done directly in SQL. Otherwise, ordering falls back to in-Python comparisons.
+

examples/py_api/custom_metric_example.py

@@ -0,0 +1,32 @@
+import random
+
+import opentuner
+from opentuner import MeasurementInterface
+from opentuner import Result
+from opentuner.search.manipulator import ConfigurationManipulator, IntegerParameter
+from opentuner.search.objective import MaximizeAttribute
+
+
+class CustomMetricExample(MeasurementInterface):
+    def manipulator(self):
+        m = ConfigurationManipulator()
+        m.add_parameter(IntegerParameter('threads', 1, 8))
+        return m
+
+    def objective(self):
+        return MaximizeAttribute('qps', missing_value=float('-inf'))
+
+    def compile_and_run(self, desired_result, input, limit):
+        cfg = desired_result.configuration.data
+        threads = cfg['threads']
+        # Synthetic: qps grows with threads but has diminishing returns and noise
+        base_qps = 1000.0 * (1 - 0.1 / max(1, threads))
+        noise = random.uniform(-10, 10)
+        qps = base_qps + noise
+        time = 1.0 / max(1, qps)  # smaller time for higher qps, just for demo
+        return Result(time=time).update_attributes({'qps': qps, 'threads': threads})
+
+
+if __name__ == '__main__':
+    argparser = opentuner.default_argparser()
+    CustomMetricExample.main(argparser.parse_args()) 

opentuner/resultsdb/connect.py

@@ -11,7 +11,7 @@
 
 log = logging.getLogger(__name__)
 
-DB_VERSION = "0.0"
+DB_VERSION = "0.1"
 
 if False:  # profiling of queries
     import atexit

opentuner/resultsdb/models.py

@@ -14,6 +14,7 @@
 
 from pickle import dumps, loads
 from gzip import zlib
+from sqlalchemy.ext.mutable import MutableDict
 
 
 class CompressedPickler(object):
@@ -263,10 +264,31 @@ class Result(Base):
     size = Column(Float)
     confidence = Column(Float)
     # extra = Column(PickleType)
+    extra = Column(MutableDict.as_mutable(PickleType(pickler=CompressedPickler)), default=dict)
 
     # set by SearchDriver
     was_new_best = Column(Boolean)
 
+    # Convenience helpers for per-result arbitrary attributes
+    def set_attribute(self, key, value):
+        if self.extra is None:
+            self.extra = {}
+        self.extra[key] = value
+        return self
+
+    def get_attribute(self, key, default=None):
+        if self.extra is None:
+            return default
+        return self.extra.get(key, default)
+
+    def update_attributes(self, mapping):
+        if not mapping:
+            return self
+        if self.extra is None:
+            self.extra = {}
+        self.extra.update(mapping)
+        return self
+
 
 Index('ix_result_custom1', Result.tuning_run_id, Result.was_new_best)
 

opentuner/search/objective.py

@@ -299,3 +299,77 @@ def result_relative(self, result1, result2):
         log.warning('result_relative() not yet implemented for %s',
                     self.__class__.__name__)
         return None
+
+
+class MinimizeAttribute(SearchObjective):
+    """
+    Minimize an attribute on Result. If the attribute is not a concrete
+    database column, it will be read from Result.extra at runtime.
+    """
+
+    def __init__(self, attribute_name, missing_value=None):
+        super(MinimizeAttribute, self).__init__()
+        self.attribute_name = attribute_name
+        self.missing_value = missing_value
+
+    def result_order_by_terms(self):
+        # Use DB ordering only if this is a concrete column on Result
+        col = getattr(Result, self.attribute_name, None)
+        return [col] if col is not None else []
+
+    def _value_of(self, result):
+        if hasattr(result, self.attribute_name):
+            return getattr(result, self.attribute_name)
+        return (result.get_attribute(self.attribute_name, self.missing_value)
+                if hasattr(result, 'get_attribute') else self.missing_value)
+
+    def result_compare(self, result1, result2):
+        return cmp(self._value_of(result1), self._value_of(result2))
+
+    def result_relative(self, result1, result2):
+        v1 = self._value_of(result1)
+        v2 = self._value_of(result2)
+        try:
+            if v1 is None or v2 in (None, 0):
+                return None
+            return old_div(v1, v2)
+        except Exception:
+            return None
+
+
+class MaximizeAttribute(SearchObjective):
+    """
+    Maximize an attribute on Result. If the attribute is not a concrete
+    database column, it will be read from Result.extra at runtime.
+    """
+
+    def __init__(self, attribute_name, missing_value=None):
+        super(MaximizeAttribute, self).__init__()
+        self.attribute_name = attribute_name
+        self.missing_value = missing_value
+
+    def result_order_by_terms(self):
+        # Use DB ordering only if this is a concrete column on Result
+        col = getattr(Result, self.attribute_name, None)
+        return [-col] if col is not None else []
+
+    def _value_of(self, result):
+        if hasattr(result, self.attribute_name):
+            return getattr(result, self.attribute_name)
+        return (result.get_attribute(self.attribute_name, self.missing_value)
+                if hasattr(result, 'get_attribute') else self.missing_value)
+
+    def result_compare(self, result1, result2):
+        # note opposite order for maximize
+        return cmp(self._value_of(result2), self._value_of(result1))
+
+    def result_relative(self, result1, result2):
+        # For maximize, relative goodness mirrors MaximizeAccuracy
+        v1 = self._value_of(result1)
+        v2 = self._value_of(result2)
+        try:
+            if v1 in (None, 0) or v2 is None:
+                return None
+            return old_div(v2, v1)
+        except Exception:
+            return None

tests/test_custom_metric.py

@@ -0,0 +1,29 @@
+import math
+
+from opentuner.resultsdb.models import Result, Configuration, Program
+from opentuner.search.objective import MaximizeAttribute, MinimizeAttribute
+
+
+def test_result_extra_persistence(tmp_path, Session=None):
+    # Create a couple of Results and ensure extra dict persists values
+    r1 = Result(time=1.0)
+    r1.set_attribute('qps', 100.0)
+    assert r1.get_attribute('qps') == 100.0
+    r1.update_attributes({'latency_ms': 10.0})
+    assert r1.get_attribute('latency_ms') == 10.0
+
+
+def test_objective_with_custom_metric_compare():
+    # Two results with only extra metric
+    r1 = Result(time=2.0).update_attributes({'qps': 100.0})
+    r2 = Result(time=2.0).update_attributes({'qps': 120.0})
+
+    max_qps = MaximizeAttribute('qps', missing_value=float('-inf'))
+    max_qps.set_driver(type('D', (), {})())  # minimal driver stub
+    assert max_qps.result_compare(r2, r1) < 0  # r2 better than r1
+
+    min_lat = MinimizeAttribute('latency_ms', missing_value=float('inf'))
+    min_lat.set_driver(type('D', (), {})())
+    r3 = Result(time=1.0).update_attributes({'latency_ms': 5.0})
+    r4 = Result(time=1.0).update_attributes({'latency_ms': 10.0})
+    assert min_lat.result_compare(r3, r4) < 0  # r3 better (lower latency)