Merge pull request #226 from czgdp1807/zetasql_dep

Deprecate `ZetaSQL`-based `filter_query` functionality for `v1.18.0` removal

Author: aktech

RELEASE.md

@@ -6,6 +6,12 @@
 
 ## Deprecations
 
+*   Deprecate ZetaSQL-based filter_query functionality. The `filter_query`
+    parameter in ListOperationOptions that relies on ZetaSQL for declarative
+    filtering is deprecated and will be removed in version 1.18.0. ZetaSQL
+    dependency is being removed from ML Metadata. Users should migrate to
+    alternative filtering approaches before the 1.18.0 release.
+
 ## Bug Fixed and Other Changes
 
 # Version 1.17.0

docs/index.md

@@ -253,6 +253,7 @@ artifacts = store.get_artifacts()
 # Plus, there are many ways to query the same Artifact
 [stored_data_artifact] = store.get_artifacts_by_id([data_artifact_id])
 artifacts_with_uri = store.get_artifacts_by_uri(data_artifact.uri)
+# Note: filter_query is deprecated and will be removed in version 1.18.0.
 artifacts_with_conditions = store.get_artifacts(
       list_options=mlmd.ListOptions(
           filter_query='uri LIKE "%/data" AND properties.day.int_value > 0'))
@@ -270,6 +271,7 @@ trainer_run.properties["state"].string_value = "RUNNING"
 # Query all registered Execution
 executions = store.get_executions_by_id([run_id])
 # Similarly, the same execution can be queried with conditions.
+# Note: filter_query is deprecated and will be removed in version 1.18.0.
 executions_with_conditions = store.get_executions(
     list_options = mlmd.ListOptions(
         filter_query='type = "Trainer" AND properties.state.string_value IS NOT NULL'))
@@ -355,6 +357,7 @@ experiment_executions = store.get_executions_by_context(experiment_id)
 
 # You can also use neighborhood queries to fetch these artifacts and executions
 # with conditions.
+# Note: filter_query is deprecated and will be removed in version 1.18.0.
 experiment_artifacts_with_conditions = store.get_artifacts(
     list_options = mlmd.ListOptions(
         filter_query=('contexts_a.type = "Experiment" AND contexts_a.name = "exp1"')))

ml_metadata/metadata_store/metadata_store.py

@@ -81,9 +81,13 @@ class ListOptions:
     is_asc: Specifies `order_by` is ascending or descending. If `order_by` is
       not given, the field is ignored. If `order_by` is set, then by default
       ascending order is used for performance benefit.
-    filter_query: An optional boolean expression in SQL syntax to specify
-      conditions on node attributes and directly connected assets. See
-      https://github.com/google/ml-metadata/blob/master/ml_metadata/proto/metadata_store.proto#L705-L783 for the query capabilities and syntax.
+    filter_query: DEPRECATED (will be removed in v1.18.0) - An optional boolean
+      expression in SQL syntax to specify conditions on node attributes and
+      directly connected assets. This feature depends on ZetaSQL which is being
+      removed from ML Metadata. Please migrate to alternative filtering approaches
+      before version 1.18.0. See
+      https://github.com/google/ml-metadata/blob/master/ml_metadata/proto/metadata_store.proto#L705-L783
+      for the query capabilities and syntax.
   """
 
   limit: Optional[int] = None
@@ -1478,6 +1482,16 @@ def _call_method_with_list_options(
       if list_options.order_by:
         request.options.order_by_field.field = list_options.order_by.value
       if list_options.filter_query:
+        # DEPRECATED: ZetaSQL-based filter_query will be removed in v1.18.0
+        import warnings
+        warnings.warn(
+            'DEPRECATION WARNING: filter_query is deprecated and will be '
+            'removed in version 1.18.0. This feature depends on ZetaSQL '
+            'which is being removed from ML Metadata. Please migrate to '
+            'alternative filtering approaches before the 1.18.0 release.',
+            DeprecationWarning,
+            stacklevel=3
+        )
         request.options.filter_query = list_options.filter_query
 
     result = []

ml_metadata/metadata_store/metadata_store_test.py

@@ -15,6 +15,7 @@
 import collections
 import os
 import uuid
+import warnings
 
 import pytest
 from absl.testing import absltest, parameterized
@@ -1803,14 +1804,20 @@ def test_get_nodes_by_filter_query(self, create_type_fn, put_type_fn,
       nodes[i].custom_properties["p"].int_value = i
     node_ids = put_nodes_fn(store, nodes)
 
-    got_nodes = get_nodes_fn(
-        store,
-        list_options=mlmd.ListOptions(
-            order_by=mlmd.OrderByField.ID,
-            is_asc=True,
-            filter_query=("custom_properties.p.int_value < 21 AND "
-                          "name LIKE 'node_2%'")
-        ))
+    # Verify deprecation warning is raised when using filter_query
+    with self.assertWarns(DeprecationWarning) as warning_context:
+      got_nodes = get_nodes_fn(
+          store,
+          list_options=mlmd.ListOptions(
+              order_by=mlmd.OrderByField.ID,
+              is_asc=True,
+              filter_query=("custom_properties.p.int_value < 21 AND "
+                            "name LIKE 'node_2%'")
+          ))
+    # Verify the warning message mentions version 1.18.0
+    self.assertIn("1.18.0", str(warning_context.warning))
+    self.assertIn("filter_query", str(warning_context.warning))
+
     self.assertLen(got_nodes, 2)
     self.assertEqual(got_nodes[0].id, node_ids[2])
     self.assertEqual(got_nodes[0].name, "node_2")
@@ -1822,9 +1829,15 @@ def test_get_nodes_by_filter_query(self, create_type_fn, put_type_fn,
                             (mlmd.MetadataStore.get_contexts))
   def test_get_nodes_by_filter_query_syntax_errors(self, get_nodes_fn):
     store = _get_metadata_store(self.cli_args)
-    with self.assertRaises(errors.InvalidArgumentError):
+    # Verify deprecation warning is raised even for syntax errors
+    with (
+        self.assertWarns(DeprecationWarning) as warning_context,
+        self.assertRaises(errors.InvalidArgumentError),
+    ):
       _ = get_nodes_fn(
           store, list_options=mlmd.ListOptions(filter_query="invalid syntax"))
+    # Verify the warning message mentions version 1.18.0
+    self.assertIn("1.18.0", str(warning_context.warning))
 
   def test_put_contexts_get_context_by_type_and_name(self):
     # Prepare test data.
@@ -2112,12 +2125,15 @@ def test_put_lineage_subgraph_get_lineage_subgraph(self):
     # Test get_lineage_subgraph() with max_num_hops = 10 and field mask paths =
     # ["events", "associations", "attributions"], the whole lineage subgraph
     # skeleton will be returned.
-    query_options = metadata_store_pb2.LineageSubgraphQueryOptions(
-        starting_artifacts=metadata_store_pb2.LineageSubgraphQueryOptions.StartingNodes(
-            filter_query="uri = 'output_artifact'"
-        ),
-        max_num_hops=10,
-    )
+    # Note: filter_query is deprecated but tested here for backward compatibility.
+    with warnings.catch_warnings():
+      warnings.filterwarnings("ignore", category=DeprecationWarning)
+      query_options = metadata_store_pb2.LineageSubgraphQueryOptions(
+          starting_artifacts=metadata_store_pb2.LineageSubgraphQueryOptions.StartingNodes(
+              filter_query="uri = 'output_artifact'"
+          ),
+          max_num_hops=10,
+      )
 
     subgraph_skeleton = store.get_lineage_subgraph(
         query_options, ["events", "associations", "attributions"]
@@ -2169,12 +2185,15 @@ def test_put_lineage_subgraph_get_lineage_subgraph(self):
 
     # Test get_lineage_subgraph() with max_num_hops = 0 from starting executions
     # filtered by context name. All the executions will be returned.
-    query_options = metadata_store_pb2.LineageSubgraphQueryOptions(
-        starting_executions=metadata_store_pb2.LineageSubgraphQueryOptions.StartingNodes(
-            filter_query="contexts_a.name='existing_context'"
-        ),
-        max_num_hops=0,
-    )
+    # Note: filter_query is deprecated but tested here for backward compatibility.
+    with warnings.catch_warnings():
+      warnings.filterwarnings("ignore", category=DeprecationWarning)
+      query_options = metadata_store_pb2.LineageSubgraphQueryOptions(
+          starting_executions=metadata_store_pb2.LineageSubgraphQueryOptions.StartingNodes(
+              filter_query="contexts_a.name='existing_context'"
+          ),
+          max_num_hops=0,
+      )
     subgraph = store.get_lineage_subgraph(query_options)
     self.assertEmpty(subgraph.artifacts)
     self.assertLen(subgraph.executions, 2)
@@ -2195,12 +2214,15 @@ def test_put_lineage_subgraph_get_lineage_subgraph(self):
     self.assertEmpty(subgraph.attributions)
 
     # Test get_lineage_subgraph() with various field mask paths.
-    query_options = metadata_store_pb2.LineageSubgraphQueryOptions(
-        starting_artifacts=metadata_store_pb2.LineageSubgraphQueryOptions.StartingNodes(
-            filter_query="uri = 'output_artifact'"
-        ),
-        max_num_hops=10,
-    )
+    # Note: filter_query is deprecated but tested here for backward compatibility.
+    with warnings.catch_warnings():
+      warnings.filterwarnings("ignore", category=DeprecationWarning)
+      query_options = metadata_store_pb2.LineageSubgraphQueryOptions(
+          starting_artifacts=metadata_store_pb2.LineageSubgraphQueryOptions.StartingNodes(
+              filter_query="uri = 'output_artifact'"
+          ),
+          max_num_hops=10,
+      )
 
     subgraph = store.get_lineage_subgraph(
         query_options, ["artifact_types", "execution_types", "context_types"]
@@ -2291,13 +2313,16 @@ def test_put_lineage_subgraph_get_lineage_subgraph_with_direction(self):
     )
 
     # Test get_lineage_subgraph() with direction.
-    query_options = metadata_store_pb2.LineageSubgraphQueryOptions(
-        starting_executions=metadata_store_pb2.LineageSubgraphQueryOptions.StartingNodes(
-            filter_query="name = 'test_execution'"
-        ),
-        max_num_hops=2,
-        direction=metadata_store_pb2.LineageSubgraphQueryOptions.Direction.DOWNSTREAM,
-    )
+    # Note: filter_query is deprecated but tested here for backward compatibility.
+    with warnings.catch_warnings():
+      warnings.filterwarnings("ignore", category=DeprecationWarning)
+      query_options = metadata_store_pb2.LineageSubgraphQueryOptions(
+          starting_executions=metadata_store_pb2.LineageSubgraphQueryOptions.StartingNodes(
+              filter_query="name = 'test_execution'"
+          ),
+          max_num_hops=2,
+          direction=metadata_store_pb2.LineageSubgraphQueryOptions.Direction.DOWNSTREAM,
+      )
     subgraph = store.get_lineage_subgraph(query_options)
     self.assertLen(subgraph.artifacts, 1)
     self.assertLen(subgraph.executions, 1)

ml_metadata/metadata_store/postgresql_query_executor.cc

@@ -885,6 +885,12 @@ absl::Status PostgreSQLQueryExecutor::ListNodeIDsUsingOptions(
   }
 
   if (options.has_filter_query() && !options.filter_query().empty()) {
+    // DEPRECATED: ZetaSQL-based filter_query is deprecated and will be removed
+    // in version 1.18.0. This feature depends on ZetaSQL which is being phased
+    // out. Please migrate to alternative filtering approaches.
+    LOG(WARNING) << "DEPRECATION WARNING: ZetaSQL-based filter_query is "
+                 << "deprecated and will be removed in version 1.18.0. "
+                 << "This feature depends on ZetaSQL which is being removed.";
     node_table_alias = ml_metadata::FilterQueryBuilder<Node>::kBaseTableAlias;
     ml_metadata::FilterQueryAstResolver<Node> ast_resolver(
         options.filter_query());

ml_metadata/metadata_store/query_config_executor.cc

@@ -826,6 +826,12 @@ absl::Status QueryConfigExecutor::ListNodeIDsUsingOptions(
   }
 
   if (options.has_filter_query() && !options.filter_query().empty()) {
+    // DEPRECATED: ZetaSQL-based filter_query is deprecated and will be removed
+    // in version 1.18.0. This feature depends on ZetaSQL which is being phased
+    // out. Please migrate to alternative filtering approaches.
+    LOG(WARNING) << "DEPRECATION WARNING: ZetaSQL-based filter_query is "
+                 << "deprecated and will be removed in version 1.18.0. "
+                 << "This feature depends on ZetaSQL which is being removed.";
     node_table_alias = ml_metadata::FilterQueryBuilder<Node>::kBaseTableAlias;
     ml_metadata::FilterQueryAstResolver<Node> ast_resolver(
         options.filter_query());

ml_metadata/query/filter_query_ast_resolver.cc

@@ -12,6 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+
+// DEPRECATED: This file and its associated ZetaSQL-based filter_query
+// functionality is deprecated and will be removed in version 1.18.0.
+// ZetaSQL dependency is being phased out from ML Metadata.
+// Please migrate to alternative filtering approaches.
+
 #include "ml_metadata/query/filter_query_ast_resolver.h"
 #include <vector>
 

ml_metadata/query/filter_query_ast_resolver.h

@@ -22,6 +22,11 @@ limitations under the License.
 
 namespace ml_metadata {
 
+// DEPRECATED: This class and its associated ZetaSQL-based filter_query
+// functionality is deprecated and will be removed in version 1.18.0.
+// ZetaSQL dependency is being phased out from ML Metadata.
+// Please migrate to alternative filtering approaches.
+//
 // FilterQueryAstResolver parses the MLMD filtering query string and generates
 // an AST via ZetaSQL analyzer. It can be instantiated with MLMD nodes types:
 // Artifact, Execution and Context.

ml_metadata/query/filter_query_builder.cc

@@ -12,6 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+
+// DEPRECATED: This file and its associated ZetaSQL-based filter_query
+// functionality is deprecated and will be removed in version 1.18.0.
+// ZetaSQL dependency is being phased out from ML Metadata.
+// Please migrate to alternative filtering approaches.
+
 #include "ml_metadata/query/filter_query_builder.h"
 
 #include <glog/logging.h>

ml_metadata/query/filter_query_builder.h

@@ -21,6 +21,11 @@ limitations under the License.
 
 namespace ml_metadata {
 
+// DEPRECATED: This class and its associated ZetaSQL-based filter_query
+// functionality is deprecated and will be removed in version 1.18.0.
+// ZetaSQL dependency is being phased out from ML Metadata.
+// Please migrate to alternative filtering approaches.
+//
 // FilterQueryBuilder is a ZetaSQL AST Visitor. It walks through a ZetaSQL
 // boolean expression AST parsed from a filtering query string and generates
 // FROM clauses and WHERE clauses which can be used by MLMD query executors. It