diff --git a/RELEASE.md b/RELEASE.md index a8455184c..5f5b00092 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -6,6 +6,12 @@ ## Deprecations +* Deprecate ZetaSQL-based filter_query functionality. The `filter_query` + parameter in ListOperationOptions that relies on ZetaSQL for declarative + filtering is deprecated and will be removed in version 1.19.0. ZetaSQL + dependency is being removed from ML Metadata. Users should migrate to + alternative filtering approaches before the 1.19.0 release. + ## Bug Fixed and Other Changes # Version 1.17.0 diff --git a/ml_metadata/metadata_store/metadata_store.py b/ml_metadata/metadata_store/metadata_store.py index 1bdc50b07..81789fa14 100644 --- a/ml_metadata/metadata_store/metadata_store.py +++ b/ml_metadata/metadata_store/metadata_store.py @@ -81,9 +81,13 @@ class ListOptions: is_asc: Specifies `order_by` is ascending or descending. If `order_by` is not given, the field is ignored. If `order_by` is set, then by default ascending order is used for performance benefit. - filter_query: An optional boolean expression in SQL syntax to specify - conditions on node attributes and directly connected assets. See - https://github.com/google/ml-metadata/blob/master/ml_metadata/proto/metadata_store.proto#L705-L783 for the query capabilities and syntax. + filter_query: DEPRECATED (will be removed in v1.19.0) - An optional boolean + expression in SQL syntax to specify conditions on node attributes and + directly connected assets. This feature depends on ZetaSQL which is being + removed from ML Metadata. Please migrate to alternative filtering approaches + before version 1.19.0. See + https://github.com/google/ml-metadata/blob/master/ml_metadata/proto/metadata_store.proto#L705-L783 + for the query capabilities and syntax. """ limit: Optional[int] = None @@ -1478,6 +1482,16 @@ def _call_method_with_list_options( if list_options.order_by: request.options.order_by_field.field = list_options.order_by.value if list_options.filter_query: + # DEPRECATED: ZetaSQL-based filter_query will be removed in v1.19.0 + import warnings + warnings.warn( + 'DEPRECATION WARNING: filter_query is deprecated and will be ' + 'removed in version 1.19.0. This feature depends on ZetaSQL ' + 'which is being removed from ML Metadata. Please migrate to ' + 'alternative filtering approaches before the 1.19.0 release.', + DeprecationWarning, + stacklevel=3 + ) request.options.filter_query = list_options.filter_query result = [] diff --git a/ml_metadata/metadata_store/metadata_store_test.py b/ml_metadata/metadata_store/metadata_store_test.py index 85cf8f78a..e75c050d7 100644 --- a/ml_metadata/metadata_store/metadata_store_test.py +++ b/ml_metadata/metadata_store/metadata_store_test.py @@ -15,6 +15,7 @@ import collections import os import uuid +import warnings import pytest from absl.testing import absltest, parameterized @@ -1803,14 +1804,20 @@ def test_get_nodes_by_filter_query(self, create_type_fn, put_type_fn, nodes[i].custom_properties["p"].int_value = i node_ids = put_nodes_fn(store, nodes) - got_nodes = get_nodes_fn( - store, - list_options=mlmd.ListOptions( - order_by=mlmd.OrderByField.ID, - is_asc=True, - filter_query=("custom_properties.p.int_value < 21 AND " - "name LIKE 'node_2%'") - )) + # Verify deprecation warning is raised when using filter_query + with self.assertWarns(DeprecationWarning) as warning_context: + got_nodes = get_nodes_fn( + store, + list_options=mlmd.ListOptions( + order_by=mlmd.OrderByField.ID, + is_asc=True, + filter_query=("custom_properties.p.int_value < 21 AND " + "name LIKE 'node_2%'") + )) + # Verify the warning message mentions version 1.19.0 + self.assertIn("1.19.0", str(warning_context.warning)) + self.assertIn("filter_query", str(warning_context.warning)) + self.assertLen(got_nodes, 2) self.assertEqual(got_nodes[0].id, node_ids[2]) self.assertEqual(got_nodes[0].name, "node_2") @@ -1822,9 +1829,15 @@ def test_get_nodes_by_filter_query(self, create_type_fn, put_type_fn, (mlmd.MetadataStore.get_contexts)) def test_get_nodes_by_filter_query_syntax_errors(self, get_nodes_fn): store = _get_metadata_store(self.cli_args) - with self.assertRaises(errors.InvalidArgumentError): + # Verify deprecation warning is raised even for syntax errors + with ( + self.assertWarns(DeprecationWarning) as warning_context, + self.assertRaises(errors.InvalidArgumentError), + ): _ = get_nodes_fn( store, list_options=mlmd.ListOptions(filter_query="invalid syntax")) + # Verify the warning message mentions version 1.19.0 + self.assertIn("1.19.0", str(warning_context.warning)) def test_put_contexts_get_context_by_type_and_name(self): # Prepare test data. @@ -2112,12 +2125,15 @@ def test_put_lineage_subgraph_get_lineage_subgraph(self): # Test get_lineage_subgraph() with max_num_hops = 10 and field mask paths = # ["events", "associations", "attributions"], the whole lineage subgraph # skeleton will be returned. - query_options = metadata_store_pb2.LineageSubgraphQueryOptions( - starting_artifacts=metadata_store_pb2.LineageSubgraphQueryOptions.StartingNodes( - filter_query="uri = 'output_artifact'" - ), - max_num_hops=10, - ) + # Note: filter_query is deprecated but tested here for backward compatibility. + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + query_options = metadata_store_pb2.LineageSubgraphQueryOptions( + starting_artifacts=metadata_store_pb2.LineageSubgraphQueryOptions.StartingNodes( + filter_query="uri = 'output_artifact'" + ), + max_num_hops=10, + ) subgraph_skeleton = store.get_lineage_subgraph( query_options, ["events", "associations", "attributions"] @@ -2169,12 +2185,15 @@ def test_put_lineage_subgraph_get_lineage_subgraph(self): # Test get_lineage_subgraph() with max_num_hops = 0 from starting executions # filtered by context name. All the executions will be returned. - query_options = metadata_store_pb2.LineageSubgraphQueryOptions( - starting_executions=metadata_store_pb2.LineageSubgraphQueryOptions.StartingNodes( - filter_query="contexts_a.name='existing_context'" - ), - max_num_hops=0, - ) + # Note: filter_query is deprecated but tested here for backward compatibility. + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + query_options = metadata_store_pb2.LineageSubgraphQueryOptions( + starting_executions=metadata_store_pb2.LineageSubgraphQueryOptions.StartingNodes( + filter_query="contexts_a.name='existing_context'" + ), + max_num_hops=0, + ) subgraph = store.get_lineage_subgraph(query_options) self.assertEmpty(subgraph.artifacts) self.assertLen(subgraph.executions, 2) @@ -2195,12 +2214,15 @@ def test_put_lineage_subgraph_get_lineage_subgraph(self): self.assertEmpty(subgraph.attributions) # Test get_lineage_subgraph() with various field mask paths. - query_options = metadata_store_pb2.LineageSubgraphQueryOptions( - starting_artifacts=metadata_store_pb2.LineageSubgraphQueryOptions.StartingNodes( - filter_query="uri = 'output_artifact'" - ), - max_num_hops=10, - ) + # Note: filter_query is deprecated but tested here for backward compatibility. + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + query_options = metadata_store_pb2.LineageSubgraphQueryOptions( + starting_artifacts=metadata_store_pb2.LineageSubgraphQueryOptions.StartingNodes( + filter_query="uri = 'output_artifact'" + ), + max_num_hops=10, + ) subgraph = store.get_lineage_subgraph( query_options, ["artifact_types", "execution_types", "context_types"] @@ -2291,13 +2313,16 @@ def test_put_lineage_subgraph_get_lineage_subgraph_with_direction(self): ) # Test get_lineage_subgraph() with direction. - query_options = metadata_store_pb2.LineageSubgraphQueryOptions( - starting_executions=metadata_store_pb2.LineageSubgraphQueryOptions.StartingNodes( - filter_query="name = 'test_execution'" - ), - max_num_hops=2, - direction=metadata_store_pb2.LineageSubgraphQueryOptions.Direction.DOWNSTREAM, - ) + # Note: filter_query is deprecated but tested here for backward compatibility. + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + query_options = metadata_store_pb2.LineageSubgraphQueryOptions( + starting_executions=metadata_store_pb2.LineageSubgraphQueryOptions.StartingNodes( + filter_query="name = 'test_execution'" + ), + max_num_hops=2, + direction=metadata_store_pb2.LineageSubgraphQueryOptions.Direction.DOWNSTREAM, + ) subgraph = store.get_lineage_subgraph(query_options) self.assertLen(subgraph.artifacts, 1) self.assertLen(subgraph.executions, 1) diff --git a/ml_metadata/metadata_store/postgresql_query_executor.cc b/ml_metadata/metadata_store/postgresql_query_executor.cc index 501a9251b..1cfd07d82 100644 --- a/ml_metadata/metadata_store/postgresql_query_executor.cc +++ b/ml_metadata/metadata_store/postgresql_query_executor.cc @@ -885,6 +885,12 @@ absl::Status PostgreSQLQueryExecutor::ListNodeIDsUsingOptions( } if (options.has_filter_query() && !options.filter_query().empty()) { + // DEPRECATED: ZetaSQL-based filter_query is deprecated and will be removed + // in version 1.19.0. This feature depends on ZetaSQL which is being phased + // out. Please migrate to alternative filtering approaches. + LOG(WARNING) << "DEPRECATION WARNING: ZetaSQL-based filter_query is " + << "deprecated and will be removed in version 1.19.0. " + << "This feature depends on ZetaSQL which is being removed."; node_table_alias = ml_metadata::FilterQueryBuilder::kBaseTableAlias; ml_metadata::FilterQueryAstResolver ast_resolver( options.filter_query()); diff --git a/ml_metadata/metadata_store/query_config_executor.cc b/ml_metadata/metadata_store/query_config_executor.cc index d369a511d..546282b7c 100644 --- a/ml_metadata/metadata_store/query_config_executor.cc +++ b/ml_metadata/metadata_store/query_config_executor.cc @@ -826,6 +826,12 @@ absl::Status QueryConfigExecutor::ListNodeIDsUsingOptions( } if (options.has_filter_query() && !options.filter_query().empty()) { + // DEPRECATED: ZetaSQL-based filter_query is deprecated and will be removed + // in version 1.19.0. This feature depends on ZetaSQL which is being phased + // out. Please migrate to alternative filtering approaches. + LOG(WARNING) << "DEPRECATION WARNING: ZetaSQL-based filter_query is " + << "deprecated and will be removed in version 1.19.0. " + << "This feature depends on ZetaSQL which is being removed."; node_table_alias = ml_metadata::FilterQueryBuilder::kBaseTableAlias; ml_metadata::FilterQueryAstResolver ast_resolver( options.filter_query()); diff --git a/ml_metadata/query/filter_query_ast_resolver.cc b/ml_metadata/query/filter_query_ast_resolver.cc index 5a7b859b4..faba77e66 100644 --- a/ml_metadata/query/filter_query_ast_resolver.cc +++ b/ml_metadata/query/filter_query_ast_resolver.cc @@ -12,6 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ + +// DEPRECATED: This file and its associated ZetaSQL-based filter_query +// functionality is deprecated and will be removed in version 1.19.0. +// ZetaSQL dependency is being phased out from ML Metadata. +// Please migrate to alternative filtering approaches. + #include "ml_metadata/query/filter_query_ast_resolver.h" #include diff --git a/ml_metadata/query/filter_query_ast_resolver.h b/ml_metadata/query/filter_query_ast_resolver.h index f5338e2b9..983192c48 100644 --- a/ml_metadata/query/filter_query_ast_resolver.h +++ b/ml_metadata/query/filter_query_ast_resolver.h @@ -22,6 +22,11 @@ limitations under the License. namespace ml_metadata { +// DEPRECATED: This class and its associated ZetaSQL-based filter_query +// functionality is deprecated and will be removed in version 1.19.0. +// ZetaSQL dependency is being phased out from ML Metadata. +// Please migrate to alternative filtering approaches. +// // FilterQueryAstResolver parses the MLMD filtering query string and generates // an AST via ZetaSQL analyzer. It can be instantiated with MLMD nodes types: // Artifact, Execution and Context. diff --git a/ml_metadata/query/filter_query_builder.cc b/ml_metadata/query/filter_query_builder.cc index cbd6e3b79..256ef766a 100644 --- a/ml_metadata/query/filter_query_builder.cc +++ b/ml_metadata/query/filter_query_builder.cc @@ -12,6 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ + +// DEPRECATED: This file and its associated ZetaSQL-based filter_query +// functionality is deprecated and will be removed in version 1.19.0. +// ZetaSQL dependency is being phased out from ML Metadata. +// Please migrate to alternative filtering approaches. + #include "ml_metadata/query/filter_query_builder.h" #include diff --git a/ml_metadata/query/filter_query_builder.h b/ml_metadata/query/filter_query_builder.h index 803ae0034..d42258fa2 100644 --- a/ml_metadata/query/filter_query_builder.h +++ b/ml_metadata/query/filter_query_builder.h @@ -21,6 +21,11 @@ limitations under the License. namespace ml_metadata { +// DEPRECATED: This class and its associated ZetaSQL-based filter_query +// functionality is deprecated and will be removed in version 1.19.0. +// ZetaSQL dependency is being phased out from ML Metadata. +// Please migrate to alternative filtering approaches. +// // FilterQueryBuilder is a ZetaSQL AST Visitor. It walks through a ZetaSQL // boolean expression AST parsed from a filtering query string and generates // FROM clauses and WHERE clauses which can be used by MLMD query executors. It