apache_beam.testing.util.BeamAssertException: Failed assert: [('abc', 1), ('abc', 1), ('abc', 1), ('abc', 1), ('abc', 2), ('abc', 2), ('abc', 2), ('abc', 2), ('abc', 3), ('abc', 3), ('abc', 3), ('abc', 3), ('abc', 4), ('abc', 4), ('abc', 4), ('abc', 4)] == [('abc', 1), ('abc', 1), ('abc', 2), ('abc', 5), ('abc', 6), ('abc', 6), ('abc', 6), ('abc', 6), ('abc', 7), ('abc', 7), ('abc', 7), ('abc', 7), ('abc', 8), ('abc', 8), ('abc', 8), ('abc', 8)], unexpected elements [('abc', 5), ('abc', 6), ('abc', 6), ('abc', 6), ('abc', 6), ('abc', 7), ('abc', 7), ('abc', 7), ('abc', 7), ('abc', 8), ('abc', 8), ('abc', 8), ('abc', 8)], missing elements [('abc', 1), ('abc', 1), ('abc', 2), ('abc', 2), ('abc', 2), ('abc', 3), ('abc', 3), ('abc', 3), ('abc', 3), ('abc', 4), ('abc', 4), ('abc', 4), ('abc', 4)] [while running 'assert:inferences/Match']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1055: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.process_method(*args_for_process, **kwargs_for_process),
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/transforms/core.py:2084: in <lambda>
    wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

actual = [('abc', 1), ('abc', 1), ('abc', 2), ('abc', 5), ('abc', 6), ('abc', 6), ...]
equals_fn = <function equal_to.<locals>._equal.<locals>.<lambda> at 0x7a6e87b5e290>

    def _equal(actual, equals_fn=equals_fn):
      expected_list = list(expected)
    
      # Try to compare actual and expected by sorting. This fails with a
      # TypeError in Python 3 if different types are present in the same
      # collection. It can also raise false negatives for types that don't have
      # a deterministic sort order, like pyarrow Tables as of 0.14.1
      if not equals_fn:
        equals_fn = lambda e, a: e == a
        try:
          sorted_expected = sorted(expected)
          sorted_actual = sorted(actual)
          if sorted_expected == sorted_actual:
            return
        except TypeError:
          pass
      # Slower method, used in two cases:
      # 1) If sorted expected != actual, use this method to verify the inequality.
      #    This ensures we don't raise any false negatives for types that don't
      #    have a deterministic sort order.
      # 2) As a fallback if we encounter a TypeError in python 3. this method
      #    works on collections that have different types.
      unexpected = []
      for element in actual:
        found = False
        for i, v in enumerate(expected_list):
          if equals_fn(v, element):
            found = True
            expected_list.pop(i)
            break
        if not found:
          unexpected.append(element)
      if unexpected or expected_list:
        msg = 'Failed assert: %r == %r' % (expected, actual)
        if unexpected:
          msg = msg + ', unexpected elements %r' % unexpected
        if expected_list:
          msg = msg + ', missing elements %r' % expected_list
>       raise BeamAssertException(msg)
E       apache_beam.testing.util.BeamAssertException: Failed assert: [('abc', 1), ('abc', 1), ('abc', 1), ('abc', 1), ('abc', 2), ('abc', 2), ('abc', 2), ('abc', 2), ('abc', 3), ('abc', 3), ('abc', 3), ('abc', 3), ('abc', 4), ('abc', 4), ('abc', 4), ('abc', 4)] == [('abc', 1), ('abc', 1), ('abc', 2), ('abc', 5), ('abc', 6), ('abc', 6), ('abc', 6), ('abc', 6), ('abc', 7), ('abc', 7), ('abc', 7), ('abc', 7), ('abc', 8), ('abc', 8), ('abc', 8), ('abc', 8)], unexpected elements [('abc', 5), ('abc', 6), ('abc', 6), ('abc', 6), ('abc', 6), ('abc', 7), ('abc', 7), ('abc', 7), ('abc', 7), ('abc', 8), ('abc', 8), ('abc', 8), ('abc', 8)], missing elements [('abc', 1), ('abc', 1), ('abc', 2), ('abc', 2), ('abc', 2), ('abc', 3), ('abc', 3), ('abc', 3), ('abc', 3), ('abc', 4), ('abc', 4), ('abc', 4), ('abc', 4)]

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/util.py:192: BeamAssertException

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.inference.base_test.RunInferenceBaseTest testMethod=test_run_inference_impl_keyed_mps_incrementing_multi_copy>

    def test_run_inference_impl_keyed_mps_incrementing_multi_copy(self):
>     with TestPipeline() as pipeline:

apache_beam/ml/inference/base_test.py:363: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/test_pipeline.py:115: in run
    result = super().run(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:594: in run
    self._options).run(False)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1274: in process_bundle
    input_op_by_transform_id[element.transform_id].process_encoded(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:237: in process_encoded
    self.output(decoded_value)
apache_beam/runners/worker/operations.py:566: in apache_beam.runners.worker.operations.Operation.output
    def output(self, windowed_value, output_index=0):
apache_beam/runners/worker/operations.py:568: in apache_beam.runners.worker.operations.Operation.output
    _cast_to_receiver(self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:259: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    def receive(self, windowed_value):
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1055: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.process_method(*args_for_process, **kwargs_for_process),
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/transforms/core.py:2084: in <lambda>
    wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

actual = [('abc', 1), ('abc', 1), ('abc', 2), ('abc', 5), ('abc', 6), ('abc', 6), ...]
equals_fn = <function equal_to.<locals>._equal.<locals>.<lambda> at 0x7a6e87b5e290>

    def _equal(actual, equals_fn=equals_fn):
      expected_list = list(expected)
    
      # Try to compare actual and expected by sorting. This fails with a
      # TypeError in Python 3 if different types are present in the same
      # collection. It can also raise false negatives for types that don't have
      # a deterministic sort order, like pyarrow Tables as of 0.14.1
      if not equals_fn:
        equals_fn = lambda e, a: e == a
        try:
          sorted_expected = sorted(expected)
          sorted_actual = sorted(actual)
          if sorted_expected == sorted_actual:
            return
        except TypeError:
          pass
      # Slower method, used in two cases:
      # 1) If sorted expected != actual, use this method to verify the inequality.
      #    This ensures we don't raise any false negatives for types that don't
      #    have a deterministic sort order.
      # 2) As a fallback if we encounter a TypeError in python 3. this method
      #    works on collections that have different types.
      unexpected = []
      for element in actual:
        found = False
        for i, v in enumerate(expected_list):
          if equals_fn(v, element):
            found = True
            expected_list.pop(i)
            break
        if not found:
          unexpected.append(element)
      if unexpected or expected_list:
        msg = 'Failed assert: %r == %r' % (expected, actual)
        if unexpected:
          msg = msg + ', unexpected elements %r' % unexpected
        if expected_list:
          msg = msg + ', missing elements %r' % expected_list
>       raise BeamAssertException(msg)
E       apache_beam.testing.util.BeamAssertException: Failed assert: [('abc', 1), ('abc', 1), ('abc', 1), ('abc', 1), ('abc', 2), ('abc', 2), ('abc', 2), ('abc', 2), ('abc', 3), ('abc', 3), ('abc', 3), ('abc', 3), ('abc', 4), ('abc', 4), ('abc', 4), ('abc', 4)] == [('abc', 1), ('abc', 1), ('abc', 2), ('abc', 5), ('abc', 6), ('abc', 6), ('abc', 6), ('abc', 6), ('abc', 7), ('abc', 7), ('abc', 7), ('abc', 7), ('abc', 8), ('abc', 8), ('abc', 8), ('abc', 8)], unexpected elements [('abc', 5), ('abc', 6), ('abc', 6), ('abc', 6), ('abc', 6), ('abc', 7), ('abc', 7), ('abc', 7), ('abc', 7), ('abc', 8), ('abc', 8), ('abc', 8), ('abc', 8)], missing elements [('abc', 1), ('abc', 1), ('abc', 2), ('abc', 2), ('abc', 2), ('abc', 3), ('abc', 3), ('abc', 3), ('abc', 3), ('abc', 4), ('abc', 4), ('abc', 4), ('abc', 4)] [while running 'assert:inferences/Match']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/util.py:192: BeamAssertException

AssertionError: [] is not true
self = <apache_beam.ml.inference.base_test.RunInferenceBaseTest testMethod=test_counted_metrics>

    def test_counted_metrics(self):
      pipeline = TestPipeline()
      examples = [1, 5, 3, 10]
      pcoll = pipeline | 'start' >> beam.Create(examples)
      _ = pcoll | base.RunInference(FakeModelHandler())
      run_result = pipeline.run()
      run_result.wait_until_finish()
    
      metric_results = (
          run_result.metrics().query(MetricsFilter().with_name('num_inferences')))
      num_inferences_counter = metric_results['counters'][0]
      self.assertEqual(num_inferences_counter.committed, 4)
    
      inference_request_batch_size = run_result.metrics().query(
          MetricsFilter().with_name('inference_request_batch_size'))
      self.assertTrue(inference_request_batch_size['distributions'])
      self.assertEqual(
          inference_request_batch_size['distributions'][0].result.sum, 4)
      inference_request_batch_byte_size = run_result.metrics().query(
          MetricsFilter().with_name('inference_request_batch_byte_size'))
      self.assertTrue(inference_request_batch_byte_size['distributions'])
      self.assertGreaterEqual(
          inference_request_batch_byte_size['distributions'][0].result.sum,
          len(pickle.dumps(examples)))
      inference_request_batch_byte_size = run_result.metrics().query(
          MetricsFilter().with_name('model_byte_size'))
>     self.assertTrue(inference_request_batch_byte_size['distributions'])
E     AssertionError: [] is not true

apache_beam/ml/inference/base_test.py:1011: AssertionError

apache_beam.testing.util.BeamAssertException: Failed assert: [4, 5, 6] == [10, 11, 12], unexpected elements [10, 11, 12], missing elements [4, 5, 6] [while running 'assert:inferences/Match']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1055: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.process_method(*args_for_process, **kwargs_for_process),
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/transforms/core.py:2084: in <lambda>
    wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

actual = [10, 11, 12]
equals_fn = <function equal_to.<locals>._equal.<locals>.<lambda> at 0x7c10e817fd00>

    def _equal(actual, equals_fn=equals_fn):
      expected_list = list(expected)
    
      # Try to compare actual and expected by sorting. This fails with a
      # TypeError in Python 3 if different types are present in the same
      # collection. It can also raise false negatives for types that don't have
      # a deterministic sort order, like pyarrow Tables as of 0.14.1
      if not equals_fn:
        equals_fn = lambda e, a: e == a
        try:
          sorted_expected = sorted(expected)
          sorted_actual = sorted(actual)
          if sorted_expected == sorted_actual:
            return
        except TypeError:
          pass
      # Slower method, used in two cases:
      # 1) If sorted expected != actual, use this method to verify the inequality.
      #    This ensures we don't raise any false negatives for types that don't
      #    have a deterministic sort order.
      # 2) As a fallback if we encounter a TypeError in python 3. this method
      #    works on collections that have different types.
      unexpected = []
      for element in actual:
        found = False
        for i, v in enumerate(expected_list):
          if equals_fn(v, element):
            found = True
            expected_list.pop(i)
            break
        if not found:
          unexpected.append(element)
      if unexpected or expected_list:
        msg = 'Failed assert: %r == %r' % (expected, actual)
        if unexpected:
          msg = msg + ', unexpected elements %r' % unexpected
        if expected_list:
          msg = msg + ', missing elements %r' % expected_list
>       raise BeamAssertException(msg)
E       apache_beam.testing.util.BeamAssertException: Failed assert: [4, 5, 6] == [10, 11, 12], unexpected elements [10, 11, 12], missing elements [4, 5, 6]

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/util.py:192: BeamAssertException

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.inference.base_test.RunInferenceBaseTest testMethod=test_runinference_loads_same_model_with_identifier_multi_process_shared>

    def test_runinference_loads_same_model_with_identifier_multi_process_shared(
        self):
      mh1 = FakeModelHandler(
          incrementing=True, min_batch_size=3, multi_process_shared=True)
>     with TestPipeline() as pipeline:

apache_beam/ml/inference/base_test.py:1749: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/test_pipeline.py:115: in run
    result = super().run(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:594: in run
    self._options).run(False)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1274: in process_bundle
    input_op_by_transform_id[element.transform_id].process_encoded(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:237: in process_encoded
    self.output(decoded_value)
apache_beam/runners/worker/operations.py:566: in apache_beam.runners.worker.operations.Operation.output
    def output(self, windowed_value, output_index=0):
apache_beam/runners/worker/operations.py:568: in apache_beam.runners.worker.operations.Operation.output
    _cast_to_receiver(self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:259: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    def receive(self, windowed_value):
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1055: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.process_method(*args_for_process, **kwargs_for_process),
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/transforms/core.py:2084: in <lambda>
    wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

actual = [10, 11, 12]
equals_fn = <function equal_to.<locals>._equal.<locals>.<lambda> at 0x7c10e817fd00>

    def _equal(actual, equals_fn=equals_fn):
      expected_list = list(expected)
    
      # Try to compare actual and expected by sorting. This fails with a
      # TypeError in Python 3 if different types are present in the same
      # collection. It can also raise false negatives for types that don't have
      # a deterministic sort order, like pyarrow Tables as of 0.14.1
      if not equals_fn:
        equals_fn = lambda e, a: e == a
        try:
          sorted_expected = sorted(expected)
          sorted_actual = sorted(actual)
          if sorted_expected == sorted_actual:
            return
        except TypeError:
          pass
      # Slower method, used in two cases:
      # 1) If sorted expected != actual, use this method to verify the inequality.
      #    This ensures we don't raise any false negatives for types that don't
      #    have a deterministic sort order.
      # 2) As a fallback if we encounter a TypeError in python 3. this method
      #    works on collections that have different types.
      unexpected = []
      for element in actual:
        found = False
        for i, v in enumerate(expected_list):
          if equals_fn(v, element):
            found = True
            expected_list.pop(i)
            break
        if not found:
          unexpected.append(element)
      if unexpected or expected_list:
        msg = 'Failed assert: %r == %r' % (expected, actual)
        if unexpected:
          msg = msg + ', unexpected elements %r' % unexpected
        if expected_list:
          msg = msg + ', missing elements %r' % expected_list
>       raise BeamAssertException(msg)
E       apache_beam.testing.util.BeamAssertException: Failed assert: [4, 5, 6] == [10, 11, 12], unexpected elements [10, 11, 12], missing elements [4, 5, 6] [while running 'assert:inferences/Match']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/util.py:192: BeamAssertException

apache_beam.testing.util.BeamAssertException: Failed assert: [1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4] == [1, 2, 3, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8], unexpected elements [5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8], missing elements [1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4] [while running 'assert:inferences/Match']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1055: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.process_method(*args_for_process, **kwargs_for_process),
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/transforms/core.py:2084: in <lambda>
    wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

actual = [1, 2, 3, 5, 6, 6, ...]
equals_fn = <function equal_to.<locals>._equal.<locals>.<lambda> at 0x7a6e60271d80>

    def _equal(actual, equals_fn=equals_fn):
      expected_list = list(expected)
    
      # Try to compare actual and expected by sorting. This fails with a
      # TypeError in Python 3 if different types are present in the same
      # collection. It can also raise false negatives for types that don't have
      # a deterministic sort order, like pyarrow Tables as of 0.14.1
      if not equals_fn:
        equals_fn = lambda e, a: e == a
        try:
          sorted_expected = sorted(expected)
          sorted_actual = sorted(actual)
          if sorted_expected == sorted_actual:
            return
        except TypeError:
          pass
      # Slower method, used in two cases:
      # 1) If sorted expected != actual, use this method to verify the inequality.
      #    This ensures we don't raise any false negatives for types that don't
      #    have a deterministic sort order.
      # 2) As a fallback if we encounter a TypeError in python 3. this method
      #    works on collections that have different types.
      unexpected = []
      for element in actual:
        found = False
        for i, v in enumerate(expected_list):
          if equals_fn(v, element):
            found = True
            expected_list.pop(i)
            break
        if not found:
          unexpected.append(element)
      if unexpected or expected_list:
        msg = 'Failed assert: %r == %r' % (expected, actual)
        if unexpected:
          msg = msg + ', unexpected elements %r' % unexpected
        if expected_list:
          msg = msg + ', missing elements %r' % expected_list
>       raise BeamAssertException(msg)
E       apache_beam.testing.util.BeamAssertException: Failed assert: [1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4] == [1, 2, 3, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8], unexpected elements [5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8], missing elements [1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4]

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/util.py:192: BeamAssertException

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.inference.base_test.RunInferenceBaseTest testMethod=test_run_inference_impl_mps_nobatch_incrementing_multi_copy>

    def test_run_inference_impl_mps_nobatch_incrementing_multi_copy(self):
>     with TestPipeline() as pipeline:

apache_beam/ml/inference/base_test.py:351: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/test_pipeline.py:115: in run
    result = super().run(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:594: in run
    self._options).run(False)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1274: in process_bundle
    input_op_by_transform_id[element.transform_id].process_encoded(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:237: in process_encoded
    self.output(decoded_value)
apache_beam/runners/worker/operations.py:566: in apache_beam.runners.worker.operations.Operation.output
    def output(self, windowed_value, output_index=0):
apache_beam/runners/worker/operations.py:568: in apache_beam.runners.worker.operations.Operation.output
    _cast_to_receiver(self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:259: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    def receive(self, windowed_value):
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1055: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.process_method(*args_for_process, **kwargs_for_process),
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/transforms/core.py:2084: in <lambda>
    wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

actual = [1, 2, 3, 5, 6, 6, ...]
equals_fn = <function equal_to.<locals>._equal.<locals>.<lambda> at 0x7a6e60271d80>

    def _equal(actual, equals_fn=equals_fn):
      expected_list = list(expected)
    
      # Try to compare actual and expected by sorting. This fails with a
      # TypeError in Python 3 if different types are present in the same
      # collection. It can also raise false negatives for types that don't have
      # a deterministic sort order, like pyarrow Tables as of 0.14.1
      if not equals_fn:
        equals_fn = lambda e, a: e == a
        try:
          sorted_expected = sorted(expected)
          sorted_actual = sorted(actual)
          if sorted_expected == sorted_actual:
            return
        except TypeError:
          pass
      # Slower method, used in two cases:
      # 1) If sorted expected != actual, use this method to verify the inequality.
      #    This ensures we don't raise any false negatives for types that don't
      #    have a deterministic sort order.
      # 2) As a fallback if we encounter a TypeError in python 3. this method
      #    works on collections that have different types.
      unexpected = []
      for element in actual:
        found = False
        for i, v in enumerate(expected_list):
          if equals_fn(v, element):
            found = True
            expected_list.pop(i)
            break
        if not found:
          unexpected.append(element)
      if unexpected or expected_list:
        msg = 'Failed assert: %r == %r' % (expected, actual)
        if unexpected:
          msg = msg + ', unexpected elements %r' % unexpected
        if expected_list:
          msg = msg + ', missing elements %r' % expected_list
>       raise BeamAssertException(msg)
E       apache_beam.testing.util.BeamAssertException: Failed assert: [1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4] == [1, 2, 3, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8], unexpected elements [5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8], missing elements [1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4] [while running 'assert:inferences/Match']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/util.py:192: BeamAssertException

IndexError: list index out of range
self = <apache_beam.ml.inference.base_test.RunInferenceBaseTest testMethod=test_timing_metrics>

    def test_timing_metrics(self):
      pipeline = TestPipeline()
      examples = [1, 5, 3, 10]
      pcoll = pipeline | 'start' >> beam.Create(examples)
      fake_clock = FakeClock()
      _ = pcoll | base.RunInference(
          FakeModelHandler(clock=fake_clock), clock=fake_clock)
      res = pipeline.run()
      res.wait_until_finish()
    
      metric_results = (
          res.metrics().query(
              MetricsFilter().with_name('inference_batch_latency_micro_secs')))
      batch_latency = metric_results['distributions'][0]
      self.assertEqual(batch_latency.result.count, 3)
      self.assertEqual(batch_latency.result.mean, 3000)
    
      metric_results = (
          res.metrics().query(
              MetricsFilter().with_name('load_model_latency_milli_secs')))
>     load_model_latency = metric_results['distributions'][0]
E     IndexError: list index out of range

apache_beam/ml/inference/base_test.py:1033: IndexError

apache_beam.testing.util.BeamAssertException: Failed assert: [1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4] == [1, 2, 2, 3, 3, 5, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8], unexpected elements [5, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8], missing elements [1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4] [while running 'assert:inferences/Match']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1055: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.process_method(*args_for_process, **kwargs_for_process),
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/transforms/core.py:2084: in <lambda>
    wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

actual = [1, 2, 2, 3, 3, 5, ...]
equals_fn = <function equal_to.<locals>._equal.<locals>.<lambda> at 0x7a6e883eab90>

    def _equal(actual, equals_fn=equals_fn):
      expected_list = list(expected)
    
      # Try to compare actual and expected by sorting. This fails with a
      # TypeError in Python 3 if different types are present in the same
      # collection. It can also raise false negatives for types that don't have
      # a deterministic sort order, like pyarrow Tables as of 0.14.1
      if not equals_fn:
        equals_fn = lambda e, a: e == a
        try:
          sorted_expected = sorted(expected)
          sorted_actual = sorted(actual)
          if sorted_expected == sorted_actual:
            return
        except TypeError:
          pass
      # Slower method, used in two cases:
      # 1) If sorted expected != actual, use this method to verify the inequality.
      #    This ensures we don't raise any false negatives for types that don't
      #    have a deterministic sort order.
      # 2) As a fallback if we encounter a TypeError in python 3. this method
      #    works on collections that have different types.
      unexpected = []
      for element in actual:
        found = False
        for i, v in enumerate(expected_list):
          if equals_fn(v, element):
            found = True
            expected_list.pop(i)
            break
        if not found:
          unexpected.append(element)
      if unexpected or expected_list:
        msg = 'Failed assert: %r == %r' % (expected, actual)
        if unexpected:
          msg = msg + ', unexpected elements %r' % unexpected
        if expected_list:
          msg = msg + ', missing elements %r' % expected_list
>       raise BeamAssertException(msg)
E       apache_beam.testing.util.BeamAssertException: Failed assert: [1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4] == [1, 2, 2, 3, 3, 5, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8], unexpected elements [5, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8], missing elements [1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4]

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/util.py:192: BeamAssertException

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.inference.base_test.RunInferenceBaseTest testMethod=test_run_inference_impl_multi_process_shared_incrementing_multi_copy>

    def test_run_inference_impl_multi_process_shared_incrementing_multi_copy(
        self):
>     with TestPipeline() as pipeline:

apache_beam/ml/inference/base_test.py:338: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/test_pipeline.py:115: in run
    result = super().run(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:594: in run
    self._options).run(False)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1274: in process_bundle
    input_op_by_transform_id[element.transform_id].process_encoded(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:237: in process_encoded
    self.output(decoded_value)
apache_beam/runners/worker/operations.py:566: in apache_beam.runners.worker.operations.Operation.output
    def output(self, windowed_value, output_index=0):
apache_beam/runners/worker/operations.py:568: in apache_beam.runners.worker.operations.Operation.output
    _cast_to_receiver(self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:259: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    def receive(self, windowed_value):
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1055: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.process_method(*args_for_process, **kwargs_for_process),
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/transforms/core.py:2084: in <lambda>
    wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

actual = [1, 2, 2, 3, 3, 5, ...]
equals_fn = <function equal_to.<locals>._equal.<locals>.<lambda> at 0x7a6e883eab90>

    def _equal(actual, equals_fn=equals_fn):
      expected_list = list(expected)
    
      # Try to compare actual and expected by sorting. This fails with a
      # TypeError in Python 3 if different types are present in the same
      # collection. It can also raise false negatives for types that don't have
      # a deterministic sort order, like pyarrow Tables as of 0.14.1
      if not equals_fn:
        equals_fn = lambda e, a: e == a
        try:
          sorted_expected = sorted(expected)
          sorted_actual = sorted(actual)
          if sorted_expected == sorted_actual:
            return
        except TypeError:
          pass
      # Slower method, used in two cases:
      # 1) If sorted expected != actual, use this method to verify the inequality.
      #    This ensures we don't raise any false negatives for types that don't
      #    have a deterministic sort order.
      # 2) As a fallback if we encounter a TypeError in python 3. this method
      #    works on collections that have different types.
      unexpected = []
      for element in actual:
        found = False
        for i, v in enumerate(expected_list):
          if equals_fn(v, element):
            found = True
            expected_list.pop(i)
            break
        if not found:
          unexpected.append(element)
      if unexpected or expected_list:
        msg = 'Failed assert: %r == %r' % (expected, actual)
        if unexpected:
          msg = msg + ', unexpected elements %r' % unexpected
        if expected_list:
          msg = msg + ', missing elements %r' % expected_list
>       raise BeamAssertException(msg)
E       apache_beam.testing.util.BeamAssertException: Failed assert: [1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4] == [1, 2, 2, 3, 3, 5, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8], unexpected elements [5, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8], missing elements [1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4] [while running 'assert:inferences/Match']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/util.py:192: BeamAssertException

IndexError: list index out of range
self = <apache_beam.ml.inference.base_test.RunInferenceBaseTest testMethod=test_run_inference_impl_with_keyed_examples_many_model_handlers_metrics>

    def test_run_inference_impl_with_keyed_examples_many_model_handlers_metrics(
        self):
      pipeline = TestPipeline()
      examples = [1, 5, 3, 10]
      metrics_namespace = 'test_namespace'
      keyed_examples = [(i, example) for i, example in enumerate(examples)]
      pcoll = pipeline | 'start' >> beam.Create(keyed_examples)
      mhs = [
          base.KeyModelMapping([0],
                               FakeModelHandler(
                                   state=200, multi_process_shared=True)),
          base.KeyModelMapping([1, 2, 3],
                               FakeModelHandler(multi_process_shared=True))
      ]
      _ = pcoll | base.RunInference(
          base.KeyedModelHandler(mhs), metrics_namespace=metrics_namespace)
      result = pipeline.run()
      result.wait_until_finish()
    
      metrics_filter = MetricsFilter().with_namespace(namespace=metrics_namespace)
      metrics = result.metrics().query(metrics_filter)
      assert len(metrics['counters']) != 0
      assert len(metrics['distributions']) != 0
    
      metrics_filter = MetricsFilter().with_name('0-_num_inferences')
      metrics = result.metrics().query(metrics_filter)
      num_inferences_counter_key_0 = metrics['counters'][0]
      self.assertEqual(num_inferences_counter_key_0.committed, 1)
    
      metrics_filter = MetricsFilter().with_name('1-_num_inferences')
      metrics = result.metrics().query(metrics_filter)
      num_inferences_counter_key_1 = metrics['counters'][0]
      self.assertEqual(num_inferences_counter_key_1.committed, 3)
    
      metrics_filter = MetricsFilter().with_name('num_inferences')
      metrics = result.metrics().query(metrics_filter)
      num_inferences_counter_aggregate = metrics['counters'][0]
      self.assertEqual(num_inferences_counter_aggregate.committed, 4)
    
      metrics_filter = MetricsFilter().with_name('0-_failed_batches_counter')
      metrics = result.metrics().query(metrics_filter)
      failed_batches_counter_key_0 = metrics['counters']
      self.assertEqual(len(failed_batches_counter_key_0), 0)
    
      metrics_filter = MetricsFilter().with_name('failed_batches_counter')
      metrics = result.metrics().query(metrics_filter)
      failed_batches_counter_aggregate = metrics['counters']
      self.assertEqual(len(failed_batches_counter_aggregate), 0)
    
      metrics_filter = MetricsFilter().with_name(
          '0-_load_model_latency_milli_secs')
      metrics = result.metrics().query(metrics_filter)
>     load_latency_dist_key_0 = metrics['distributions'][0]
E     IndexError: list index out of range

apache_beam/ml/inference/base_test.py:457: IndexError

RuntimeError: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx [while running 'RunInference/BeamML_RunInference']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1055: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.process_method(*args_for_process, **kwargs_for_process),
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/ml/inference/base.py:1942: in process
    return self._run_inference(batch, inference_args)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/ml/inference/base.py:1912: in _run_inference
    raise e
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/ml/inference/base.py:1898: in _run_inference
    result_generator = self._model_handler.run_inference(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/ml/inference/pytorch_inference.py:328: in run_inference
    return self._inference_fn(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/ml/inference/pytorch_inference.py:151: in default_tensor_inference_fn
    batched_tensors = _convert_to_device(batched_tensors, device)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/ml/inference/pytorch_inference.py:136: in _convert_to_device
    examples = examples.to(device)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

    def _lazy_init():
        global _initialized, _queued_calls
        if is_initialized() or hasattr(_tls, "is_initializing"):
            return
        with _initialization_lock:
            # We be double-checked locking, boys!  This is OK because
            # the above test was GIL protected anyway.  The inner test
            # is for when a thread blocked on some other thread which was
            # doing the initialization; when they get the lock, they will
            # find there is nothing left to do.
            if is_initialized():
                return
            # It is important to prevent other threads from entering _lazy_init
            # immediately, while we are still guaranteed to have the GIL, because some
            # of the C calls we make below will release the GIL
            if _is_in_bad_fork():
                raise RuntimeError(
                    "Cannot re-initialize CUDA in forked subprocess. To use CUDA with "
                    "multiprocessing, you must use the 'spawn' start method"
                )
            if not hasattr(torch._C, "_cuda_getDeviceCount"):
                raise AssertionError("Torch not compiled with CUDA enabled")
            if _cudart is None:
                raise AssertionError(
                    "libcudart functions unavailable. It looks like you have a broken build?"
                )
            # This function throws if there's a driver initialization error, no GPUs
            # are found or any other error occurs
            if "CUDA_MODULE_LOADING" not in os.environ:
                os.environ["CUDA_MODULE_LOADING"] = "LAZY"
>           torch._C._cuda_init()
E           RuntimeError: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/torch/cuda/__init__.py:319: RuntimeError

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.inference.pytorch_inference_test.PytorchRunInferencePipelineTest testMethod=test_gpu_auto_convert_to_cpu>

    def test_gpu_auto_convert_to_cpu(self):
      """
      This tests the scenario in which the user defines `device='GPU'` for the
      PytorchModelHandlerX, but runs the pipeline on a machine without GPU, we
      automatically detect this discrepancy and do automatic conversion to CPU.
      A warning is also logged to inform the user.
      """
      with self.assertLogs() as log:
>       with TestPipeline() as pipeline:

apache_beam/ml/inference/pytorch_inference_test.py:762: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/test_pipeline.py:115: in run
    result = super().run(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:594: in run
    self._options).run(False)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1274: in process_bundle
    input_op_by_transform_id[element.transform_id].process_encoded(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:237: in process_encoded
    self.output(decoded_value)
apache_beam/runners/worker/operations.py:566: in apache_beam.runners.worker.operations.Operation.output
    def output(self, windowed_value, output_index=0):
apache_beam/runners/worker/operations.py:568: in apache_beam.runners.worker.operations.Operation.output
    _cast_to_receiver(self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:259: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    def receive(self, windowed_value):
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1055: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.process_method(*args_for_process, **kwargs_for_process),
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/ml/inference/base.py:1942: in process
    return self._run_inference(batch, inference_args)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/ml/inference/base.py:1912: in _run_inference
    raise e
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/ml/inference/base.py:1898: in _run_inference
    result_generator = self._model_handler.run_inference(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/ml/inference/pytorch_inference.py:328: in run_inference
    return self._inference_fn(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/ml/inference/pytorch_inference.py:151: in default_tensor_inference_fn
    batched_tensors = _convert_to_device(batched_tensors, device)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/ml/inference/pytorch_inference.py:136: in _convert_to_device
    examples = examples.to(device)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

    def _lazy_init():
        global _initialized, _queued_calls
        if is_initialized() or hasattr(_tls, "is_initializing"):
            return
        with _initialization_lock:
            # We be double-checked locking, boys!  This is OK because
            # the above test was GIL protected anyway.  The inner test
            # is for when a thread blocked on some other thread which was
            # doing the initialization; when they get the lock, they will
            # find there is nothing left to do.
            if is_initialized():
                return
            # It is important to prevent other threads from entering _lazy_init
            # immediately, while we are still guaranteed to have the GIL, because some
            # of the C calls we make below will release the GIL
            if _is_in_bad_fork():
                raise RuntimeError(
                    "Cannot re-initialize CUDA in forked subprocess. To use CUDA with "
                    "multiprocessing, you must use the 'spawn' start method"
                )
            if not hasattr(torch._C, "_cuda_getDeviceCount"):
                raise AssertionError("Torch not compiled with CUDA enabled")
            if _cudart is None:
                raise AssertionError(
                    "libcudart functions unavailable. It looks like you have a broken build?"
                )
            # This function throws if there's a driver initialization error, no GPUs
            # are found or any other error occurs
            if "CUDA_MODULE_LOADING" not in os.environ:
                os.environ["CUDA_MODULE_LOADING"] = "LAZY"
>           torch._C._cuda_init()
E           RuntimeError: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx [while running 'RunInference/BeamML_RunInference']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/torch/cuda/__init__.py:319: RuntimeError

RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x78f2842000d0>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: int64>
  child 0, item: int64
y: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmpu9olw33k/b7d444/tftransform_tmp/f62cebbe963747d1b19eb92fc5b772b4'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     AssertionError

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: AssertionError

During handling of the above exception, another exception occurred:

a = (<apache_beam.ml.transforms.base_test.BaseMLTransformTest testMethod=test_ml_transform_dict_output_pcoll_schema_0>,)
kw = {}

    @wraps(func)
    def standalone_func(*a, **kw):
>       return func(*(a + p.args), **p.kwargs, **kw)

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/parameterized/parameterized.py:620: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/ml/transforms/base_test.py:193: in test_ml_transform_dict_output_pcoll_schema
    with beam.Pipeline() as p:
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1282: in process_bundle
    op.finish()
apache_beam/runners/worker/operations.py:981: in apache_beam.runners.worker.operations.DoOperation.finish
    def finish(self):
apache_beam/runners/worker/operations.py:984: in apache_beam.runners.worker.operations.DoOperation.finish
    with self.scoped_finish_state:
apache_beam/runners/worker/operations.py:985: in apache_beam.runners.worker.operations.DoOperation.finish
    self.dofn_runner.finish()
apache_beam/runners/common.py:1573: in apache_beam.runners.common.DoFnRunner.finish
    self._invoke_bundle_method(self.do_fn_invoker.invoke_finish_bundle)
apache_beam/runners/common.py:1554: in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
    self._reraise_augmented(exn)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1552: in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
    bundle_method()
apache_beam/runners/common.py:615: in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
    def invoke_finish_bundle(self):
apache_beam/runners/common.py:620: in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
    self.output_handler.finish_bundle_outputs(
apache_beam/runners/common.py:1844: in apache_beam.runners.common._OutputHandler.finish_bundle_outputs
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x78f2842000d0>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: int64>
  child 0, item: int64
y: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmpu9olw33k/b7d444/tftransform_tmp/f62cebbe963747d1b19eb92fc5b772b4'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: RuntimeError

apache_beam.testing.util.BeamAssertException: Failed assert: [1, 2, 3] == [4, 5, 6], unexpected elements [4, 5, 6], missing elements [1, 2, 3] [while running 'assert:inferences/Match']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1055: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.process_method(*args_for_process, **kwargs_for_process),
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/transforms/core.py:2084: in <lambda>
    wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

actual = [4, 5, 6]
equals_fn = <function equal_to.<locals>._equal.<locals>.<lambda> at 0x7a6e40193250>

    def _equal(actual, equals_fn=equals_fn):
      expected_list = list(expected)
    
      # Try to compare actual and expected by sorting. This fails with a
      # TypeError in Python 3 if different types are present in the same
      # collection. It can also raise false negatives for types that don't have
      # a deterministic sort order, like pyarrow Tables as of 0.14.1
      if not equals_fn:
        equals_fn = lambda e, a: e == a
        try:
          sorted_expected = sorted(expected)
          sorted_actual = sorted(actual)
          if sorted_expected == sorted_actual:
            return
        except TypeError:
          pass
      # Slower method, used in two cases:
      # 1) If sorted expected != actual, use this method to verify the inequality.
      #    This ensures we don't raise any false negatives for types that don't
      #    have a deterministic sort order.
      # 2) As a fallback if we encounter a TypeError in python 3. this method
      #    works on collections that have different types.
      unexpected = []
      for element in actual:
        found = False
        for i, v in enumerate(expected_list):
          if equals_fn(v, element):
            found = True
            expected_list.pop(i)
            break
        if not found:
          unexpected.append(element)
      if unexpected or expected_list:
        msg = 'Failed assert: %r == %r' % (expected, actual)
        if unexpected:
          msg = msg + ', unexpected elements %r' % unexpected
        if expected_list:
          msg = msg + ', missing elements %r' % expected_list
>       raise BeamAssertException(msg)
E       apache_beam.testing.util.BeamAssertException: Failed assert: [1, 2, 3] == [4, 5, 6], unexpected elements [4, 5, 6], missing elements [1, 2, 3]

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/util.py:192: BeamAssertException

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.inference.base_test.RunInferenceBaseTest testMethod=test_run_inference_loads_different_models>

    def test_run_inference_loads_different_models(self):
      mh1 = FakeModelHandler(incrementing=True, min_batch_size=3)
>     with TestPipeline() as pipeline:

apache_beam/ml/inference/base_test.py:1726: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/test_pipeline.py:115: in run
    result = super().run(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:594: in run
    self._options).run(False)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1274: in process_bundle
    input_op_by_transform_id[element.transform_id].process_encoded(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:237: in process_encoded
    self.output(decoded_value)
apache_beam/runners/worker/operations.py:566: in apache_beam.runners.worker.operations.Operation.output
    def output(self, windowed_value, output_index=0):
apache_beam/runners/worker/operations.py:568: in apache_beam.runners.worker.operations.Operation.output
    _cast_to_receiver(self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:259: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    def receive(self, windowed_value):
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1055: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.process_method(*args_for_process, **kwargs_for_process),
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/transforms/core.py:2084: in <lambda>
    wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

actual = [4, 5, 6]
equals_fn = <function equal_to.<locals>._equal.<locals>.<lambda> at 0x7a6e40193250>

    def _equal(actual, equals_fn=equals_fn):
      expected_list = list(expected)
    
      # Try to compare actual and expected by sorting. This fails with a
      # TypeError in Python 3 if different types are present in the same
      # collection. It can also raise false negatives for types that don't have
      # a deterministic sort order, like pyarrow Tables as of 0.14.1
      if not equals_fn:
        equals_fn = lambda e, a: e == a
        try:
          sorted_expected = sorted(expected)
          sorted_actual = sorted(actual)
          if sorted_expected == sorted_actual:
            return
        except TypeError:
          pass
      # Slower method, used in two cases:
      # 1) If sorted expected != actual, use this method to verify the inequality.
      #    This ensures we don't raise any false negatives for types that don't
      #    have a deterministic sort order.
      # 2) As a fallback if we encounter a TypeError in python 3. this method
      #    works on collections that have different types.
      unexpected = []
      for element in actual:
        found = False
        for i, v in enumerate(expected_list):
          if equals_fn(v, element):
            found = True
            expected_list.pop(i)
            break
        if not found:
          unexpected.append(element)
      if unexpected or expected_list:
        msg = 'Failed assert: %r == %r' % (expected, actual)
        if unexpected:
          msg = msg + ', unexpected elements %r' % unexpected
        if expected_list:
          msg = msg + ', missing elements %r' % expected_list
>       raise BeamAssertException(msg)
E       apache_beam.testing.util.BeamAssertException: Failed assert: [1, 2, 3] == [4, 5, 6], unexpected elements [4, 5, 6], missing elements [1, 2, 3] [while running 'assert:inferences/Match']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/util.py:192: BeamAssertException

RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x7c10d81160e0>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
y: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmpmt9_3ac_/99eb30/tftransform_tmp/4c9c43c3657243b7a63ca72a188d6826'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     AssertionError

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: AssertionError

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.transforms.base_test.BaseMLTransformTest testMethod=test_ml_transform_on_multiple_columns_single_transform>

    @unittest.skipIf(tft is None, 'tft module is not installed.')
    def test_ml_transform_on_multiple_columns_single_transform(self):
      transforms = [tft.ScaleTo01(columns=['x', 'y'])]
      data = [{'x': [1, 2, 3], 'y': [1.0, 10.0, 20.0]}]
>     with beam.Pipeline() as p:

apache_beam/ml/transforms/base_test.py:226: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1282: in process_bundle
    op.finish()
apache_beam/runners/worker/operations.py:981: in apache_beam.runners.worker.operations.DoOperation.finish
    def finish(self):
apache_beam/runners/worker/operations.py:984: in apache_beam.runners.worker.operations.DoOperation.finish
    with self.scoped_finish_state:
apache_beam/runners/worker/operations.py:985: in apache_beam.runners.worker.operations.DoOperation.finish
    self.dofn_runner.finish()
apache_beam/runners/common.py:1573: in apache_beam.runners.common.DoFnRunner.finish
    self._invoke_bundle_method(self.do_fn_invoker.invoke_finish_bundle)
apache_beam/runners/common.py:1554: in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
    self._reraise_augmented(exn)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1552: in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
    bundle_method()
apache_beam/runners/common.py:615: in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
    def invoke_finish_bundle(self):
apache_beam/runners/common.py:620: in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
    self.output_handler.finish_bundle_outputs(
apache_beam/runners/common.py:1844: in apache_beam.runners.common._OutputHandler.finish_bundle_outputs
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x7c10d81160e0>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
y: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmpmt9_3ac_/99eb30/tftransform_tmp/4c9c43c3657243b7a63ca72a188d6826'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: RuntimeError

apache_beam.testing.util.BeamAssertException: Failed assert: [1, 2, 3] == [4, 5, 6], unexpected elements [4, 5, 6], missing elements [1, 2, 3] [while running 'assert:inferences/Match']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1055: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.process_method(*args_for_process, **kwargs_for_process),
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/transforms/core.py:2084: in <lambda>
    wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

actual = [4, 5, 6]
equals_fn = <function equal_to.<locals>._equal.<locals>.<lambda> at 0x7a6e28460160>

    def _equal(actual, equals_fn=equals_fn):
      expected_list = list(expected)
    
      # Try to compare actual and expected by sorting. This fails with a
      # TypeError in Python 3 if different types are present in the same
      # collection. It can also raise false negatives for types that don't have
      # a deterministic sort order, like pyarrow Tables as of 0.14.1
      if not equals_fn:
        equals_fn = lambda e, a: e == a
        try:
          sorted_expected = sorted(expected)
          sorted_actual = sorted(actual)
          if sorted_expected == sorted_actual:
            return
        except TypeError:
          pass
      # Slower method, used in two cases:
      # 1) If sorted expected != actual, use this method to verify the inequality.
      #    This ensures we don't raise any false negatives for types that don't
      #    have a deterministic sort order.
      # 2) As a fallback if we encounter a TypeError in python 3. this method
      #    works on collections that have different types.
      unexpected = []
      for element in actual:
        found = False
        for i, v in enumerate(expected_list):
          if equals_fn(v, element):
            found = True
            expected_list.pop(i)
            break
        if not found:
          unexpected.append(element)
      if unexpected or expected_list:
        msg = 'Failed assert: %r == %r' % (expected, actual)
        if unexpected:
          msg = msg + ', unexpected elements %r' % unexpected
        if expected_list:
          msg = msg + ', missing elements %r' % expected_list
>       raise BeamAssertException(msg)
E       apache_beam.testing.util.BeamAssertException: Failed assert: [1, 2, 3] == [4, 5, 6], unexpected elements [4, 5, 6], missing elements [1, 2, 3]

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/util.py:192: BeamAssertException

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.inference.base_test.RunInferenceBaseTest testMethod=test_run_inference_loads_different_models_multi_process_shared>

    def test_run_inference_loads_different_models_multi_process_shared(self):
      mh1 = FakeModelHandler(
          incrementing=True, min_batch_size=3, multi_process_shared=True)
>     with TestPipeline() as pipeline:

apache_beam/ml/inference/base_test.py:1737: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/test_pipeline.py:115: in run
    result = super().run(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:594: in run
    self._options).run(False)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1274: in process_bundle
    input_op_by_transform_id[element.transform_id].process_encoded(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:237: in process_encoded
    self.output(decoded_value)
apache_beam/runners/worker/operations.py:566: in apache_beam.runners.worker.operations.Operation.output
    def output(self, windowed_value, output_index=0):
apache_beam/runners/worker/operations.py:568: in apache_beam.runners.worker.operations.Operation.output
    _cast_to_receiver(self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:259: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    def receive(self, windowed_value):
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1055: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.process_method(*args_for_process, **kwargs_for_process),
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/transforms/core.py:2084: in <lambda>
    wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

actual = [4, 5, 6]
equals_fn = <function equal_to.<locals>._equal.<locals>.<lambda> at 0x7a6e28460160>

    def _equal(actual, equals_fn=equals_fn):
      expected_list = list(expected)
    
      # Try to compare actual and expected by sorting. This fails with a
      # TypeError in Python 3 if different types are present in the same
      # collection. It can also raise false negatives for types that don't have
      # a deterministic sort order, like pyarrow Tables as of 0.14.1
      if not equals_fn:
        equals_fn = lambda e, a: e == a
        try:
          sorted_expected = sorted(expected)
          sorted_actual = sorted(actual)
          if sorted_expected == sorted_actual:
            return
        except TypeError:
          pass
      # Slower method, used in two cases:
      # 1) If sorted expected != actual, use this method to verify the inequality.
      #    This ensures we don't raise any false negatives for types that don't
      #    have a deterministic sort order.
      # 2) As a fallback if we encounter a TypeError in python 3. this method
      #    works on collections that have different types.
      unexpected = []
      for element in actual:
        found = False
        for i, v in enumerate(expected_list):
          if equals_fn(v, element):
            found = True
            expected_list.pop(i)
            break
        if not found:
          unexpected.append(element)
      if unexpected or expected_list:
        msg = 'Failed assert: %r == %r' % (expected, actual)
        if unexpected:
          msg = msg + ', unexpected elements %r' % unexpected
        if expected_list:
          msg = msg + ', missing elements %r' % expected_list
>       raise BeamAssertException(msg)
E       apache_beam.testing.util.BeamAssertException: Failed assert: [1, 2, 3] == [4, 5, 6], unexpected elements [4, 5, 6], missing elements [1, 2, 3] [while running 'assert:inferences/Match']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/util.py:192: BeamAssertException

RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x78f2844168c0>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: int64>
  child 0, item: int64
y: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmporhg5huk/7c9201/tftransform_tmp/fb58c7e25734448687e897b478b144a0'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     AssertionError

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: AssertionError

During handling of the above exception, another exception occurred:

a = (<apache_beam.ml.transforms.base_test.BaseMLTransformTest testMethod=test_ml_transform_dict_output_pcoll_schema_1>,)
kw = {}

    @wraps(func)
    def standalone_func(*a, **kw):
>       return func(*(a + p.args), **p.kwargs, **kw)

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/parameterized/parameterized.py:620: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/ml/transforms/base_test.py:193: in test_ml_transform_dict_output_pcoll_schema
    with beam.Pipeline() as p:
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1282: in process_bundle
    op.finish()
apache_beam/runners/worker/operations.py:981: in apache_beam.runners.worker.operations.DoOperation.finish
    def finish(self):
apache_beam/runners/worker/operations.py:984: in apache_beam.runners.worker.operations.DoOperation.finish
    with self.scoped_finish_state:
apache_beam/runners/worker/operations.py:985: in apache_beam.runners.worker.operations.DoOperation.finish
    self.dofn_runner.finish()
apache_beam/runners/common.py:1573: in apache_beam.runners.common.DoFnRunner.finish
    self._invoke_bundle_method(self.do_fn_invoker.invoke_finish_bundle)
apache_beam/runners/common.py:1554: in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
    self._reraise_augmented(exn)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1552: in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
    bundle_method()
apache_beam/runners/common.py:615: in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
    def invoke_finish_bundle(self):
apache_beam/runners/common.py:620: in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
    self.output_handler.finish_bundle_outputs(
apache_beam/runners/common.py:1844: in apache_beam.runners.common._OutputHandler.finish_bundle_outputs
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x78f2844168c0>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: int64>
  child 0, item: int64
y: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmporhg5huk/7c9201/tftransform_tmp/fb58c7e25734448687e897b478b144a0'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: RuntimeError

RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x7c1084da2bc0>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
y: large_list<item: large_binary>
  child 0, item: large_binary
saved_model_dir = '/tmp/tmpe7i70sfz/f54ce1/tftransform_tmp/f1140a7dc5364b1d93a1379886f5f029'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     AssertionError

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: AssertionError

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.transforms.base_test.BaseMLTransformTest testMethod=test_ml_transforms_on_multiple_columns_multiple_transforms>

    @unittest.skipIf(tft is None, 'tft module is not installed.')
    def test_ml_transforms_on_multiple_columns_multiple_transforms(self):
      transforms = [
          tft.ScaleTo01(columns=['x']),
          tft.ComputeAndApplyVocabulary(columns=['y'])
      ]
      data = [{'x': [1, 2, 3], 'y': ['a', 'b', 'c']}]
>     with beam.Pipeline() as p:

apache_beam/ml/transforms/base_test.py:254: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1282: in process_bundle
    op.finish()
apache_beam/runners/worker/operations.py:981: in apache_beam.runners.worker.operations.DoOperation.finish
    def finish(self):
apache_beam/runners/worker/operations.py:984: in apache_beam.runners.worker.operations.DoOperation.finish
    with self.scoped_finish_state:
apache_beam/runners/worker/operations.py:985: in apache_beam.runners.worker.operations.DoOperation.finish
    self.dofn_runner.finish()
apache_beam/runners/common.py:1573: in apache_beam.runners.common.DoFnRunner.finish
    self._invoke_bundle_method(self.do_fn_invoker.invoke_finish_bundle)
apache_beam/runners/common.py:1554: in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
    self._reraise_augmented(exn)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1552: in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
    bundle_method()
apache_beam/runners/common.py:615: in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
    def invoke_finish_bundle(self):
apache_beam/runners/common.py:620: in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
    self.output_handler.finish_bundle_outputs(
apache_beam/runners/common.py:1844: in apache_beam.runners.common._OutputHandler.finish_bundle_outputs
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x7c1084da2bc0>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
y: large_list<item: large_binary>
  child 0, item: large_binary
saved_model_dir = '/tmp/tmpe7i70sfz/f54ce1/tftransform_tmp/f1140a7dc5364b1d93a1379886f5f029'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: RuntimeError

RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x78f2357e6b00>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: int64>
  child 0, item: int64
y: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmphfl_r44g/4b8802/tftransform_tmp/b5b2287666634240b0af89e5616bfccb'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     AssertionError

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: AssertionError

During handling of the above exception, another exception occurred:

a = (<apache_beam.ml.transforms.base_test.BaseMLTransformTest testMethod=test_ml_transform_dict_output_pcoll_schema_2>,)
kw = {}

    @wraps(func)
    def standalone_func(*a, **kw):
>       return func(*(a + p.args), **p.kwargs, **kw)

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/parameterized/parameterized.py:620: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/ml/transforms/base_test.py:193: in test_ml_transform_dict_output_pcoll_schema
    with beam.Pipeline() as p:
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1282: in process_bundle
    op.finish()
apache_beam/runners/worker/operations.py:981: in apache_beam.runners.worker.operations.DoOperation.finish
    def finish(self):
apache_beam/runners/worker/operations.py:984: in apache_beam.runners.worker.operations.DoOperation.finish
    with self.scoped_finish_state:
apache_beam/runners/worker/operations.py:985: in apache_beam.runners.worker.operations.DoOperation.finish
    self.dofn_runner.finish()
apache_beam/runners/common.py:1573: in apache_beam.runners.common.DoFnRunner.finish
    self._invoke_bundle_method(self.do_fn_invoker.invoke_finish_bundle)
apache_beam/runners/common.py:1554: in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
    self._reraise_augmented(exn)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1552: in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
    bundle_method()
apache_beam/runners/common.py:615: in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
    def invoke_finish_bundle(self):
apache_beam/runners/common.py:620: in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
    self.output_handler.finish_bundle_outputs(
apache_beam/runners/common.py:1844: in apache_beam.runners.common._OutputHandler.finish_bundle_outputs
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x78f2357e6b00>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: int64>
  child 0, item: int64
y: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmphfl_r44g/4b8802/tftransform_tmp/b5b2287666634240b0af89e5616bfccb'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: RuntimeError

RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x7c1084de7250>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
y: large_list<item: large_binary>
  child 0, item: large_binary
saved_model_dir = '/tmp/tmpr7k_1jgu/d1624a/tftransform_tmp/325dc52e456c4f68aa0ca60abbd71f1a'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     AssertionError

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: AssertionError

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.transforms.base_test.BaseMLTransformTest testMethod=test_mltransform_with_counter>

    @unittest.skipIf(tft is None, 'tft module is not installed.')
    def test_mltransform_with_counter(self):
      transforms = [
          tft.ComputeAndApplyVocabulary(columns=['y']),
          tft.ScaleTo01(columns=['x'])
      ]
      data = [{'x': [1, 2, 3], 'y': ['a', 'b', 'c']}]
>     with beam.Pipeline() as p:

apache_beam/ml/transforms/base_test.py:283: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1282: in process_bundle
    op.finish()
apache_beam/runners/worker/operations.py:981: in apache_beam.runners.worker.operations.DoOperation.finish
    def finish(self):
apache_beam/runners/worker/operations.py:984: in apache_beam.runners.worker.operations.DoOperation.finish
    with self.scoped_finish_state:
apache_beam/runners/worker/operations.py:985: in apache_beam.runners.worker.operations.DoOperation.finish
    self.dofn_runner.finish()
apache_beam/runners/common.py:1573: in apache_beam.runners.common.DoFnRunner.finish
    self._invoke_bundle_method(self.do_fn_invoker.invoke_finish_bundle)
apache_beam/runners/common.py:1554: in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
    self._reraise_augmented(exn)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1552: in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
    bundle_method()
apache_beam/runners/common.py:615: in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
    def invoke_finish_bundle(self):
apache_beam/runners/common.py:620: in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
    self.output_handler.finish_bundle_outputs(
apache_beam/runners/common.py:1844: in apache_beam.runners.common._OutputHandler.finish_bundle_outputs
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x7c1084de7250>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
y: large_list<item: large_binary>
  child 0, item: large_binary
saved_model_dir = '/tmp/tmpr7k_1jgu/d1624a/tftransform_tmp/325dc52e456c4f68aa0ca60abbd71f1a'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: RuntimeError

RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x78f2945a2890>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: int64>
  child 0, item: int64
y: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmpb7730azg/f50ee9/tftransform_tmp/937b3047754e4204aba8e652674bfdfd'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     AssertionError

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: AssertionError

During handling of the above exception, another exception occurred:

a = (<apache_beam.ml.transforms.base_test.BaseMLTransformTest testMethod=test_ml_transform_dict_output_pcoll_schema_3>,)
kw = {}

    @wraps(func)
    def standalone_func(*a, **kw):
>       return func(*(a + p.args), **p.kwargs, **kw)

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/parameterized/parameterized.py:620: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/ml/transforms/base_test.py:193: in test_ml_transform_dict_output_pcoll_schema
    with beam.Pipeline() as p:
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1282: in process_bundle
    op.finish()
apache_beam/runners/worker/operations.py:981: in apache_beam.runners.worker.operations.DoOperation.finish
    def finish(self):
apache_beam/runners/worker/operations.py:984: in apache_beam.runners.worker.operations.DoOperation.finish
    with self.scoped_finish_state:
apache_beam/runners/worker/operations.py:985: in apache_beam.runners.worker.operations.DoOperation.finish
    self.dofn_runner.finish()
apache_beam/runners/common.py:1573: in apache_beam.runners.common.DoFnRunner.finish
    self._invoke_bundle_method(self.do_fn_invoker.invoke_finish_bundle)
apache_beam/runners/common.py:1554: in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
    self._reraise_augmented(exn)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1552: in apache_beam.runners.common.DoFnRunner._invoke_bundle_method
    bundle_method()
apache_beam/runners/common.py:615: in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
    def invoke_finish_bundle(self):
apache_beam/runners/common.py:620: in apache_beam.runners.common.DoFnInvoker.invoke_finish_bundle
    self.output_handler.finish_bundle_outputs(
apache_beam/runners/common.py:1844: in apache_beam.runners.common._OutputHandler.finish_bundle_outputs
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x78f2945a2890>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: int64>
  child 0, item: int64
y: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmpb7730azg/f50ee9/tftransform_tmp/937b3047754e4204aba8e652674bfdfd'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: RuntimeError

RuntimeError: AssertionError [while running 'TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x7cee8477dde0>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmpd4nud91j/tftransform_tmp/3c04f34b2001447e8e983a8e1cc5ab2f'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     AssertionError

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: AssertionError

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.transforms.handlers_test.TFTProcessHandlerTest testMethod=test_consume_mode_with_extra_columns_in_the_input>

    def test_consume_mode_with_extra_columns_in_the_input(self):
>     with beam.Pipeline() as p:

apache_beam/ml/transforms/handlers_test.py:515: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1274: in process_bundle
    input_op_by_transform_id[element.transform_id].process_encoded(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:237: in process_encoded
    self.output(decoded_value)
apache_beam/runners/worker/operations.py:566: in apache_beam.runners.worker.operations.Operation.output
    def output(self, windowed_value, output_index=0):
apache_beam/runners/worker/operations.py:568: in apache_beam.runners.worker.operations.Operation.output
    _cast_to_receiver(self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:259: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    def receive(self, windowed_value):
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x7cee8477dde0>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmpd4nud91j/tftransform_tmp/3c04f34b2001447e8e983a8e1cc5ab2f'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     RuntimeError: AssertionError [while running 'TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: RuntimeError

RuntimeError: AssertionError [while running 'TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x79d36427c850>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmpirexxy98/tftransform_tmp/7a2cb35dc75b4f41a20e41de1adda8dc'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     AssertionError

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: AssertionError

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.transforms.handlers_test.TFTProcessHandlerTest testMethod=test_tft_process_handler_unused_column>

    def test_tft_process_handler_unused_column(self):
      data = [
          {
              'x': [5, 1], 'y': 2
          },
          {
              'x': [8, 4], 'y': 5
          },
          {
              'x': [9, 5], 'y': 6
          },
          {
              'x': [10, 6], 'y': 7
          },
          {
              'x': [11, 7], 'y': 8
          },
          {
              'x': [12, 8], 'y': 9
          },
          {
              'x': [13, 9], 'y': 10
          },
          {
              'x': [14, 10], 'y': 11
          },
          {
              'x': [15, 11], 'y': 12
          },
          {
              'x': [16, 12], 'y': 13
          },
          {
              'x': [17, 13], 'y': 14
          },
          {
              'x': [18, 14], 'y': 15
          },
          {
              'x': [19, 15], 'y': 16
          },
          {
              'x': [20, 16], 'y': 17
          },
          {
              'x': [21, 17], 'y': 18
          },
          {
              'x': [22, 18], 'y': 19
          },
          {
              'x': [23, 19], 'y': 20
          },
          {
              'x': [24, 20], 'y': 21
          },
          {
              'x': [25, 21], 'y': 22
          },
      ]
      # pylint: disable=line-too-long
      expected_data = [
          beam.Row(
              x=np.array([0.16666667, 0.], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=2),
          beam.Row(
              x=np.array([0.29166666, 0.125], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=5),
          beam.Row(
              x=np.array([0.33333334, 0.16666667], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=6),
          beam.Row(
              x=np.array([0.375, 0.20833333], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=7),
          beam.Row(
              x=np.array([0.41666666, 0.25], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=8),
          beam.Row(
              x=np.array([0.45833334, 0.29166666], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=9),
          beam.Row(
              x=np.array([0.5, 0.33333334], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=10),
          beam.Row(
              x=np.array([0.5416667, 0.375], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=11),
          beam.Row(
              x=np.array([0.5833333, 0.41666666], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=12),
          beam.Row(
              x=np.array([0.625, 0.45833334], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=13),
          beam.Row(
              x=np.array([0.6666667, 0.5], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=14),
          beam.Row(
              x=np.array([0.7083333, 0.5416667], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=15),
          beam.Row(
              x=np.array([0.75, 0.5833333], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=16),
          beam.Row(
              x=np.array([0.7916667, 0.625], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=17),
          beam.Row(
              x=np.array([0.8333333, 0.6666667], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=18),
          beam.Row(
              x=np.array([0.875, 0.7083333], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=19),
          beam.Row(
              x=np.array([0.9166667, 0.75], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=20),
          beam.Row(
              x=np.array([0.9583333, 0.7916667], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=21),
          beam.Row(
              x=np.array([1., 0.8333333], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=22),
      ]
      # pylint: enable=line-too-long
    
      expected_data_x = [row.x for row in expected_data]
      expected_data_y = [row.y for row in expected_data]
    
      scale_to_0_1_fn = tft.ScaleTo01(columns=['x'])
>     with beam.Pipeline() as p:

apache_beam/ml/transforms/handlers_test.py:495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1274: in process_bundle
    input_op_by_transform_id[element.transform_id].process_encoded(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:237: in process_encoded
    self.output(decoded_value)
apache_beam/runners/worker/operations.py:566: in apache_beam.runners.worker.operations.Operation.output
    def output(self, windowed_value, output_index=0):
apache_beam/runners/worker/operations.py:568: in apache_beam.runners.worker.operations.Operation.output
    _cast_to_receiver(self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:259: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    def receive(self, windowed_value):
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x79d36427c850>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmpirexxy98/tftransform_tmp/7a2cb35dc75b4f41a20e41de1adda8dc'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     RuntimeError: AssertionError [while running 'TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: RuntimeError

apache_beam.testing.util.BeamAssertException: Failed assert: [PredictionResult(example=1744404304, inference=0, model_id='fake_model_id_default'), PredictionResult(example=1744404307, inference=0, model_id='fake_model_id_default'), PredictionResult(example=1744404314, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404321, inference=0, model_id='fake_model_id_1'), PredictionResult(example=1744404328, inference=0, model_id='fake_model_id_2'), PredictionResult(example=1744404335, inference=1, model_id='fake_model_id_1')] == [PredictionResult(example=1744404304, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404307, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404314, inference=3, model_id='fake_model_id_default'), PredictionResult(example=1744404321, inference=1, model_id='fake_model_id_1'), PredictionResult(example=1744404328, inference=1, model_id='fake_model_id_2'), PredictionResult(example=1744404335, inference=3, model_id='fake_model_id_1')], unexpected elements [PredictionResult(example=1744404304, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404307, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404314, inference=3, model_id='fake_model_id_default'), PredictionResult(example=1744404321, inference=1, model_id='fake_model_id_1'), PredictionResult(example=1744404328, inference=1, model_id='fake_model_id_2'), PredictionResult(example=1744404335, inference=3, model_id='fake_model_id_1')], missing elements [PredictionResult(example=1744404304, inference=0, model_id='fake_model_id_default'), PredictionResult(example=1744404307, inference=0, model_id='fake_model_id_default'), PredictionResult(example=1744404314, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404321, inference=0, model_id='fake_model_id_1'), PredictionResult(example=1744404328, inference=0, model_id='fake_model_id_2'), PredictionResult(example=1744404335, inference=1, model_id='fake_model_id_1')] [while running 'assert_that/Match']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1055: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.process_method(*args_for_process, **kwargs_for_process),
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/transforms/core.py:2084: in <lambda>
    wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

actual = [PredictionResult(example=1744404304, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404...nference=1, model_id='fake_model_id_2'), PredictionResult(example=1744404335, inference=3, model_id='fake_model_id_1')]
equals_fn = <function equal_to.<locals>._equal.<locals>.<lambda> at 0x7ffa137dfd90>

    def _equal(actual, equals_fn=equals_fn):
      expected_list = list(expected)
    
      # Try to compare actual and expected by sorting. This fails with a
      # TypeError in Python 3 if different types are present in the same
      # collection. It can also raise false negatives for types that don't have
      # a deterministic sort order, like pyarrow Tables as of 0.14.1
      if not equals_fn:
        equals_fn = lambda e, a: e == a
        try:
          sorted_expected = sorted(expected)
          sorted_actual = sorted(actual)
          if sorted_expected == sorted_actual:
            return
        except TypeError:
          pass
      # Slower method, used in two cases:
      # 1) If sorted expected != actual, use this method to verify the inequality.
      #    This ensures we don't raise any false negatives for types that don't
      #    have a deterministic sort order.
      # 2) As a fallback if we encounter a TypeError in python 3. this method
      #    works on collections that have different types.
      unexpected = []
      for element in actual:
        found = False
        for i, v in enumerate(expected_list):
          if equals_fn(v, element):
            found = True
            expected_list.pop(i)
            break
        if not found:
          unexpected.append(element)
      if unexpected or expected_list:
        msg = 'Failed assert: %r == %r' % (expected, actual)
        if unexpected:
          msg = msg + ', unexpected elements %r' % unexpected
        if expected_list:
          msg = msg + ', missing elements %r' % expected_list
>       raise BeamAssertException(msg)
E       apache_beam.testing.util.BeamAssertException: Failed assert: [PredictionResult(example=1744404304, inference=0, model_id='fake_model_id_default'), PredictionResult(example=1744404307, inference=0, model_id='fake_model_id_default'), PredictionResult(example=1744404314, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404321, inference=0, model_id='fake_model_id_1'), PredictionResult(example=1744404328, inference=0, model_id='fake_model_id_2'), PredictionResult(example=1744404335, inference=1, model_id='fake_model_id_1')] == [PredictionResult(example=1744404304, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404307, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404314, inference=3, model_id='fake_model_id_default'), PredictionResult(example=1744404321, inference=1, model_id='fake_model_id_1'), PredictionResult(example=1744404328, inference=1, model_id='fake_model_id_2'), PredictionResult(example=1744404335, inference=3, model_id='fake_model_id_1')], unexpected elements [PredictionResult(example=1744404304, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404307, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404314, inference=3, model_id='fake_model_id_default'), PredictionResult(example=1744404321, inference=1, model_id='fake_model_id_1'), PredictionResult(example=1744404328, inference=1, model_id='fake_model_id_2'), PredictionResult(example=1744404335, inference=3, model_id='fake_model_id_1')], missing elements [PredictionResult(example=1744404304, inference=0, model_id='fake_model_id_default'), PredictionResult(example=1744404307, inference=0, model_id='fake_model_id_default'), PredictionResult(example=1744404314, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404321, inference=0, model_id='fake_model_id_1'), PredictionResult(example=1744404328, inference=0, model_id='fake_model_id_2'), PredictionResult(example=1744404335, inference=1, model_id='fake_model_id_1')]

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/util.py:192: BeamAssertException

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.inference.base_test.RunInferenceBaseTest testMethod=test_run_inference_side_input_in_batch_per_key_models>

    def test_run_inference_side_input_in_batch_per_key_models(self):
      first_ts = math.floor(time.time()) - 30
      interval = 7
    
      sample_main_input_elements = ([
          ('key1', first_ts - 2),
          ('key2', first_ts + 1),
          ('key2', first_ts + 8),
          ('key1', first_ts + 15),
          ('key2', first_ts + 22),
          ('key1', first_ts + 29),
      ])
    
      sample_side_input_elements = [
          (
              first_ts + 1,
              [
                  base.KeyModelPathMapping(
                      keys=['key1'], update_path='fake_model_id_default'),
                  base.KeyModelPathMapping(
                      keys=['key2'], update_path='fake_model_id_default')
              ]),
          # if model_id is empty string, we use the default model
          # handler model URI.
          (
              first_ts + 8,
              [
                  base.KeyModelPathMapping(
                      keys=['key1'], update_path='fake_model_id_1'),
                  base.KeyModelPathMapping(
                      keys=['key2'], update_path='fake_model_id_default')
              ]),
          (
              first_ts + 15,
              [
                  base.KeyModelPathMapping(
                      keys=['key1'], update_path='fake_model_id_1'),
                  base.KeyModelPathMapping(
                      keys=['key2'], update_path='fake_model_id_2')
              ]),
      ]
    
      model_handler = base.KeyedModelHandler([
          base.KeyModelMapping(['key1'],
                               FakeModelHandlerReturnsPredictionResult(
                                   multi_process_shared=True, state=True)),
          base.KeyModelMapping(['key2'],
                               FakeModelHandlerReturnsPredictionResult(
                                   multi_process_shared=True, state=True))
      ])
    
      class _EmitElement(beam.DoFn):
        def process(self, element):
          for e in element:
            yield e
    
>     with TestPipeline() as pipeline:

apache_beam/ml/inference/base_test.py:1330: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/test_pipeline.py:115: in run
    result = super().run(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:594: in run
    self._options).run(False)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1274: in process_bundle
    input_op_by_transform_id[element.transform_id].process_encoded(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:237: in process_encoded
    self.output(decoded_value)
apache_beam/runners/worker/operations.py:566: in apache_beam.runners.worker.operations.Operation.output
    def output(self, windowed_value, output_index=0):
apache_beam/runners/worker/operations.py:568: in apache_beam.runners.worker.operations.Operation.output
    _cast_to_receiver(self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:259: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    def receive(self, windowed_value):
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1055: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.process_method(*args_for_process, **kwargs_for_process),
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/transforms/core.py:2084: in <lambda>
    wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

actual = [PredictionResult(example=1744404304, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404...nference=1, model_id='fake_model_id_2'), PredictionResult(example=1744404335, inference=3, model_id='fake_model_id_1')]
equals_fn = <function equal_to.<locals>._equal.<locals>.<lambda> at 0x7ffa137dfd90>

    def _equal(actual, equals_fn=equals_fn):
      expected_list = list(expected)
    
      # Try to compare actual and expected by sorting. This fails with a
      # TypeError in Python 3 if different types are present in the same
      # collection. It can also raise false negatives for types that don't have
      # a deterministic sort order, like pyarrow Tables as of 0.14.1
      if not equals_fn:
        equals_fn = lambda e, a: e == a
        try:
          sorted_expected = sorted(expected)
          sorted_actual = sorted(actual)
          if sorted_expected == sorted_actual:
            return
        except TypeError:
          pass
      # Slower method, used in two cases:
      # 1) If sorted expected != actual, use this method to verify the inequality.
      #    This ensures we don't raise any false negatives for types that don't
      #    have a deterministic sort order.
      # 2) As a fallback if we encounter a TypeError in python 3. this method
      #    works on collections that have different types.
      unexpected = []
      for element in actual:
        found = False
        for i, v in enumerate(expected_list):
          if equals_fn(v, element):
            found = True
            expected_list.pop(i)
            break
        if not found:
          unexpected.append(element)
      if unexpected or expected_list:
        msg = 'Failed assert: %r == %r' % (expected, actual)
        if unexpected:
          msg = msg + ', unexpected elements %r' % unexpected
        if expected_list:
          msg = msg + ', missing elements %r' % expected_list
>       raise BeamAssertException(msg)
E       apache_beam.testing.util.BeamAssertException: Failed assert: [PredictionResult(example=1744404304, inference=0, model_id='fake_model_id_default'), PredictionResult(example=1744404307, inference=0, model_id='fake_model_id_default'), PredictionResult(example=1744404314, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404321, inference=0, model_id='fake_model_id_1'), PredictionResult(example=1744404328, inference=0, model_id='fake_model_id_2'), PredictionResult(example=1744404335, inference=1, model_id='fake_model_id_1')] == [PredictionResult(example=1744404304, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404307, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404314, inference=3, model_id='fake_model_id_default'), PredictionResult(example=1744404321, inference=1, model_id='fake_model_id_1'), PredictionResult(example=1744404328, inference=1, model_id='fake_model_id_2'), PredictionResult(example=1744404335, inference=3, model_id='fake_model_id_1')], unexpected elements [PredictionResult(example=1744404304, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404307, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404314, inference=3, model_id='fake_model_id_default'), PredictionResult(example=1744404321, inference=1, model_id='fake_model_id_1'), PredictionResult(example=1744404328, inference=1, model_id='fake_model_id_2'), PredictionResult(example=1744404335, inference=3, model_id='fake_model_id_1')], missing elements [PredictionResult(example=1744404304, inference=0, model_id='fake_model_id_default'), PredictionResult(example=1744404307, inference=0, model_id='fake_model_id_default'), PredictionResult(example=1744404314, inference=1, model_id='fake_model_id_default'), PredictionResult(example=1744404321, inference=0, model_id='fake_model_id_1'), PredictionResult(example=1744404328, inference=0, model_id='fake_model_id_2'), PredictionResult(example=1744404335, inference=1, model_id='fake_model_id_1')] [while running 'assert_that/Match']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/testing/util.py:192: BeamAssertException

RuntimeError: AssertionError [while running 'TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x7a6e407943a0>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
saved_model_dir = 'gs://temp-storage-for-perf-tests/tft_handler/5dd1df5d62ea4b059abfe6f30bc55b39/tftransform_tmp/3b21d4ac03f3403b814d4cb917043b02'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     AssertionError

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: AssertionError

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.transforms.handlers_test.TFTProcessHandlerTestWithGCSLocation testMethod=test_tft_process_handler_unused_column>

    def test_tft_process_handler_unused_column(self):
      data = [
          {
              'x': [5, 1], 'y': 2
          },
          {
              'x': [8, 4], 'y': 5
          },
          {
              'x': [9, 5], 'y': 6
          },
          {
              'x': [10, 6], 'y': 7
          },
          {
              'x': [11, 7], 'y': 8
          },
          {
              'x': [12, 8], 'y': 9
          },
          {
              'x': [13, 9], 'y': 10
          },
          {
              'x': [14, 10], 'y': 11
          },
          {
              'x': [15, 11], 'y': 12
          },
          {
              'x': [16, 12], 'y': 13
          },
          {
              'x': [17, 13], 'y': 14
          },
          {
              'x': [18, 14], 'y': 15
          },
          {
              'x': [19, 15], 'y': 16
          },
          {
              'x': [20, 16], 'y': 17
          },
          {
              'x': [21, 17], 'y': 18
          },
          {
              'x': [22, 18], 'y': 19
          },
          {
              'x': [23, 19], 'y': 20
          },
          {
              'x': [24, 20], 'y': 21
          },
          {
              'x': [25, 21], 'y': 22
          },
      ]
      # pylint: disable=line-too-long
      expected_data = [
          beam.Row(
              x=np.array([0.16666667, 0.], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=2),
          beam.Row(
              x=np.array([0.29166666, 0.125], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=5),
          beam.Row(
              x=np.array([0.33333334, 0.16666667], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=6),
          beam.Row(
              x=np.array([0.375, 0.20833333], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=7),
          beam.Row(
              x=np.array([0.41666666, 0.25], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=8),
          beam.Row(
              x=np.array([0.45833334, 0.29166666], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=9),
          beam.Row(
              x=np.array([0.5, 0.33333334], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=10),
          beam.Row(
              x=np.array([0.5416667, 0.375], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=11),
          beam.Row(
              x=np.array([0.5833333, 0.41666666], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=12),
          beam.Row(
              x=np.array([0.625, 0.45833334], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=13),
          beam.Row(
              x=np.array([0.6666667, 0.5], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=14),
          beam.Row(
              x=np.array([0.7083333, 0.5416667], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=15),
          beam.Row(
              x=np.array([0.75, 0.5833333], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=16),
          beam.Row(
              x=np.array([0.7916667, 0.625], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=17),
          beam.Row(
              x=np.array([0.8333333, 0.6666667], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=18),
          beam.Row(
              x=np.array([0.875, 0.7083333], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=19),
          beam.Row(
              x=np.array([0.9166667, 0.75], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=20),
          beam.Row(
              x=np.array([0.9583333, 0.7916667], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=21),
          beam.Row(
              x=np.array([1., 0.8333333], dtype=np.float32),
              x_max=np.array([25.], dtype=np.float32),
              x_min=np.array([1.], dtype=np.float32),
              y=22),
      ]
      # pylint: enable=line-too-long
    
      expected_data_x = [row.x for row in expected_data]
      expected_data_y = [row.y for row in expected_data]
    
      scale_to_0_1_fn = tft.ScaleTo01(columns=['x'])
>     with beam.Pipeline() as p:

apache_beam/ml/transforms/handlers_test.py:495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1274: in process_bundle
    input_op_by_transform_id[element.transform_id].process_encoded(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:237: in process_encoded
    self.output(decoded_value)
apache_beam/runners/worker/operations.py:566: in apache_beam.runners.worker.operations.Operation.output
    def output(self, windowed_value, output_index=0):
apache_beam/runners/worker/operations.py:568: in apache_beam.runners.worker.operations.Operation.output
    _cast_to_receiver(self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:259: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    def receive(self, windowed_value):
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x7a6e407943a0>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
saved_model_dir = 'gs://temp-storage-for-perf-tests/tft_handler/5dd1df5d62ea4b059abfe6f30bc55b39/tftransform_tmp/3b21d4ac03f3403b814d4cb917043b02'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     RuntimeError: AssertionError [while running 'TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: RuntimeError

RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x78f254445960>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmpv7sb89d0/7a8d54/tftransform_tmp/8d19a27889754022ba91733688eff857'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     AssertionError

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: AssertionError

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.transforms.base_test.BaseMLTransformTest testMethod=test_ml_transform_on_dict>

    @unittest.skipIf(tft is None, 'tft module is not installed.')
    def test_ml_transform_on_dict(self):
      transforms = [tft.ScaleTo01(columns=['x'])]
      data = [{'x': 1}, {'x': 2}]
>     with beam.Pipeline() as p:

apache_beam/ml/transforms/base_test.py:102: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1274: in process_bundle
    input_op_by_transform_id[element.transform_id].process_encoded(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:237: in process_encoded
    self.output(decoded_value)
apache_beam/runners/worker/operations.py:566: in apache_beam.runners.worker.operations.Operation.output
    def output(self, windowed_value, output_index=0):
apache_beam/runners/worker/operations.py:568: in apache_beam.runners.worker.operations.Operation.output
    _cast_to_receiver(self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:259: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    def receive(self, windowed_value):
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x78f254445960>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmpv7sb89d0/7a8d54/tftransform_tmp/8d19a27889754022ba91733688eff857'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: RuntimeError

RuntimeError: AssertionError [while running 'TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x7cefa846c2b0>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: large_binary>
  child 0, item: large_binary
saved_model_dir = '/tmp/tmpbqdfvl2q/tftransform_tmp/8df430a5c89948f6ac53eeab541f050d'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     AssertionError

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: AssertionError

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.transforms.handlers_test.TFTProcessHandlerTest testMethod=test_handler_with_same_input_elements>

    def test_handler_with_same_input_elements(self):
>     with beam.Pipeline() as p:

apache_beam/ml/transforms/handlers_test.py:577: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1274: in process_bundle
    input_op_by_transform_id[element.transform_id].process_encoded(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:237: in process_encoded
    self.output(decoded_value)
apache_beam/runners/worker/operations.py:566: in apache_beam.runners.worker.operations.Operation.output
    def output(self, windowed_value, output_index=0):
apache_beam/runners/worker/operations.py:568: in apache_beam.runners.worker.operations.Operation.output
    _cast_to_receiver(self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:259: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    def receive(self, windowed_value):
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x7cefa846c2b0>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: large_binary>
  child 0, item: large_binary
saved_model_dir = '/tmp/tmpbqdfvl2q/tftransform_tmp/8df430a5c89948f6ac53eeab541f050d'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     RuntimeError: AssertionError [while running 'TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: RuntimeError

RuntimeError: AssertionError [while running 'TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x79d3e5cc8970>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmptnbep5zk/tftransform_tmp/4efb8b9cb6434b00bdf11c6111226f3c'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     AssertionError

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: AssertionError

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.transforms.handlers_test.TFTProcessHandlerTest testMethod=test_tft_process_handler_verify_artifacts>

    def test_tft_process_handler_verify_artifacts(self):
>     with beam.Pipeline() as p:

apache_beam/ml/transforms/handlers_test.py:290: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1274: in process_bundle
    input_op_by_transform_id[element.transform_id].process_encoded(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:237: in process_encoded
    self.output(decoded_value)
apache_beam/runners/worker/operations.py:566: in apache_beam.runners.worker.operations.Operation.output
    def output(self, windowed_value, output_index=0):
apache_beam/runners/worker/operations.py:568: in apache_beam.runners.worker.operations.Operation.output
    _cast_to_receiver(self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:259: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    def receive(self, windowed_value):
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x79d3e5cc8970>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmptnbep5zk/tftransform_tmp/4efb8b9cb6434b00bdf11c6111226f3c'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     RuntimeError: AssertionError [while running 'TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: RuntimeError

RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']
>   return self.do_fn_invoker.invoke_process(windowed_value)

apache_beam/runners/common.py:1495: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x7c10d83a4df0>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmpuka2v5x0/a84299/tftransform_tmp/1f6a464fcca24232aba526751dadddeb'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     AssertionError

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: AssertionError

During handling of the above exception, another exception occurred:

self = <apache_beam.ml.transforms.tft_test.ScaleZScoreTest testMethod=test_z_score>

    def test_z_score(self):
      data = [
          {
              'x': 1
          },
          {
              'x': 2
          },
          {
              'x': 3
          },
          {
              'x': 4
          },
          {
              'x': 5
          },
          {
              'x': 6
          },
      ]
    
>     with beam.Pipeline() as p:

apache_beam/ml/transforms/tft_test.py:72: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:644: in __exit__
    self.result = self.run()
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/pipeline.py:618: in run
    return self.runner.run_pipeline(self, self._options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py:190: in run_pipeline
    return runner.run_pipeline(pipeline, options)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:196: in run_pipeline
    self._latest_run_result = self.run_via_runner_api(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:223: in run_via_runner_api
    return self.run_stages(stage_context, stages)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:470: in run_stages
    bundle_results = self._execute_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:795: in _execute_bundle
    self._run_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1034: in _run_bundle
    result, splits = bundle_manager.process_bundle(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py:1360: in process_bundle
    result_future = self._worker_handler.control_conn.push(process_bundle_req)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/worker_handlers.py:386: in push
    response = self.worker.do_instruction(request)
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:658: in do_instruction
    return getattr(self, request_type)(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:696: in process_bundle
    bundle_processor.process_bundle(instruction_id))
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:1274: in process_bundle
    input_op_by_transform_id[element.transform_id].process_encoded(
target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:237: in process_encoded
    self.output(decoded_value)
apache_beam/runners/worker/operations.py:566: in apache_beam.runners.worker.operations.Operation.output
    def output(self, windowed_value, output_index=0):
apache_beam/runners/worker/operations.py:568: in apache_beam.runners.worker.operations.Operation.output
    _cast_to_receiver(self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:259: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    def receive(self, windowed_value):
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:348: in apache_beam.runners.worker.operations.GeneralPurposeConsumerSet.receive
    _cast_to_operation(consumer).process(windowed_value)
apache_beam/runners/worker/operations.py:947: in apache_beam.runners.worker.operations.DoOperation.process
    def process(self, o):
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1585: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:683: in apache_beam.runners.common.SimpleInvoker.invoke_process
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1680: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    self._write_value_to_tag(tag, windowed_value, watermark_estimator)
apache_beam/runners/common.py:1793: in apache_beam.runners.common._OutputHandler._write_value_to_tag
    self.main_receivers.receive(windowed_value)
apache_beam/runners/worker/operations.py:262: in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive
    self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:949: in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
apache_beam/runners/worker/operations.py:950: in apache_beam.runners.worker.operations.DoOperation.process
    delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1497: in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1606: in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise new_exn
apache_beam/runners/common.py:1495: in apache_beam.runners.common.DoFnRunner.process
    return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:911: in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_process_per_window(
apache_beam/runners/common.py:1053: in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
    self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1670: in apache_beam.runners.common._OutputHandler.handle_process_outputs
    for result in results:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tensorflow_transform.beam.impl._RunMetaGraphDoFn object at 0x7c10d83a4df0>
batch = pyarrow.RecordBatch
CODED_SAMPLE: large_list<item: large_binary>
  child 0, item: large_binary
x: large_list<item: float>
  child 0, item: float
saved_model_dir = '/tmp/tmpuka2v5x0/a84299/tftransform_tmp/1f6a464fcca24232aba526751dadddeb'

    def process(self, batch, saved_model_dir):
      """Runs the given graph to realize the outputs.
    
      Runs the graph in a TF session for computing the output values of the
      `Tensor`s, `SparseTensor`s, or `RaggedTensor`s, given an input row of data
      (input `Tensor`s, `SparseTensor`s, or `RaggedTensor`s).
    
      Args:
        batch: the batch of elements being processed by the DoFn
        saved_model_dir: Directory containing saved model.
    
      Yields:
        A representation of output features as a dict mapping keys (logical column
        names) to values.
      """
      if not hasattr(self, '_graph_state'):
        # If available, acquire will return a cached _GraphStateCommon, since
        # calling _make_graph_state is expensive.
        self._graph_state = self._shared_graph_state_handle.acquire(
            lambda: self._make_graph_state(saved_model_dir))
    
      # This should remain true throughout the lifetime of this DoFn, regardless
      # of whether or not self._graph_state was cached.
>     assert self._graph_state.saved_model_dir == saved_model_dir
E     RuntimeError: AssertionError [while running 'MLTransform/TFTProcessHandler/AnalyzeDataset/ApplySavedModel[Phase0]/ApplySavedModel']

target/.tox-py310-ml/py310-ml/lib/python3.10/site-packages/tensorflow_transform/beam/impl.py:413: RuntimeError