Open
Description
System information.
== check python ===================================================
python version: 3.11.8
python branch: v3.11.8
python build version: ('v3.11.8:db85d51d3e', 'Feb 6 2024 18:02:37')
python compiler version: Clang 13.0.0 (clang-1300.0.29.30)
python implementation: CPython
== check os platform ===============================================
== are we in docker =============================================
No
== compiler =====================================================
Apple clang version 15.0.0 (clang-1500.1.0.2.5)
Target: arm64-apple-darwin23.4.0
Thread model: posix
InstalledDir: /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin
== check pips ===================================================
numpy 1.26.4
protobuf 4.25.3
tensorflow 2.16.1
tensorflow-io-gcs-filesystem 0.36.0
tensorflow-metal 1.1.0
== check for virtualenv =========================================
False
== tensorflow import ============================================
tf.version.VERSION = 2.16.1
tf.version.GIT_VERSION = v2.16.1-0-g5bc9d26649c
tf.version.COMPILER_VERSION = Apple LLVM 14.0.3 (clang-1403.0.22.14.1)
== env ==========================================================
LD_LIBRARY_PATH is unset
DYLD_LIBRARY_PATH is unset
== nvidia-smi ===================================================
tf_env_collect.sh: line 147: nvidia-smi: command not found
== cuda libs ===================================================
== tensorflow installed from info ==================
Name: tensorflow
Version: 2.16.1
Summary: TensorFlow is an open source machine learning framework for everyone.
Home-page: https://www.tensorflow.org/
Author-email: [email protected]
License: Apache 2.0
Location: /Users/marksusol/.virtualenvs/venv-tf16/lib/python3.11/site-packages
Required-by: tf_keras
== python version ==============================================
(major, minor, micro, releaselevel, serial)
(3, 11, 8, 'final', 0)
== bazel version ===============================================
Build label: 6.1.0
Build time: Mon Mar 6 17:10:27 2023 (1678122627)
Build timestamp: 1678122627
Build timestamp as int: 1678122627
Describe the problem/behavior.
fbeta_score = tf.keras.metrics.FBetaScore(average='micro', beta=5.0, threshold=0.6, name='fbeta_score')
model.compile(optimizer=optimizer, metrics=[fbeta_score])ValueError: FBetaScore expects 2D inputs with shape (batch_size, output_dim). Received input shapes: y_pred.shape=(16, 512, 12) and y_true.shape=(16, 512).
Describe the expected behavior.
Should be able to calculate an FBeta score and compile my model with this metric (discovered in Kaggle competition).
Seemed to work with TFA, similar issue as mine.
https://stackoverflow.com/questions/77236233/f1-score-in-tensorflow
Standalone code to reproduce the issue.
Provide a reproducible test case that is the bare minimum necessary to generate
the problem. If possible, please share a link to Colab/Jupyter/any notebook.
Source code / logs.
If including tracebacks, please include the full traceback.
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
File <timed exec>:111
File ~/.virtualenvs/venv-tf16/lib/python3.11/site-packages/transformers/modeling_tf_utils.py:1170, in TFPreTrainedModel.fit(self, *args, **kwargs)
1167 @functools.wraps(keras.Model.fit)
1168 def fit(self, *args, **kwargs):
1169 args, kwargs = convert_batch_encoding(*args, **kwargs)
-> 1170 return super().fit(*args, **kwargs)
File ~/.virtualenvs/venv-tf16/lib/python3.11/site-packages/tf_keras/src/utils/traceback_utils.py:70, in filter_traceback.<locals>.error_handler(*args, **kwargs)
67 filtered_tb = _process_traceback_frames(e.__traceback__)
68 # To get the full stack trace, call:
69 # `tf.debugging.disable_traceback_filtering()`
---> 70 raise e.with_traceback(filtered_tb) from None
71 finally:
72 del filtered_tb
File /var/folders/c3/gnpb2cfd5znckvplsjxhcy2r0000gn/T/__autograph_generated_filethuyzozh.py:15, in outer_factory.<locals>.inner_factory.<locals>.tf__train_function(iterator)
13 try:
14 do_return = True
---> 15 retval_ = ag__.converted_call(ag__.ld(step_function), (ag__.ld(self), ag__.ld(iterator)), None, fscope)
16 except:
17 do_return = False
File /var/folders/c3/gnpb2cfd5znckvplsjxhcy2r0000gn/T/__autograph_generated_filehvupbbbe.py:45, in outer_factory.<locals>.inner_factory.<locals>.tf__step_function(model, iterator)
43 ag__.if_stmt(ag__.ld(self).jit_compile, if_body, else_body, get_state, set_state, ('run_step',), 1)
44 data = ag__.converted_call(ag__.ld(next), (ag__.ld(iterator),), None, fscope)
---> 45 outputs = ag__.converted_call(ag__.ld(model).distribute_strategy.run, (ag__.ld(run_step),), dict(args=(ag__.ld(data),)), fscope)
46 outputs = ag__.converted_call(ag__.ld(reduce_per_replica), (ag__.ld(outputs), ag__.ld(self).distribute_strategy), dict(reduction=ag__.ld(self).distribute_reduction_method), fscope)
47 try:
File /var/folders/c3/gnpb2cfd5znckvplsjxhcy2r0000gn/T/__autograph_generated_filehvupbbbe.py:18, in outer_factory.<locals>.inner_factory.<locals>.tf__step_function.<locals>.run_step(data)
16 do_return_1 = False
17 retval__1 = ag__.UndefinedReturnValue()
---> 18 outputs = ag__.converted_call(ag__.ld(model).train_step, (ag__.ld(data),), None, fscope_1)
19 with ag__.ld(tf).control_dependencies(ag__.ld(_minimum_control_deps)(ag__.ld(outputs))):
20 ag__.converted_call(ag__.ld(model)._train_counter.assign_add, (1,), None, fscope_1)
File /var/folders/c3/gnpb2cfd5znckvplsjxhcy2r0000gn/T/__autograph_generated_fileqsocv9pw.py:411, in outer_factory.<locals>.inner_factory.<locals>.tf__train_step(self, data)
409 ag__.if_stmt(ag__.ld(loss) is None, if_body_23, else_body_23, get_state_24, set_state_24, ('loss',), 1)
410 ag__.converted_call(ag__.ld(self).optimizer.minimize, (ag__.ld(loss), ag__.ld(self).trainable_variables), dict(tape=ag__.ld(tape)), fscope)
--> 411 ag__.converted_call(ag__.ld(self).compiled_metrics.update_state, (ag__.ld(y), ag__.ld(y_pred), ag__.ld(sample_weight)), None, fscope)
412 return_metrics = {}
414 def get_state_26():
File /var/folders/c3/gnpb2cfd5znckvplsjxhcy2r0000gn/T/__autograph_generated_filewr15gxbw.py:163, in outer_factory.<locals>.inner_factory.<locals>.tf__update_state(self, y_true, y_pred, sample_weight)
161 continue__1 = ag__.Undefined('continue__1')
162 metric_objs = ag__.Undefined('metric_objs')
--> 163 ag__.for_stmt(ag__.converted_call(ag__.ld(zip), tuple(ag__.ld(zip_args)), None, fscope), None, loop_body_2, get_state_9, set_state_9, (), {'iterate_names': '(y_t, y_p, sw, metric_objs, weighted_metric_objs)'})
File /var/folders/c3/gnpb2cfd5znckvplsjxhcy2r0000gn/T/__autograph_generated_filewr15gxbw.py:151, in outer_factory.<locals>.inner_factory.<locals>.tf__update_state.<locals>.loop_body_2(itr_2)
149 nonlocal sw, y_p, y_t
150 pass
--> 151 ag__.if_stmt(ag__.not_(continue_), if_body_6, else_body_6, get_state_8, set_state_8, ('sw', 'y_p', 'y_t'), 0)
File /var/folders/c3/gnpb2cfd5znckvplsjxhcy2r0000gn/T/__autograph_generated_filewr15gxbw.py:106, in outer_factory.<locals>.inner_factory.<locals>.tf__update_state.<locals>.loop_body_2.<locals>.if_body_6()
104 pass
105 ag__.if_stmt(ag__.not_(continue__1), if_body_3, else_body_3, get_state_3, set_state_3, (), 0)
--> 106 ag__.for_stmt(ag__.ld(metric_objs), None, loop_body, get_state_4, set_state_4, (), {'iterate_names': 'metric_obj'})
108 def get_state_7():
109 return ()
File /var/folders/c3/gnpb2cfd5znckvplsjxhcy2r0000gn/T/__autograph_generated_filewr15gxbw.py:105, in outer_factory.<locals>.inner_factory.<locals>.tf__update_state.<locals>.loop_body_2.<locals>.if_body_6.<locals>.loop_body(itr)
103 def else_body_3():
104 pass
--> 105 ag__.if_stmt(ag__.not_(continue__1), if_body_3, else_body_3, get_state_3, set_state_3, (), 0)
File /var/folders/c3/gnpb2cfd5znckvplsjxhcy2r0000gn/T/__autograph_generated_filewr15gxbw.py:101, in outer_factory.<locals>.inner_factory.<locals>.tf__update_state.<locals>.loop_body_2.<locals>.if_body_6.<locals>.loop_body.<locals>.if_body_3()
100 def if_body_3():
--> 101 ag__.converted_call(ag__.ld(metric_obj).update_state, (ag__.ld(y_t), ag__.ld(y_p)), dict(sample_weight=ag__.ld(mask)), fscope)
File /var/folders/c3/gnpb2cfd5znckvplsjxhcy2r0000gn/T/__autograph_generated_fileg8quah8b.py:38, in outer_factory.<locals>.inner_factory.<locals>.tf__decorated(metric_obj, *args, **kwargs)
36 ag__.for_stmt(ag__.ld(metric_obj).weights, None, loop_body, get_state_1, set_state_1, (), {'iterate_names': 'weight'})
37 with ag__.ld(tf_utils).graph_context_for_symbolic_tensors(*ag__.ld(args), **ag__.ld(kwargs)):
---> 38 result = ag__.converted_call(ag__.ld(update_state_fn), tuple(ag__.ld(args)), dict(**ag__.ld(kwargs)), fscope)
40 def get_state_2():
41 return (result,)
File /var/folders/c3/gnpb2cfd5znckvplsjxhcy2r0000gn/T/__autograph_generated_fileuhto43hd.py:15, in outer_factory.<locals>.inner_factory.<locals>.tf__update_state_fn(*args, **kwargs)
13 try:
14 do_return = True
---> 15 retval_ = ag__.converted_call(ag__.ld(ag_update_state), tuple(ag__.ld(args)), dict(**ag__.ld(kwargs)), fscope)
16 except:
17 do_return = False
File /var/folders/c3/gnpb2cfd5znckvplsjxhcy2r0000gn/T/__autograph_generated_filezy_cvrig.py:37, in outer_factory.<locals>.inner_factory.<locals>.tf__update_state(self, y_true, y_pred, sample_weight)
35 y_pred, y_true = ag__.converted_call(ag__.ld(losses_utils).squeeze_or_expand_dimensions, (ag__.ld(y_pred), ag__.ld(y_true)), None, fscope)
36 ag_fn = ag__.converted_call(ag__.ld(tf).__internal__.autograph.tf_convert, (ag__.ld(self)._fn, ag__.converted_call(ag__.ld(tf).__internal__.autograph.control_status_ctx, (), None, fscope)), None, fscope)
---> 37 matches = ag__.converted_call(ag__.ld(ag_fn), (ag__.ld(y_true), ag__.ld(y_pred)), dict(**ag__.ld(self)._fn_kwargs), fscope)
38 mask = ag__.converted_call(ag__.ld(losses_utils).get_mask, (ag__.ld(matches),), None, fscope)
39 sample_weight = ag__.converted_call(ag__.ld(losses_utils).apply_valid_mask, (ag__.ld(matches), ag__.ld(sample_weight), ag__.ld(mask), ag__.ld(self).reduction), None, fscope)
File ~/.virtualenvs/venv-tf16/lib/python3.11/site-packages/keras/src/metrics/metric.py:212, in Metric.__call__(self, *args, **kwargs)
210 def __call__(self, *args, **kwargs):
211 self._check_super_called()
--> 212 self.update_state(*args, **kwargs)
213 return self.result()
File ~/.virtualenvs/venv-tf16/lib/python3.11/site-packages/keras/src/metrics/f_score_metrics.py:162, in FBetaScore.update_state(self, y_true, y_pred, sample_weight)
160 y_pred = ops.convert_to_tensor(y_pred, dtype=self.dtype)
161 if not self._built:
--> 162 self._build(y_true.shape, y_pred.shape)
164 if self.threshold is None:
165 threshold = ops.max(y_pred, axis=-1, keepdims=True)
File ~/.virtualenvs/venv-tf16/lib/python3.11/site-packages/keras/src/metrics/f_score_metrics.py:124, in FBetaScore._build(self, y_true_shape, y_pred_shape)
122 def _build(self, y_true_shape, y_pred_shape):
123 if len(y_pred_shape) != 2 or len(y_true_shape) != 2:
--> 124 raise ValueError(
125 "FBetaScore expects 2D inputs with shape "
126 "(batch_size, output_dim). Received input "
127 f"shapes: y_pred.shape={y_pred_shape} and "
128 f"y_true.shape={y_true_shape}."
129 )
130 if y_pred_shape[-1] is None or y_true_shape[-1] is None:
131 raise ValueError(
132 "FBetaScore expects 2D inputs with shape "
133 "(batch_size, output_dim), with output_dim fully "
(...)
136 f"y_true.shape={y_true_shape}."
137 )
ValueError: in user code:
File "/Users/marksusol/.virtualenvs/venv-tf16/lib/python3.11/site-packages/tf_keras/src/engine/training.py", line 1398, in train_function *
return step_function(self, iterator)
File "/Users/marksusol/.virtualenvs/venv-tf16/lib/python3.11/site-packages/tf_keras/src/engine/training.py", line 1370, in run_step *
outputs = model.train_step(data)
File "/Users/marksusol/.virtualenvs/venv-tf16/lib/python3.11/site-packages/transformers/modeling_tf_utils.py", line 1652, in train_step *
self.compiled_metrics.update_state(y, y_pred, sample_weight)
File "/Users/marksusol/.virtualenvs/venv-tf16/lib/python3.11/site-packages/tf_keras/src/engine/compile_utils.py", line 620, in update_state *
metric_obj.update_state(y_t, y_p, sample_weight=mask)
File "/Users/marksusol/.virtualenvs/venv-tf16/lib/python3.11/site-packages/tf_keras/src/metrics/base_metric.py", line 153, in decorated *
result = update_state_fn(*args, **kwargs)
File "/Users/marksusol/.virtualenvs/venv-tf16/lib/python3.11/site-packages/tf_keras/src/metrics/base_metric.py", line 140, in update_state_fn *
return ag_update_state(*args, **kwargs)
File "/Users/marksusol/.virtualenvs/venv-tf16/lib/python3.11/site-packages/tf_keras/src/metrics/base_metric.py", line 722, in update_state *
matches = ag_fn(y_true, y_pred, **self._fn_kwargs)
File "/Users/marksusol/.virtualenvs/venv-tf16/lib/python3.11/site-packages/keras/src/metrics/metric.py", line 212, in __call__ **
self.update_state(*args, **kwargs)
File "/Users/marksusol/.virtualenvs/venv-tf16/lib/python3.11/site-packages/keras/src/metrics/f_score_metrics.py", line 162, in update_state
self._build(y_true.shape, y_pred.shape)
File "/Users/marksusol/.virtualenvs/venv-tf16/lib/python3.11/site-packages/keras/src/metrics/f_score_metrics.py", line 124, in _build
raise ValueError(
ValueError: FBetaScore expects 2D inputs with shape (batch_size, output_dim). Received input shapes: y_pred.shape=(16, 512, 12) and y_true.shape=(16, 512).