From 0c0bf3b8d11a715fd9f71e8d703e52f4653ab247 Mon Sep 17 00:00:00 2001 From: Alex Olivas Date: Mon, 13 May 2024 08:55:48 -0600 Subject: [PATCH 1/2] Fix transformer.ipynb In the latest version of tensorflow, this example fails, because vocab_size is a scalar tensor (type EagerTensor) and keras can't determine the dtype. The failure occurs pretty deep in the call stack, so it's best to determine as early as possible whether that type conversion is going to succeed or not. Python3.10 TensorFlow 2.16.1 Attached is the relevant part of the traceback: File "/home/olivas/qaintum.ai/./transformer.py", line 404, in call x = self.embedding(x) File "/home/olivas/qaintum.ai/venv/lib/python3.10/site-packages/keras/src/utils/traceback_utils.py", line 113, in error_handler return fn(*args, **kwargs) File "/home/olivas/qaintum.ai/venv/lib/python3.10/site-packages/keras/src/layers/layer.py", line 771, in __call__ self._maybe_build(call_spec) File "/home/olivas/qaintum.ai/venv/lib/python3.10/site-packages/keras/src/layers/layer.py", line 1279, in _maybe_build self.build(**shapes_dict) File "/home/olivas/qaintum.ai/venv/lib/python3.10/site-packages/keras/src/layers/layer.py", line 223, in build_wrapper original_build_method(*args, **kwargs) File "/home/olivas/qaintum.ai/venv/lib/python3.10/site-packages/keras/src/layers/core/embedding.py", line 124, in build self._embeddings = self.add_weight( File "/home/olivas/qaintum.ai/venv/lib/python3.10/site-packages/keras/src/layers/layer.py", line 511, in add_weight variable = backend.Variable( File "/home/olivas/qaintum.ai/venv/lib/python3.10/site-packages/keras/src/backend/common/variables.py", line 161, in __init__ shape = self._validate_shape(shape) File "/home/olivas/qaintum.ai/venv/lib/python3.10/site-packages/keras/src/backend/common/variables.py", line 184, in _validate_shape shape = standardize_shape(shape) File "/home/olivas/qaintum.ai/venv/lib/python3.10/site-packages/keras/src/backend/common/variables.py", line 549, in standardize_shape if not is_int_dtype(type(e)): File "/home/olivas/qaintum.ai/venv/lib/python3.10/site-packages/keras/src/backend/common/variables.py", line 580, in is_int_dtype dtype = standardize_dtype(dtype) File "/home/olivas/qaintum.ai/venv/lib/python3.10/site-packages/keras/src/backend/common/variables.py", line 521, in standardize_dtype raise ValueError(f"Invalid dtype: {dtype}") ValueError: Invalid dtype: Exception ignored in: --- docs/tutorials/transformer.ipynb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/tutorials/transformer.ipynb b/docs/tutorials/transformer.ipynb index 8b1683d55..4458767c6 100644 --- a/docs/tutorials/transformer.ipynb +++ b/docs/tutorials/transformer.ipynb @@ -975,6 +975,16 @@ " def __init__(self, vocab_size, d_model):\n", " super().__init__()\n", " self.d_model = d_model\n", + " try:\n", + " input_dim = int(vocab_size)\n", + " except: ValueError:\n", + " if hasattr(input_dim, \"numpy\"):\n", + " try:\n", + " input_dim = int(vocab_size.numpy())\n", + " except ValueError:\n", + " print(f\"Can't convert vocab_size (type = {type(vocab_size)} to int.\")\n", + " \n", + " self.embedding = tf.keras.layers.Embedding(input_dim=input_dim, output_dim=d_model, mask_zero=True) \n", " self.embedding = tf.keras.layers.Embedding(vocab_size, d_model, mask_zero=True) \n", " self.pos_encoding = positional_encoding(length=2048, depth=d_model)\n", "\n", From cba36cd5b5ea863d711be94a577c1723ecc80a67 Mon Sep 17 00:00:00 2001 From: Alex Olivas Date: Mon, 13 May 2024 13:08:21 -0600 Subject: [PATCH 2/2] Update transformer.ipynb hasattr doesn't detect the numpy method. --- docs/tutorials/transformer.ipynb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/tutorials/transformer.ipynb b/docs/tutorials/transformer.ipynb index 4458767c6..7061ecd22 100644 --- a/docs/tutorials/transformer.ipynb +++ b/docs/tutorials/transformer.ipynb @@ -978,11 +978,10 @@ " try:\n", " input_dim = int(vocab_size)\n", " except: ValueError:\n", - " if hasattr(input_dim, \"numpy\"):\n", - " try:\n", - " input_dim = int(vocab_size.numpy())\n", - " except ValueError:\n", - " print(f\"Can't convert vocab_size (type = {type(vocab_size)} to int.\")\n", + " try: + " input_dim = int(vocab_size.numpy())\n", + " except: ValueError:\n", + " print(f\"Can't convert vocab_size (type = {type(vocab_size)} to int.\")\n", " \n", " self.embedding = tf.keras.layers.Embedding(input_dim=input_dim, output_dim=d_model, mask_zero=True) \n", " self.embedding = tf.keras.layers.Embedding(vocab_size, d_model, mask_zero=True) \n",