Merge pull request #7 from eliorc/dev

Version update 0.4.1

Author: eliorc

.circleci/config.yml

@@ -28,14 +28,15 @@ jobs:
     steps:
       # Checkout repository
       - checkout
-      # Install dependencies, uses cache if not update
+      # Install dependencies, uses cache if not updat
       - restore_cache:
           key: test-deps-{{ .Branch }}-{{ checksum "requirements/test.txt" }}
       - run:
           name: install tests dependencies
           command: |
             python3 -m venv venv
             source venv/bin/activate
+            pip install -U pip
             pip install -r requirements/test.txt
             pip install -U pip setuptools
       - save_cache:

README.rst

@@ -9,7 +9,7 @@
 .. image:: https://img.shields.io/badge/python-3.5%20%7C%203.6%20%7C%203.7-blue.svg
     :alt: Supported Python versions
 
-.. image:: https://img.shields.io/badge/tensorflow-2.0.0--beta1-orange.svg
+.. image:: https://img.shields.io/badge/tensorflow-2.0.0--rc0-orange.svg
     :alt: Supported TensorFlow versions
 
 .. image:: https://codecov.io/gh/eliorc/tavolo/branch/master/graph/badge.svg
@@ -28,7 +28,7 @@ Tavolo
 | tavolo gathers implementations of these useful ideas from the community (by contribution, from `Kaggle`_ etc.)
   and makes them accessible in a single PyPI hosted package that compliments the `tf.keras`_ module.
 |
-| *Notice: tavolo is developed for TensorFlow 2.0 (right now on beta), most modules will work with earlier versions but some won't (like LayerNormalization)*
+| *Notice: tavolo is developed for TensorFlow 2.0 (right now on pre-release), most modules will work with earlier versions but some won't (like LayerNormalization)*
 
 Documentation
 -------------

docs/source/index.rst

@@ -11,7 +11,7 @@ Welcome to tavolo's documentation!
 
 .. warning::
 
-    tavolo is developed for TensorFlow 2.0 (right now on beta), most modules will work with earlier versions but some won't (like LayerNormalization)
+    tavolo is developed for TensorFlow 2.0 (right now on pre-release), most modules will work with earlier versions but some won't (like LayerNormalization)
 
 Showcase
 --------

docs/source/install.rst

@@ -1,7 +1,12 @@
 Installation
 ============
 
-| tavolo is hosted on PyPI, and the source code is available on Github
+| tavolo is hosted on PyPI, and the source code is available on Github.
+
+.. note::
+
+    Tavolo will not install tensorflow by itself, this is to prevent installations of CPU and GPU versions together.
+    It is the user's responsibility to install the tensorflow library
 
 Install from PyPI
 +++++++++++++++++

requirements/docs.txt

@@ -4,4 +4,5 @@ sphinxcontrib-httpdomain
 sphinx_rtd_theme
 pygments
 pygments-style-github
+tensorflow==2.0.0-rc0
 tavolo

requirements/test.txt

@@ -2,3 +2,4 @@ pytest
 pytest-cov
 twine
 codecov
+tensorflow==2.0.0-rc0

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup
 
-VERSION = '0.4.0'
+VERSION = '0.4.1'
 
 setup(name='tavolo',
       version=VERSION,
@@ -10,6 +10,5 @@
       classifiers=['License :: OSI Approved :: MIT License'],
       packages=['tavolo'],
       install_requires=[
-          'numpy',
-          'tensorflow==2.0.0-beta1'],
+          'numpy'],
       python_requires='>=3.5')

tavolo/__init__.py

@@ -1,5 +1,5 @@
 __name__ = 'tavolo'
-__version__ = '0.4.0'
+__version__ = '0.4.1'
 
 from . import embeddings
 from . import normalization

tavolo/seq2seq.py

@@ -96,12 +96,14 @@ def __init__(self,
         self.Q = None
         self.K = None
         self.V = None
+        self.output_projection = None
         self.dropout = tf.keras.layers.Dropout(rate=dropout_rate)
         self.very_small_value = (-2 ** 32 + 1)  # Used for padding to avoid attending
 
     def build(self, input_shape):
         # Units
-        self.n_units = self.n_units or input_shape[-1]
+        channels = input_shape[-1]
+        self.n_units = self.n_units or channels
 
         # Test units - n_heads validity
         if self.n_units % self.n_heads != 0:
@@ -110,19 +112,28 @@ def build(self, input_shape):
         # Linear projections
         self.Q = tf.keras.layers.Dense(units=self.n_units,
                                        activation=None,
+                                       use_bias=False,
                                        name='Q',
                                        dtype=self.dtype)
 
         self.K = tf.keras.layers.Dense(units=self.n_units,
                                        activation=None,
+                                       use_bias=False,
                                        name='K',
                                        dtype=self.dtype)
 
         self.V = tf.keras.layers.Dense(units=self.n_units,
                                        activation=None,
+                                       use_bias=False,
                                        name='V',
                                        dtype=self.dtype)
 
+        self.output_projection = tf.keras.layers.Dense(units=channels,
+                                                       activation=None,
+                                                       use_bias=False,
+                                                       name='output_projection',
+                                                       dtype=self.dtype)
+
         super().build(input_shape)
 
     def compute_mask(self, inputs, mask=None):
@@ -210,6 +221,9 @@ def call(self, inputs,
         outputs = tf.concat(tf.split(outputs, self.n_heads, axis=0),
                             axis=2)  # shape=(batch_size, time_steps, n_units)
 
+        # Project output
+        outputs = self.output_projection(outputs)  # shape=(batch_size, time_steps, channels)
+
         return outputs
 
     def get_config(self):

tests/seq2seq/multi_headed_self_attention_test.py

@@ -23,7 +23,7 @@ def test_shapes():
 
     # Assert correctness of output shapes
     assert output_single.shape == input_shape_3d
-    assert output_mh.shape == input_shape_3d[:-1] + (n_units_mh,)
+    assert output_mh.shape == input_shape_3d
 
 
 def test_masking():