From 3736b0b78a84e22bf6989a6c8f2638d32324a278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aylin=20Ayd=C4=B1n?= Date: Wed, 21 Sep 2022 23:31:06 +0300 Subject: [PATCH 01/25] ArcFace Loss Implementation --- .../losses/ArcFace Loss Sample Notebook.ipynb | 775 ++++++++++++++++++ tensorflow_similarity/losses/__init__.py | 13 +- tensorflow_similarity/losses/arcface_loss.py | 119 +++ tensorflow_similarity/losses/test_losses.py | 275 +++++++ 4 files changed, 1176 insertions(+), 6 deletions(-) create mode 100644 tensorflow_similarity/losses/ArcFace Loss Sample Notebook.ipynb create mode 100644 tensorflow_similarity/losses/arcface_loss.py create mode 100644 tensorflow_similarity/losses/test_losses.py diff --git a/tensorflow_similarity/losses/ArcFace Loss Sample Notebook.ipynb b/tensorflow_similarity/losses/ArcFace Loss Sample Notebook.ipynb new file mode 100644 index 00000000..b481492e --- /dev/null +++ b/tensorflow_similarity/losses/ArcFace Loss Sample Notebook.ipynb @@ -0,0 +1,775 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "28956aa1", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "Copyright 2022 The TensorFlow Similarity Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24eda1a6", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# @title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "id": "7ca9d025", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "# TensorFlow Similarity ArcFace Loss Example" + ] + }, + { + "cell_type": "markdown", + "id": "d072628f", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "A Total Angular Margin Loss (ArcFace) calculates the geodetic distance in the hypersphere instead of the euclidean distance to improve the discriminatory strength of the facial recognition model and stabilize the training process. Rails are used to measure all distances in geodetic space. The geodetic trace is the path taken between two places. It specifies the geodetic distance, which is the shortest distance between two places.\n", + "\n", + "ArcFace loss determines the angle between the current feature and the target weight using the arc-cosine function since the dot product between the DCNN feature and the last fully connected layer after feature and weight normalization matches the cosine distance. The target logit is then returned by multiplying the goal angle by an additional angular margin and using the cosine function. After that, we continue as before and rescale all logits to a certain feature norm, just like with softmax loss." + ] + }, + { + "cell_type": "markdown", + "id": "808ac087", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Notebook goal\n", + "\n", + "This notebook demonstrates how to use ArcFaceLoss implementation of TensorFlow Similarity with standalone usage and to train a `SimilarityModel()` on a fraction of the MNIST classes.\n", + "\n", + "You are going to learn about the main features offered by the `ArcFaceLoss()` and will:\n", + "\n", + " 1. Standalone usage of ArcFaceLoss\n", + "\n", + " 2. Usage with `model.compile()`\n", + "\n", + " 3. 3D-Visualization of ArcFaceLoss \n", + "\n", + "### Things to try \n", + "\n", + "Along the way you can try the following things to improve the model performance:\n", + "- Adding more \"seen\" classes at training time.\n", + "- Use a larger embedding by increasing the size of the output.\n", + "- Add data augmentation pre-processing layers to the model.\n", + "- Include more examples in the index to give the models more points to choose from.\n", + "- Try a more challenging dataset, such as Fashion MNIST." + ] + }, + { + "cell_type": "markdown", + "id": "078c53c0", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Notebook goal\n", + "\n", + "This notebook demonstrates how to use ArcFaceLoss implementation of TensorFlow Similarity with standalone usage and to train a `SimilarityModel()` on a fraction of the MNIST classes.\n", + "\n", + "You are going to learn about the main features offered by the `ArcFaceLoss()` and will:\n", + "\n", + " 1. Standalone usage of ArcFaceLoss\n", + "\n", + " 2. Usage with `model.compile()`\n", + "\n", + " 3. 3D-Visualization of ArcFaceLoss \n", + "\n", + "### Things to try \n", + "\n", + "Along the way you can try the following things to improve the model performance:\n", + "- Adding more \"seen\" classes at training time.\n", + "- Use a larger embedding by increasing the size of the output.\n", + "- Add data augmentation pre-processing layers to the model.\n", + "- Include more examples in the index to give the models more points to choose from.\n", + "- Try a more challenging dataset, such as Fashion MNIST." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8fd63f16", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import gc\n", + "import os\n", + "\n", + "import numpy as np\n", + "from matplotlib import pyplot as plt\n", + "from tabulate import tabulate\n", + "from mpl_toolkits.mplot3d import Axes3D\n", + "\n", + "# INFO messages are not printed.\n", + "# This must be run before loading other modules.\n", + "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"1\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80af5fc0", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import tensorflow as tf" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ba8caf7", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# install TF similarity if needed\n", + "try:\n", + " import tensorflow_similarity as tfsim # main package\n", + "except ModuleNotFoundError:\n", + " !pip install tensorflow_similarity\n", + " import tensorflow_similarity as tfsim" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2484bd72", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "tfsim.utils.tf_cap_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3fe0344e", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Clear out any old model state.\n", + "gc.collect()\n", + "tf.keras.backend.clear_session()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "99d9bef9", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "print(\"TensorFlow:\", tf.__version__)\n", + "print(\"TensorFlow Similarity\", tfsim.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7137afbc", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "1d534ad3", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "# Standalone Usage of ArcFaceLoss\n", + "\n", + "ArcFace loss alone can be used as follows when it is desired to calculate the additive angular margin loss of the existing data set." + ] + }, + { + "cell_type": "markdown", + "id": "68d526da", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Initialize Loss function as ArcFaceLoss" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bebf6ef0", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "loss_fn = tfsim.losses.ArcFaceLoss(num_classes=8, embedding_size=10)" + ] + }, + { + "cell_type": "markdown", + "id": "d2ccfd7d", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Create own simple random dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1d1ec43a", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "labels = tf.Variable([0, 1, 2, 3, 4, 5, 6, 7])\n", + "embeddings = tf.Variable(tf.random.uniform(shape=[8, 10]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "73d0c1c6", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "print(\"\", embeddings)" + ] + }, + { + "cell_type": "markdown", + "id": "d65b3085", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Calculate loss" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cdf7c30c", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "loss = loss_fn(labels, embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16745b7d", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "print(\"loss : \" , loss)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "loss = loss_fn(labels, embeddings)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "print(\"loss : \" , loss)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Data preparation\n", + "\n", + "We are going to load the MNIST dataset to showcase how the model is able to find similar examples from classes unseen during training. The model's ability to generalize the matching to unseen classes, without retraining, is one of the main reason you would want to use metric learning.\n", + "\n", + "\n", + "**WARNING**: Tensorflow similarity expects `y_train` to be an IntTensor containing the class ids for each example instead of the standard categorical encoding traditionally used for multi-class classification." + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8a9f8122", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"loss : \" , loss)" + ] + }, + { + "cell_type": "markdown", + "id": "11ef5236", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Data preparation\n", + "\n", + "We are going to load the MNIST dataset to showcase how the model is able to find similar examples from classes unseen during training. The model's ability to generalize the matching to unseen classes, without retraining, is one of the main reason you would want to use metric learning.\n", + "\n", + "\n", + "**WARNING**: Tensorflow similarity expects `y_train` to be an IntTensor containing the class ids for each example instead of the standard categorical encoding traditionally used for multi-class classification." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "97152229", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()" + ] + }, + { + "cell_type": "markdown", + "id": "08b766d8", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Model setup" + ] + }, + { + "cell_type": "markdown", + "id": "3eac2da7", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Model definition\n", + "\n", + "`SimilarityModel()` models extend `tensorflow.keras.model.Model` with additional features and functionality that allow you to index and search for similar looking examples.\n", + "\n", + "As visible in the model definition below, similarity models output a 64 dimensional float embedding using the `MetricEmbedding()` layers. This layer is a Dense layer with L2 normalization. Thanks to the loss, the model learns to minimize the distance between similar examples and maximize the distance between dissimilar examples. As a result, the distance between examples in the embedding space is meaningful; the smaller the distance the more similar the examples are. \n", + "\n", + "Being able to use a distance as a meaningful proxy for how similar two examples are, is what enables the fast ANN (aproximate nearest neighbor) search. Using a sub-linear ANN search instead of a standard quadratic NN search is what allows deep similarity search to scale to millions of items. The built in memory index used in this notebook scales to a million indexed examples very easily... if you have enough RAM :)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a003c971", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "def get_model():\n", + " inputs = tf.keras.layers.Input(shape=(28, 28, 1))\n", + " x = tf.keras.layers.experimental.preprocessing.Rescaling(1 / 255)(inputs)\n", + " x = tf.keras.layers.Conv2D(32, 3, activation=\"relu\")(x)\n", + " x = tf.keras.layers.Conv2D(32, 3, activation=\"relu\")(x)\n", + " x = tf.keras.layers.MaxPool2D()(x)\n", + " x = tf.keras.layers.Conv2D(64, 3, activation=\"relu\")(x)\n", + " x = tf.keras.layers.Conv2D(64, 3, activation=\"relu\")(x)\n", + " x = tf.keras.layers.Flatten()(x)\n", + " # smaller embeddings will have faster lookup times while a larger embedding will improve the accuracy up to a point.\n", + " outputs = tfsim.layers.MetricEmbedding(64)(x)\n", + " return tfsim.models.SimilarityModel(inputs, outputs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2177b12", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "model = get_model()\n", + "model.summary()" + ] + }, + { + "cell_type": "markdown", + "id": "defb3961", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### ArcFace Loss definition\n", + "\n", + "Overall what makes Metric losses different from tradional losses is that:\n", + "- **They expect different inputs.** Instead of having the prediction equal the true values, they expect embeddings as `y_preds` and the id (as an int32) of the class as `y_true`. \n", + "- **They require a distance.** You need to specify which `distance` function to use to compute the distance between embeddings. `cosine` is usually a great starting point and the default.\n", + "\n", + "ArcFace Loss takes inputs as number of classes which labels includes, and embedding size which we define in model `MetricEmbedding()` layers." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13b0d745", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "distance = \"cosine\" " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c22d10cc", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "num_classes = np.unique(y_train).size\n", + "embedding_size = model.get_layer('metric_embedding').output.shape[1]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5b8e426", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "loss = tfsim.losses.ArcFaceLoss(num_classes= num_classes, embedding_size=embedding_size, name=\"ArcFaceLoss\")" + ] + }, + { + "cell_type": "markdown", + "id": "b6eaf9c8", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Compilation\n", + "\n", + "Tensorflow similarity use an extended `compile()` method that allows you to optionally specify `distance_metrics` (metrics that are computed over the distance between the embeddings), and the distance to use for the indexer.\n", + "\n", + "By default the `compile()` method tries to infer what type of distance you are using by looking at the first loss specified. If you use multiple losses, and the distance loss is not the first one, then you need to specify the distance function used as `distance=` parameter in the compile function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "673f986f", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "LR = 0.0005 # @param {type:\"number\"}\n", + "model.compile(optimizer=tf.keras.optimizers.SGD(LR), loss=loss, distance=distance)" + ] + }, + { + "cell_type": "markdown", + "id": "15961601", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Training\n", + "\n", + "Similarity models are trained like normal models. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "147a6863", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "EPOCHS = 10 # @param {type:\"integer\"}\n", + "history = model.fit(x_train, y_train, epochs=EPOCHS, validation_data=(x_test, y_test))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "88e1ee4d", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plt.plot(history.history[\"loss\"])\n", + "plt.plot(history.history[\"val_loss\"])\n", + "plt.legend([\"loss\", \"val_loss\"])\n", + "plt.title(f\"Loss: {loss.name}\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5404906", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "5ad4ba20", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Prediction\n", + "\n", + "Let's predict some features and visualiza them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1936264", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "embedded_features = model.predict(x_test, verbose=1)\n", + "embedded_features /= np.linalg.norm(embedded_features, axis=1, keepdims=True)" + ] + }, + { + "cell_type": "markdown", + "id": "7c0df63b", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### 3D-Visualization of ArcFace Loss" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5aac5d98", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "fig = plt.figure()\n", + "ax = Axes3D(fig)\n", + "for c in range(len(np.unique(y_test))):\n", + " ax.plot(embedded_features[y_test==c, 0], embedded_features[y_test==c, 1], embedded_features[y_test==c, 2], '.', alpha=0.1)\n", + "plt.title('ArcFace')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8889f840", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fef529d9", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/tensorflow_similarity/losses/__init__.py b/tensorflow_similarity/losses/__init__.py index 46c7c700..a51937b3 100644 --- a/tensorflow_similarity/losses/__init__.py +++ b/tensorflow_similarity/losses/__init__.py @@ -15,13 +15,14 @@ """ Contrastive learning specialized losses. """ -from .barlow import Barlow # noqa -from .circle_loss import CircleLoss # noqa +from .pn_loss import PNLoss # noqa +from .triplet_loss import TripletLoss # noqa from .metric_loss import MetricLoss # noqa +from .circle_loss import CircleLoss # noqa from .multisim_loss import MultiSimilarityLoss # noqa -from .pn_loss import PNLoss # noqa -from .simclr import SimCLRLoss # noqa from .simsiam import SimSiamLoss # noqa -from .softnn_loss import SoftNearestNeighborLoss # noqa -from .triplet_loss import TripletLoss # noqa +from .simclr import SimCLRLoss # noqa from .vicreg import VicReg # noqa +from .barlow import Barlow # noqa +from .softnn_loss import SoftNearestNeighborLoss # noqa +from .arcface_loss import ArcFaceLoss # noqa diff --git a/tensorflow_similarity/losses/arcface_loss.py b/tensorflow_similarity/losses/arcface_loss.py new file mode 100644 index 00000000..e0252d5c --- /dev/null +++ b/tensorflow_similarity/losses/arcface_loss.py @@ -0,0 +1,119 @@ +# Copyright 2022 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""ArcFace losses base class. + +ArcFace: Additive Angular Margin Loss for Deep Face +Recognition. [online] arXiv.org. Available at: +. + +""" + +from typing import Any, Callable, Dict, Optional, Tuple, Union + +import tensorflow as tf +from tensorflow_similarity.algebra import build_masks +from tensorflow_similarity.distances import Distance, distance_canonicalizer +from tensorflow_similarity.types import FloatTensor, IntTensor +from tensorflow_similarity.utils import is_tensor_or_variable + +from .metric_loss import MetricLoss +from .utils import logsumexp + + +@tf.keras.utils.register_keras_serializable(package="Similarity") +class ArcFaceLoss(tf.keras.losses.Loss): + """Implement of ArcFace: Additive Angular Margin Loss: + Step 1: Create a trainable kernel matrix with the shape of [embedding_size, num_classes]. + Step 2: Normalize the kernel and prediction vectors. + Step 3: Calculate the cosine similarity between the normalized prediction vector and the kernel. + Step 4: Create a one-hot vector include the margin value for the ground truth class. + Step 5: Add margin_hot to the cosine similarity and multiply it by scale. + Step 6: Calculate the cross-entropy loss. + + ArcFace: Additive Angular Margin Loss for Deep Face + Recognition. [online] arXiv.org. Available at: + . + + Standalone usage: + >>> loss_fn = tfsim.losses.ArcFaceLoss(num_classes=2, embedding_size=3) + >>> labels = tf.Variable([1, 0]) + >>> embeddings = tf.Variable([[0.2, 0.3, 0.1], [0.4, 0.5, 0.5]]) + >>> loss = loss_fn(labels, embeddings) + Args: + num_classes: Number of classes. + embedding_size: The size of the embedding vectors. + margin: The margin value. + scale: s in the paper, feature scale + name: Optional name for the operation. + reduction: Type of loss reduction to apply to the loss. + """ + + def __init__( + self, + num_classes: int, + embedding_size: int, + margin: float = 0.50, # margin in radians + scale: float = 64.0, # feature scale + name: Optional[str] = None, + reduction: Callable = tf.keras.losses.Reduction.AUTO, + **kwargs + ): + + super().__init__(reduction=reduction, name=name, **kwargs) + + self.num_classes = num_classes + self.embedding_size = embedding_size + self.margin = margin + self.scale = scale + self.name = name + self.kernel = tf.Variable(tf.random.normal([embedding_size, num_classes])) + + def call(self, y_true: FloatTensor, y_pred: FloatTensor) -> FloatTensor: + + y_pred_norm = tf.math.l2_normalize(y_pred, axis=1) + kernel_norm = tf.math.l2_normalize(self.kernel, axis=0) + + cos_theta = tf.matmul(y_pred_norm, kernel_norm) + cos_theta = tf.clip_by_value(cos_theta, -1.0, 1.0) + + m_hot = tf.one_hot(y_true, self.num_classes, on_value=self.margin, axis=1) + m_hot = tf.reshape(m_hot, [-1, self.num_classes]) + + cos_theta = tf.acos(cos_theta) + cos_theta += m_hot + cos_theta = tf.math.cos(cos_theta) + cos_theta = tf.math.multiply(cos_theta, self.scale) + + cce = tf.keras.losses.SparseCategoricalCrossentropy( + from_logits=True, reduction=self.reduction + ) + loss: FloatTensor = cce(y_true, cos_theta) + + return loss + + def get_config(self) -> Dict[str, Any]: + """Contains the loss configuration. + Returns: + A Python dict containing the configuration of the loss. + """ + config = { + "num_classes": self.num_classes, + "embedding_size": self.embedding_size, + "margin": self.margin, + "scale": self.scale, + "name": self.name, + } + base_config = super().get_config() + return {**base_config, **config} diff --git a/tensorflow_similarity/losses/test_losses.py b/tensorflow_similarity/losses/test_losses.py new file mode 100644 index 00000000..1d4fef9c --- /dev/null +++ b/tensorflow_similarity/losses/test_losses.py @@ -0,0 +1,275 @@ +import tensorflow as tf +import numpy as np +from tensorflow_similarity.losses import TripletLoss, MultiSimilarityLoss +from tensorflow_similarity.losses import PNLoss +from tensorflow_similarity.losses import SoftNearestNeighborLoss +from tensorflow_similarity.losses import ArcFaceLoss + +# [triplet loss] +from tensorflow_similarity.losses.xbm_loss import XBM + + +def test_triplet_loss_serialization(): + loss = TripletLoss() + config = loss.get_config() + print(config) + loss2 = TripletLoss.from_config(config) + assert loss.name == loss2.name + assert loss.distance == loss2.distance + + +def triplet_hard_loss_np(labels, embedding, margin, dist_func, soft=False): + num_data = embedding.shape[0] + # Reshape labels to compute adjacency matrix. + labels_reshaped = np.reshape(labels.astype(np.float32), + (labels.shape[0], 1)) + + adjacency = np.equal(labels_reshaped, labels_reshaped.T) + pdist_matrix = dist_func(embedding) + loss_np = 0.0 + for i in range(num_data): + pos_distances = [] + neg_distances = [] + for j in range(num_data): + if adjacency[i][j] == 0: + neg_distances.append(pdist_matrix[i][j]) + if adjacency[i][j] > 0.0 and i != j: + pos_distances.append(pdist_matrix[i][j]) + + # if there are no positive pairs, distance is 0 + if len(pos_distances) == 0: + pos_distances.append(0) + + # Sort by distance. + neg_distances.sort() + min_neg_distance = neg_distances[0] + pos_distances.sort(reverse=True) + max_pos_distance = pos_distances[0] + + if soft: + loss_np += np.log1p(np.exp(max_pos_distance - min_neg_distance)) + else: + loss_np += np.maximum(0.0, + max_pos_distance - min_neg_distance + margin) + + loss_np /= num_data + return loss_np + + +def test_triplet_loss(): + num_inputs = 10 + # y_true: labels + y_true = tf.random.uniform((num_inputs,), 0, 10, dtype=tf.int32) + # y_preds: embedding + y_preds = tf.random.uniform((num_inputs, 20), 0, 1) + tpl = TripletLoss() + # y_true, y_preds + loss = tpl(y_true, y_preds) + assert loss > 0.9 + + +def test_triplet_loss_easy(): + num_inputs = 10 + # y_true: labels + y_true = tf.random.uniform((num_inputs,), 0, 3, dtype=tf.int32) + # y_preds: embedding + y_preds = tf.random.uniform((num_inputs, 16), 0, 1) + tpl = TripletLoss(positive_mining_strategy='easy', + negative_mining_strategy='easy') + # y_true, y_preds + loss = tpl(y_true, y_preds) + assert loss > 0 + + +def test_triplet_loss_semi_hard(): + num_inputs = 10 + # y_true: labels + y_true = tf.random.uniform((num_inputs,), 0, 3, dtype=tf.int32) + # y_preds: embedding + y_preds = tf.random.uniform((num_inputs, 16), 0, 1) + tpl = TripletLoss(positive_mining_strategy='easy', + negative_mining_strategy='semi-hard') + # y_true, y_preds + loss = tpl(y_true, y_preds) + assert loss + + +def test_triplet_loss_hard(): + num_inputs = 10 + # y_true: labels + y_true = tf.random.uniform((num_inputs,), 0, 3, dtype=tf.int32) + # y_preds: embedding + y_preds = tf.random.uniform((num_inputs, 16), 0, 1) + tpl = TripletLoss(positive_mining_strategy='hard', + negative_mining_strategy='hard') + # y_true, y_preds + loss = tpl(y_true, y_preds) + assert loss + + +# [pn loss] +def test_pn_loss_serialization(): + loss = PNLoss() + config = loss.get_config() + print(config) + loss2 = PNLoss.from_config(config) + assert loss.name == loss2.name + assert loss.distance == loss2.distance + + +def test_np_loss(): + num_inputs = 10 + # y_true: labels + y_true = tf.random.uniform((num_inputs,), 0, 10, dtype=tf.int32) + # y_preds: embedding + y_preds = tf.random.uniform((num_inputs, 20), 0, 1) + pnl = PNLoss() + # y_true, y_preds + loss = pnl(y_true, y_preds) + assert loss > 0.9 + + +# [soft neasrest neighbor loss] +def test_softnn_loss_serialization(): + loss = SoftNearestNeighborLoss(distance="cosine", temperature=50) + config = loss.get_config() + loss2 = SoftNearestNeighborLoss.from_config(config) + assert loss.name == loss2.name + assert loss.distance == loss2.distance + assert loss.temperature == loss2.temperature + + +def softnn_util(y_true, x, temperature=1): + """ + A simple loop based implementation of soft + nearest neighbor loss to test the code. + https://arxiv.org/pdf/1902.01889.pdf + """ + + y_true = y_true.numpy() + x = x.numpy() + batch_size = y_true.shape[0] + loss = 0 + eps = 1e-9 + for i in range(batch_size): + numerator = 0 + denominator = 0 + for j in range(batch_size): + if i == j: continue + if y_true[i] == y_true[j]: + numerator += np.exp(-1 * + np.sum(np.square(x[i] - x[j])) / temperature) + denominator += np.exp(-1 * + np.sum(np.square(x[i] - x[j])) / temperature) + if numerator == 0: continue + loss += np.log(numerator / denominator) + return -loss / batch_size + + +def test_softnn_loss(): + num_inputs = 10 + n_classes = 10 + # y_true: labels + y_true = tf.random.uniform((num_inputs,), 0, n_classes, dtype=tf.int32) + # x: embeddings + x = tf.random.uniform((num_inputs, 20), 0, 1) + + temperature = np.random.uniform(0.1, 50) + softnn = SoftNearestNeighborLoss(temperature=temperature) + loss = softnn(y_true, x) + loss_check = softnn_util(y_true, x, temperature) + loss_diff = loss.numpy() - loss_check + assert np.abs(loss_diff) < 1e-3 + + +def test_xbm_loss(): + batch_size = 6 + embed_dim = 16 + + embeddings1 = tf.random.uniform(shape=[batch_size, embed_dim]) + labels1 = tf.constant( + [ + [1], + [1], + [2], + [2], + [3], + [3], + ], + dtype=tf.int32 + ) + + embeddings2 = tf.random.uniform(shape=[batch_size, embed_dim]) + labels2 = tf.constant( + [ + [4], + [4], + [5], + [5], + [6], + [6], + ], + dtype=tf.int32 + ) + + distance = "cosine" + loss = MultiSimilarityLoss(distance=distance) + loss_nowarm = XBM(loss, memory_size=12, warmup_steps=0) + + # test enqueue + loss_nowarm(labels1, embeddings1) + assert loss_nowarm._y_pred_memory.numpy().shape == (batch_size, embed_dim) + tf.assert_equal(loss_nowarm._y_true_memory, labels1) + + loss_nowarm(labels2, embeddings2) + assert loss_nowarm._y_pred_memory.numpy().shape == (2 * batch_size, embed_dim) + tf.assert_equal(loss_nowarm._y_true_memory, tf.concat([labels2, labels1], axis=0)) + + # test dequeue + loss_nowarm(labels2, embeddings2) + assert loss_nowarm._y_pred_memory.numpy().shape == (2 * batch_size, embed_dim) + tf.assert_equal(loss_nowarm._y_true_memory, tf.concat([labels2, labels2], axis=0)) + + # test warmup + loss_warm = XBM(loss, memory_size=12, warmup_steps=1) + + loss_warm(labels1, embeddings1) + assert loss_warm._y_pred_memory.numpy().shape == (0, embed_dim) + tf.assert_equal(loss_warm._y_true_memory, tf.constant([[]], dtype=tf.int32)) + + loss_warm(labels2, embeddings2) + assert loss_warm._y_pred_memory.numpy().shape == (batch_size, embed_dim) + tf.assert_equal(loss_warm._y_true_memory, labels2) + + +# arcface loss +""" +ArcFaceLoss + ArcFace: Additive Angular Margin Loss for Deep Face + Recognition. [online] arXiv.org. Available at: + . +""" + + +def test_arcface_loss_serialization(): + n_classes = 10 + embed_size = 16 + loss = ArcFaceLoss(num_classes=n_classes, embedding_size=embed_size) + config = loss.get_config() + loss2 = ArcFaceLoss.from_config(config) + assert loss.name == loss2.name + assert loss.margin == loss2.margin + assert loss.scale == loss2.scale + assert loss.num_classes == loss2.num_classes + assert loss.embedding_size == loss2.embedding_size + + + +def test_arcface_loss(): + tf.random.set_seed(128) + loss_fn = ArcFaceLoss(num_classes=4, embedding_size=5) + labels = tf.Variable([0, 1, 2, 3]) + embeddings = tf.Variable(tf.random.uniform(shape=[4, 5])) + loss = loss_fn(labels, embeddings) + + assert 60.4 < loss.numpy() < 60.5 From f49676fb649c1caccb0c15fa27415bce9c071293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aylin=20Ayd=C4=B1n?= Date: Wed, 21 Sep 2022 23:37:32 +0300 Subject: [PATCH 02/25] Add files via upload --- ArcFace Loss Sample Notebook.ipynb | 775 +++++++++++++++++++++++++++++ arcface_loss.py | 120 +++++ test_losses.py | 275 ++++++++++ 3 files changed, 1170 insertions(+) create mode 100644 ArcFace Loss Sample Notebook.ipynb create mode 100644 arcface_loss.py create mode 100644 test_losses.py diff --git a/ArcFace Loss Sample Notebook.ipynb b/ArcFace Loss Sample Notebook.ipynb new file mode 100644 index 00000000..b481492e --- /dev/null +++ b/ArcFace Loss Sample Notebook.ipynb @@ -0,0 +1,775 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "28956aa1", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "Copyright 2022 The TensorFlow Similarity Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24eda1a6", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# @title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "id": "7ca9d025", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "# TensorFlow Similarity ArcFace Loss Example" + ] + }, + { + "cell_type": "markdown", + "id": "d072628f", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "A Total Angular Margin Loss (ArcFace) calculates the geodetic distance in the hypersphere instead of the euclidean distance to improve the discriminatory strength of the facial recognition model and stabilize the training process. Rails are used to measure all distances in geodetic space. The geodetic trace is the path taken between two places. It specifies the geodetic distance, which is the shortest distance between two places.\n", + "\n", + "ArcFace loss determines the angle between the current feature and the target weight using the arc-cosine function since the dot product between the DCNN feature and the last fully connected layer after feature and weight normalization matches the cosine distance. The target logit is then returned by multiplying the goal angle by an additional angular margin and using the cosine function. After that, we continue as before and rescale all logits to a certain feature norm, just like with softmax loss." + ] + }, + { + "cell_type": "markdown", + "id": "808ac087", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Notebook goal\n", + "\n", + "This notebook demonstrates how to use ArcFaceLoss implementation of TensorFlow Similarity with standalone usage and to train a `SimilarityModel()` on a fraction of the MNIST classes.\n", + "\n", + "You are going to learn about the main features offered by the `ArcFaceLoss()` and will:\n", + "\n", + " 1. Standalone usage of ArcFaceLoss\n", + "\n", + " 2. Usage with `model.compile()`\n", + "\n", + " 3. 3D-Visualization of ArcFaceLoss \n", + "\n", + "### Things to try \n", + "\n", + "Along the way you can try the following things to improve the model performance:\n", + "- Adding more \"seen\" classes at training time.\n", + "- Use a larger embedding by increasing the size of the output.\n", + "- Add data augmentation pre-processing layers to the model.\n", + "- Include more examples in the index to give the models more points to choose from.\n", + "- Try a more challenging dataset, such as Fashion MNIST." + ] + }, + { + "cell_type": "markdown", + "id": "078c53c0", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Notebook goal\n", + "\n", + "This notebook demonstrates how to use ArcFaceLoss implementation of TensorFlow Similarity with standalone usage and to train a `SimilarityModel()` on a fraction of the MNIST classes.\n", + "\n", + "You are going to learn about the main features offered by the `ArcFaceLoss()` and will:\n", + "\n", + " 1. Standalone usage of ArcFaceLoss\n", + "\n", + " 2. Usage with `model.compile()`\n", + "\n", + " 3. 3D-Visualization of ArcFaceLoss \n", + "\n", + "### Things to try \n", + "\n", + "Along the way you can try the following things to improve the model performance:\n", + "- Adding more \"seen\" classes at training time.\n", + "- Use a larger embedding by increasing the size of the output.\n", + "- Add data augmentation pre-processing layers to the model.\n", + "- Include more examples in the index to give the models more points to choose from.\n", + "- Try a more challenging dataset, such as Fashion MNIST." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8fd63f16", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import gc\n", + "import os\n", + "\n", + "import numpy as np\n", + "from matplotlib import pyplot as plt\n", + "from tabulate import tabulate\n", + "from mpl_toolkits.mplot3d import Axes3D\n", + "\n", + "# INFO messages are not printed.\n", + "# This must be run before loading other modules.\n", + "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"1\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80af5fc0", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import tensorflow as tf" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ba8caf7", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# install TF similarity if needed\n", + "try:\n", + " import tensorflow_similarity as tfsim # main package\n", + "except ModuleNotFoundError:\n", + " !pip install tensorflow_similarity\n", + " import tensorflow_similarity as tfsim" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2484bd72", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "tfsim.utils.tf_cap_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3fe0344e", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Clear out any old model state.\n", + "gc.collect()\n", + "tf.keras.backend.clear_session()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "99d9bef9", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "print(\"TensorFlow:\", tf.__version__)\n", + "print(\"TensorFlow Similarity\", tfsim.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7137afbc", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "1d534ad3", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "# Standalone Usage of ArcFaceLoss\n", + "\n", + "ArcFace loss alone can be used as follows when it is desired to calculate the additive angular margin loss of the existing data set." + ] + }, + { + "cell_type": "markdown", + "id": "68d526da", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Initialize Loss function as ArcFaceLoss" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bebf6ef0", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "loss_fn = tfsim.losses.ArcFaceLoss(num_classes=8, embedding_size=10)" + ] + }, + { + "cell_type": "markdown", + "id": "d2ccfd7d", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Create own simple random dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1d1ec43a", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "labels = tf.Variable([0, 1, 2, 3, 4, 5, 6, 7])\n", + "embeddings = tf.Variable(tf.random.uniform(shape=[8, 10]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "73d0c1c6", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "print(\"\", embeddings)" + ] + }, + { + "cell_type": "markdown", + "id": "d65b3085", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Calculate loss" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cdf7c30c", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "loss = loss_fn(labels, embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16745b7d", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "print(\"loss : \" , loss)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "loss = loss_fn(labels, embeddings)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "print(\"loss : \" , loss)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Data preparation\n", + "\n", + "We are going to load the MNIST dataset to showcase how the model is able to find similar examples from classes unseen during training. The model's ability to generalize the matching to unseen classes, without retraining, is one of the main reason you would want to use metric learning.\n", + "\n", + "\n", + "**WARNING**: Tensorflow similarity expects `y_train` to be an IntTensor containing the class ids for each example instead of the standard categorical encoding traditionally used for multi-class classification." + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8a9f8122", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"loss : \" , loss)" + ] + }, + { + "cell_type": "markdown", + "id": "11ef5236", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Data preparation\n", + "\n", + "We are going to load the MNIST dataset to showcase how the model is able to find similar examples from classes unseen during training. The model's ability to generalize the matching to unseen classes, without retraining, is one of the main reason you would want to use metric learning.\n", + "\n", + "\n", + "**WARNING**: Tensorflow similarity expects `y_train` to be an IntTensor containing the class ids for each example instead of the standard categorical encoding traditionally used for multi-class classification." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "97152229", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()" + ] + }, + { + "cell_type": "markdown", + "id": "08b766d8", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Model setup" + ] + }, + { + "cell_type": "markdown", + "id": "3eac2da7", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Model definition\n", + "\n", + "`SimilarityModel()` models extend `tensorflow.keras.model.Model` with additional features and functionality that allow you to index and search for similar looking examples.\n", + "\n", + "As visible in the model definition below, similarity models output a 64 dimensional float embedding using the `MetricEmbedding()` layers. This layer is a Dense layer with L2 normalization. Thanks to the loss, the model learns to minimize the distance between similar examples and maximize the distance between dissimilar examples. As a result, the distance between examples in the embedding space is meaningful; the smaller the distance the more similar the examples are. \n", + "\n", + "Being able to use a distance as a meaningful proxy for how similar two examples are, is what enables the fast ANN (aproximate nearest neighbor) search. Using a sub-linear ANN search instead of a standard quadratic NN search is what allows deep similarity search to scale to millions of items. The built in memory index used in this notebook scales to a million indexed examples very easily... if you have enough RAM :)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a003c971", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "def get_model():\n", + " inputs = tf.keras.layers.Input(shape=(28, 28, 1))\n", + " x = tf.keras.layers.experimental.preprocessing.Rescaling(1 / 255)(inputs)\n", + " x = tf.keras.layers.Conv2D(32, 3, activation=\"relu\")(x)\n", + " x = tf.keras.layers.Conv2D(32, 3, activation=\"relu\")(x)\n", + " x = tf.keras.layers.MaxPool2D()(x)\n", + " x = tf.keras.layers.Conv2D(64, 3, activation=\"relu\")(x)\n", + " x = tf.keras.layers.Conv2D(64, 3, activation=\"relu\")(x)\n", + " x = tf.keras.layers.Flatten()(x)\n", + " # smaller embeddings will have faster lookup times while a larger embedding will improve the accuracy up to a point.\n", + " outputs = tfsim.layers.MetricEmbedding(64)(x)\n", + " return tfsim.models.SimilarityModel(inputs, outputs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2177b12", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "model = get_model()\n", + "model.summary()" + ] + }, + { + "cell_type": "markdown", + "id": "defb3961", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### ArcFace Loss definition\n", + "\n", + "Overall what makes Metric losses different from tradional losses is that:\n", + "- **They expect different inputs.** Instead of having the prediction equal the true values, they expect embeddings as `y_preds` and the id (as an int32) of the class as `y_true`. \n", + "- **They require a distance.** You need to specify which `distance` function to use to compute the distance between embeddings. `cosine` is usually a great starting point and the default.\n", + "\n", + "ArcFace Loss takes inputs as number of classes which labels includes, and embedding size which we define in model `MetricEmbedding()` layers." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13b0d745", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "distance = \"cosine\" " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c22d10cc", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "num_classes = np.unique(y_train).size\n", + "embedding_size = model.get_layer('metric_embedding').output.shape[1]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5b8e426", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "loss = tfsim.losses.ArcFaceLoss(num_classes= num_classes, embedding_size=embedding_size, name=\"ArcFaceLoss\")" + ] + }, + { + "cell_type": "markdown", + "id": "b6eaf9c8", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Compilation\n", + "\n", + "Tensorflow similarity use an extended `compile()` method that allows you to optionally specify `distance_metrics` (metrics that are computed over the distance between the embeddings), and the distance to use for the indexer.\n", + "\n", + "By default the `compile()` method tries to infer what type of distance you are using by looking at the first loss specified. If you use multiple losses, and the distance loss is not the first one, then you need to specify the distance function used as `distance=` parameter in the compile function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "673f986f", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "LR = 0.0005 # @param {type:\"number\"}\n", + "model.compile(optimizer=tf.keras.optimizers.SGD(LR), loss=loss, distance=distance)" + ] + }, + { + "cell_type": "markdown", + "id": "15961601", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Training\n", + "\n", + "Similarity models are trained like normal models. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "147a6863", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "EPOCHS = 10 # @param {type:\"integer\"}\n", + "history = model.fit(x_train, y_train, epochs=EPOCHS, validation_data=(x_test, y_test))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "88e1ee4d", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plt.plot(history.history[\"loss\"])\n", + "plt.plot(history.history[\"val_loss\"])\n", + "plt.legend([\"loss\", \"val_loss\"])\n", + "plt.title(f\"Loss: {loss.name}\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5404906", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "5ad4ba20", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Prediction\n", + "\n", + "Let's predict some features and visualiza them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1936264", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "embedded_features = model.predict(x_test, verbose=1)\n", + "embedded_features /= np.linalg.norm(embedded_features, axis=1, keepdims=True)" + ] + }, + { + "cell_type": "markdown", + "id": "7c0df63b", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### 3D-Visualization of ArcFace Loss" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5aac5d98", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "fig = plt.figure()\n", + "ax = Axes3D(fig)\n", + "for c in range(len(np.unique(y_test))):\n", + " ax.plot(embedded_features[y_test==c, 0], embedded_features[y_test==c, 1], embedded_features[y_test==c, 2], '.', alpha=0.1)\n", + "plt.title('ArcFace')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8889f840", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fef529d9", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/arcface_loss.py b/arcface_loss.py new file mode 100644 index 00000000..e93a98fd --- /dev/null +++ b/arcface_loss.py @@ -0,0 +1,120 @@ +# Copyright 2022 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""ArcFace losses base class. + +ArcFace: Additive Angular Margin Loss for Deep Face +Recognition. [online] arXiv.org. Available at: +. + +""" + +from typing import Any, Callable, Dict, Optional, Tuple, Union + +import tensorflow as tf + +from tensorflow_similarity.algebra import build_masks +from tensorflow_similarity.distances import Distance, distance_canonicalizer +from tensorflow_similarity.types import FloatTensor, IntTensor +from tensorflow_similarity.utils import is_tensor_or_variable + +from .metric_loss import MetricLoss +from .utils import logsumexp + + +@tf.keras.utils.register_keras_serializable(package="Similarity") +class ArcFaceLoss(tf.keras.losses.Loss): + """Implement of ArcFace: Additive Angular Margin Loss: + Step 1: Create a trainable kernel matrix with the shape of [embedding_size, num_classes]. + Step 2: Normalize the kernel and prediction vectors. + Step 3: Calculate the cosine similarity between the normalized prediction vector and the kernel. + Step 4: Create a one-hot vector include the margin value for the ground truth class. + Step 5: Add margin_hot to the cosine similarity and multiply it by scale. + Step 6: Calculate the cross-entropy loss. + + ArcFace: Additive Angular Margin Loss for Deep Face + Recognition. [online] arXiv.org. Available at: + . + + Standalone usage: + >>> loss_fn = tfsim.losses.ArcFaceLoss(num_classes=2, embedding_size=3) + >>> labels = tf.Variable([1, 0]) + >>> embeddings = tf.Variable([[0.2, 0.3, 0.1], [0.4, 0.5, 0.5]]) + >>> loss = loss_fn(labels, embeddings) + Args: + num_classes: Number of classes. + embedding_size: The size of the embedding vectors. + margin: The margin value. + scale: s in the paper, feature scale + name: Optional name for the operation. + reduction: Type of loss reduction to apply to the loss. + """ + + def __init__( + self, + num_classes: int, + embedding_size: int, + margin: float = 0.50, # margin in radians + scale: float = 64.0, # feature scale + name: Optional[str] = None, + reduction: Callable = tf.keras.losses.Reduction.AUTO, + **kwargs + ): + + super().__init__(reduction=reduction, name=name, **kwargs) + + self.num_classes = num_classes + self.embedding_size = embedding_size + self.margin = margin + self.scale = scale + self.name = name + self.kernel = tf.Variable(tf.random.normal([embedding_size, num_classes])) + + def call(self, y_true: FloatTensor, y_pred: FloatTensor) -> FloatTensor: + + y_pred_norm = tf.math.l2_normalize(y_pred, axis=1) + kernel_norm = tf.math.l2_normalize(self.kernel, axis=0) + + cos_theta = tf.matmul(y_pred_norm, kernel_norm) + cos_theta = tf.clip_by_value(cos_theta, -1.0, 1.0) + + m_hot = tf.one_hot(y_true, self.num_classes, on_value=self.margin, axis=1) + m_hot = tf.reshape(m_hot, [-1, self.num_classes]) + + cos_theta = tf.acos(cos_theta) + cos_theta += m_hot + cos_theta = tf.math.cos(cos_theta) + cos_theta = tf.math.multiply(cos_theta, self.scale) + + cce = tf.keras.losses.SparseCategoricalCrossentropy( + from_logits=True, reduction=self.reduction + ) + loss: FloatTensor = cce(y_true, cos_theta) + + return loss + + def get_config(self) -> Dict[str, Any]: + """Contains the loss configuration. + Returns: + A Python dict containing the configuration of the loss. + """ + config = { + "num_classes": self.num_classes, + "embedding_size": self.embedding_size, + "margin": self.margin, + "scale": self.scale, + "name": self.name, + } + base_config = super().get_config() + return {**base_config, **config} diff --git a/test_losses.py b/test_losses.py new file mode 100644 index 00000000..1169423f --- /dev/null +++ b/test_losses.py @@ -0,0 +1,275 @@ +import numpy as np +import tensorflow as tf + +from tensorflow_similarity.losses import ( + ArcFaceLoss, + MultiSimilarityLoss, + PNLoss, + SoftNearestNeighborLoss, + TripletLoss, +) + +# [triplet loss] +from tensorflow_similarity.losses.xbm_loss import XBM + + +def test_triplet_loss_serialization(): + loss = TripletLoss() + config = loss.get_config() + print(config) + loss2 = TripletLoss.from_config(config) + assert loss.name == loss2.name + assert loss.distance == loss2.distance + + +def triplet_hard_loss_np(labels, embedding, margin, dist_func, soft=False): + num_data = embedding.shape[0] + # Reshape labels to compute adjacency matrix. + labels_reshaped = np.reshape(labels.astype(np.float32), (labels.shape[0], 1)) + + adjacency = np.equal(labels_reshaped, labels_reshaped.T) + pdist_matrix = dist_func(embedding) + loss_np = 0.0 + for i in range(num_data): + pos_distances = [] + neg_distances = [] + for j in range(num_data): + if adjacency[i][j] == 0: + neg_distances.append(pdist_matrix[i][j]) + if adjacency[i][j] > 0.0 and i != j: + pos_distances.append(pdist_matrix[i][j]) + + # if there are no positive pairs, distance is 0 + if len(pos_distances) == 0: + pos_distances.append(0) + + # Sort by distance. + neg_distances.sort() + min_neg_distance = neg_distances[0] + pos_distances.sort(reverse=True) + max_pos_distance = pos_distances[0] + + if soft: + loss_np += np.log1p(np.exp(max_pos_distance - min_neg_distance)) + else: + loss_np += np.maximum(0.0, max_pos_distance - min_neg_distance + margin) + + loss_np /= num_data + return loss_np + + +def test_triplet_loss(): + num_inputs = 10 + # y_true: labels + y_true = tf.random.uniform((num_inputs,), 0, 10, dtype=tf.int32) + # y_preds: embedding + y_preds = tf.random.uniform((num_inputs, 20), 0, 1) + tpl = TripletLoss() + # y_true, y_preds + loss = tpl(y_true, y_preds) + assert loss > 0.9 + + +def test_triplet_loss_easy(): + num_inputs = 10 + # y_true: labels + y_true = tf.random.uniform((num_inputs,), 0, 3, dtype=tf.int32) + # y_preds: embedding + y_preds = tf.random.uniform((num_inputs, 16), 0, 1) + tpl = TripletLoss(positive_mining_strategy="easy", negative_mining_strategy="easy") + # y_true, y_preds + loss = tpl(y_true, y_preds) + assert loss > 0 + + +def test_triplet_loss_semi_hard(): + num_inputs = 10 + # y_true: labels + y_true = tf.random.uniform((num_inputs,), 0, 3, dtype=tf.int32) + # y_preds: embedding + y_preds = tf.random.uniform((num_inputs, 16), 0, 1) + tpl = TripletLoss( + positive_mining_strategy="easy", negative_mining_strategy="semi-hard" + ) + # y_true, y_preds + loss = tpl(y_true, y_preds) + assert loss + + +def test_triplet_loss_hard(): + num_inputs = 10 + # y_true: labels + y_true = tf.random.uniform((num_inputs,), 0, 3, dtype=tf.int32) + # y_preds: embedding + y_preds = tf.random.uniform((num_inputs, 16), 0, 1) + tpl = TripletLoss(positive_mining_strategy="hard", negative_mining_strategy="hard") + # y_true, y_preds + loss = tpl(y_true, y_preds) + assert loss + + +# [pn loss] +def test_pn_loss_serialization(): + loss = PNLoss() + config = loss.get_config() + print(config) + loss2 = PNLoss.from_config(config) + assert loss.name == loss2.name + assert loss.distance == loss2.distance + + +def test_np_loss(): + num_inputs = 10 + # y_true: labels + y_true = tf.random.uniform((num_inputs,), 0, 10, dtype=tf.int32) + # y_preds: embedding + y_preds = tf.random.uniform((num_inputs, 20), 0, 1) + pnl = PNLoss() + # y_true, y_preds + loss = pnl(y_true, y_preds) + assert loss > 0.9 + + +# [soft neasrest neighbor loss] +def test_softnn_loss_serialization(): + loss = SoftNearestNeighborLoss(distance="cosine", temperature=50) + config = loss.get_config() + loss2 = SoftNearestNeighborLoss.from_config(config) + assert loss.name == loss2.name + assert loss.distance == loss2.distance + assert loss.temperature == loss2.temperature + + +def softnn_util(y_true, x, temperature=1): + """ + A simple loop based implementation of soft + nearest neighbor loss to test the code. + https://arxiv.org/pdf/1902.01889.pdf + """ + + y_true = y_true.numpy() + x = x.numpy() + batch_size = y_true.shape[0] + loss = 0 + eps = 1e-9 + for i in range(batch_size): + numerator = 0 + denominator = 0 + for j in range(batch_size): + if i == j: + continue + if y_true[i] == y_true[j]: + numerator += np.exp(-1 * np.sum(np.square(x[i] - x[j])) / temperature) + denominator += np.exp(-1 * np.sum(np.square(x[i] - x[j])) / temperature) + if numerator == 0: + continue + loss += np.log(numerator / denominator) + return -loss / batch_size + + +def test_softnn_loss(): + num_inputs = 10 + n_classes = 10 + # y_true: labels + y_true = tf.random.uniform((num_inputs,), 0, n_classes, dtype=tf.int32) + # x: embeddings + x = tf.random.uniform((num_inputs, 20), 0, 1) + + temperature = np.random.uniform(0.1, 50) + softnn = SoftNearestNeighborLoss(temperature=temperature) + loss = softnn(y_true, x) + loss_check = softnn_util(y_true, x, temperature) + loss_diff = loss.numpy() - loss_check + assert np.abs(loss_diff) < 1e-3 + + +def test_xbm_loss(): + batch_size = 6 + embed_dim = 16 + + embeddings1 = tf.random.uniform(shape=[batch_size, embed_dim]) + labels1 = tf.constant( + [ + [1], + [1], + [2], + [2], + [3], + [3], + ], + dtype=tf.int32, + ) + + embeddings2 = tf.random.uniform(shape=[batch_size, embed_dim]) + labels2 = tf.constant( + [ + [4], + [4], + [5], + [5], + [6], + [6], + ], + dtype=tf.int32, + ) + + distance = "cosine" + loss = MultiSimilarityLoss(distance=distance) + loss_nowarm = XBM(loss, memory_size=12, warmup_steps=0) + + # test enqueue + loss_nowarm(labels1, embeddings1) + assert loss_nowarm._y_pred_memory.numpy().shape == (batch_size, embed_dim) + tf.assert_equal(loss_nowarm._y_true_memory, labels1) + + loss_nowarm(labels2, embeddings2) + assert loss_nowarm._y_pred_memory.numpy().shape == (2 * batch_size, embed_dim) + tf.assert_equal(loss_nowarm._y_true_memory, tf.concat([labels2, labels1], axis=0)) + + # test dequeue + loss_nowarm(labels2, embeddings2) + assert loss_nowarm._y_pred_memory.numpy().shape == (2 * batch_size, embed_dim) + tf.assert_equal(loss_nowarm._y_true_memory, tf.concat([labels2, labels2], axis=0)) + + # test warmup + loss_warm = XBM(loss, memory_size=12, warmup_steps=1) + + loss_warm(labels1, embeddings1) + assert loss_warm._y_pred_memory.numpy().shape == (0, embed_dim) + tf.assert_equal(loss_warm._y_true_memory, tf.constant([[]], dtype=tf.int32)) + + loss_warm(labels2, embeddings2) + assert loss_warm._y_pred_memory.numpy().shape == (batch_size, embed_dim) + tf.assert_equal(loss_warm._y_true_memory, labels2) + + +# arcface loss +""" +ArcFaceLoss + ArcFace: Additive Angular Margin Loss for Deep Face + Recognition. [online] arXiv.org. Available at: + . +""" + + +def test_arcface_loss_serialization(): + n_classes = 10 + embed_size = 16 + loss = ArcFaceLoss(num_classes=n_classes, embedding_size=embed_size) + config = loss.get_config() + loss2 = ArcFaceLoss.from_config(config) + assert loss.name == loss2.name + assert loss.margin == loss2.margin + assert loss.scale == loss2.scale + assert loss.num_classes == loss2.num_classes + assert loss.embedding_size == loss2.embedding_size + + +def test_arcface_loss(): + tf.random.set_seed(128) + loss_fn = ArcFaceLoss(num_classes=4, embedding_size=5) + labels = tf.Variable([0, 1, 2, 3]) + embeddings = tf.Variable(tf.random.uniform(shape=[4, 5])) + loss = loss_fn(labels, embeddings) + + assert 60.4 < loss.numpy() < 60.5 From 7b84548cafd5d2695856e5faad35bb2c7951c726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aylin=20Ayd=C4=B1n?= Date: Wed, 21 Sep 2022 23:38:44 +0300 Subject: [PATCH 03/25] Delete ArcFace Loss Sample Notebook.ipynb --- .../losses/ArcFace Loss Sample Notebook.ipynb | 775 ------------------ 1 file changed, 775 deletions(-) delete mode 100644 tensorflow_similarity/losses/ArcFace Loss Sample Notebook.ipynb diff --git a/tensorflow_similarity/losses/ArcFace Loss Sample Notebook.ipynb b/tensorflow_similarity/losses/ArcFace Loss Sample Notebook.ipynb deleted file mode 100644 index b481492e..00000000 --- a/tensorflow_similarity/losses/ArcFace Loss Sample Notebook.ipynb +++ /dev/null @@ -1,775 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "28956aa1", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "Copyright 2022 The TensorFlow Similarity Authors." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24eda1a6", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "# @title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "id": "7ca9d025", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "# TensorFlow Similarity ArcFace Loss Example" - ] - }, - { - "cell_type": "markdown", - "id": "d072628f", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "A Total Angular Margin Loss (ArcFace) calculates the geodetic distance in the hypersphere instead of the euclidean distance to improve the discriminatory strength of the facial recognition model and stabilize the training process. Rails are used to measure all distances in geodetic space. The geodetic trace is the path taken between two places. It specifies the geodetic distance, which is the shortest distance between two places.\n", - "\n", - "ArcFace loss determines the angle between the current feature and the target weight using the arc-cosine function since the dot product between the DCNN feature and the last fully connected layer after feature and weight normalization matches the cosine distance. The target logit is then returned by multiplying the goal angle by an additional angular margin and using the cosine function. After that, we continue as before and rescale all logits to a certain feature norm, just like with softmax loss." - ] - }, - { - "cell_type": "markdown", - "id": "808ac087", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Notebook goal\n", - "\n", - "This notebook demonstrates how to use ArcFaceLoss implementation of TensorFlow Similarity with standalone usage and to train a `SimilarityModel()` on a fraction of the MNIST classes.\n", - "\n", - "You are going to learn about the main features offered by the `ArcFaceLoss()` and will:\n", - "\n", - " 1. Standalone usage of ArcFaceLoss\n", - "\n", - " 2. Usage with `model.compile()`\n", - "\n", - " 3. 3D-Visualization of ArcFaceLoss \n", - "\n", - "### Things to try \n", - "\n", - "Along the way you can try the following things to improve the model performance:\n", - "- Adding more \"seen\" classes at training time.\n", - "- Use a larger embedding by increasing the size of the output.\n", - "- Add data augmentation pre-processing layers to the model.\n", - "- Include more examples in the index to give the models more points to choose from.\n", - "- Try a more challenging dataset, such as Fashion MNIST." - ] - }, - { - "cell_type": "markdown", - "id": "078c53c0", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Notebook goal\n", - "\n", - "This notebook demonstrates how to use ArcFaceLoss implementation of TensorFlow Similarity with standalone usage and to train a `SimilarityModel()` on a fraction of the MNIST classes.\n", - "\n", - "You are going to learn about the main features offered by the `ArcFaceLoss()` and will:\n", - "\n", - " 1. Standalone usage of ArcFaceLoss\n", - "\n", - " 2. Usage with `model.compile()`\n", - "\n", - " 3. 3D-Visualization of ArcFaceLoss \n", - "\n", - "### Things to try \n", - "\n", - "Along the way you can try the following things to improve the model performance:\n", - "- Adding more \"seen\" classes at training time.\n", - "- Use a larger embedding by increasing the size of the output.\n", - "- Add data augmentation pre-processing layers to the model.\n", - "- Include more examples in the index to give the models more points to choose from.\n", - "- Try a more challenging dataset, such as Fashion MNIST." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8fd63f16", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "import gc\n", - "import os\n", - "\n", - "import numpy as np\n", - "from matplotlib import pyplot as plt\n", - "from tabulate import tabulate\n", - "from mpl_toolkits.mplot3d import Axes3D\n", - "\n", - "# INFO messages are not printed.\n", - "# This must be run before loading other modules.\n", - "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"1\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "80af5fc0", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "import tensorflow as tf" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8ba8caf7", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "# install TF similarity if needed\n", - "try:\n", - " import tensorflow_similarity as tfsim # main package\n", - "except ModuleNotFoundError:\n", - " !pip install tensorflow_similarity\n", - " import tensorflow_similarity as tfsim" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2484bd72", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "tfsim.utils.tf_cap_memory()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3fe0344e", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "# Clear out any old model state.\n", - "gc.collect()\n", - "tf.keras.backend.clear_session()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "99d9bef9", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "print(\"TensorFlow:\", tf.__version__)\n", - "print(\"TensorFlow Similarity\", tfsim.__version__)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7137afbc", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "1d534ad3", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "# Standalone Usage of ArcFaceLoss\n", - "\n", - "ArcFace loss alone can be used as follows when it is desired to calculate the additive angular margin loss of the existing data set." - ] - }, - { - "cell_type": "markdown", - "id": "68d526da", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Initialize Loss function as ArcFaceLoss" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bebf6ef0", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "loss_fn = tfsim.losses.ArcFaceLoss(num_classes=8, embedding_size=10)" - ] - }, - { - "cell_type": "markdown", - "id": "d2ccfd7d", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Create own simple random dataset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1d1ec43a", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "labels = tf.Variable([0, 1, 2, 3, 4, 5, 6, 7])\n", - "embeddings = tf.Variable(tf.random.uniform(shape=[8, 10]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "73d0c1c6", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "print(\"\", embeddings)" - ] - }, - { - "cell_type": "markdown", - "id": "d65b3085", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Calculate loss" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cdf7c30c", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "loss = loss_fn(labels, embeddings)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "16745b7d", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "print(\"loss : \" , loss)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "loss = loss_fn(labels, embeddings)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "print(\"loss : \" , loss)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "markdown", - "source": [ - "## Data preparation\n", - "\n", - "We are going to load the MNIST dataset to showcase how the model is able to find similar examples from classes unseen during training. The model's ability to generalize the matching to unseen classes, without retraining, is one of the main reason you would want to use metric learning.\n", - "\n", - "\n", - "**WARNING**: Tensorflow similarity expects `y_train` to be an IntTensor containing the class ids for each example instead of the standard categorical encoding traditionally used for multi-class classification." - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8a9f8122", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"loss : \" , loss)" - ] - }, - { - "cell_type": "markdown", - "id": "11ef5236", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "## Data preparation\n", - "\n", - "We are going to load the MNIST dataset to showcase how the model is able to find similar examples from classes unseen during training. The model's ability to generalize the matching to unseen classes, without retraining, is one of the main reason you would want to use metric learning.\n", - "\n", - "\n", - "**WARNING**: Tensorflow similarity expects `y_train` to be an IntTensor containing the class ids for each example instead of the standard categorical encoding traditionally used for multi-class classification." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "97152229", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()" - ] - }, - { - "cell_type": "markdown", - "id": "08b766d8", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "## Model setup" - ] - }, - { - "cell_type": "markdown", - "id": "3eac2da7", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Model definition\n", - "\n", - "`SimilarityModel()` models extend `tensorflow.keras.model.Model` with additional features and functionality that allow you to index and search for similar looking examples.\n", - "\n", - "As visible in the model definition below, similarity models output a 64 dimensional float embedding using the `MetricEmbedding()` layers. This layer is a Dense layer with L2 normalization. Thanks to the loss, the model learns to minimize the distance between similar examples and maximize the distance between dissimilar examples. As a result, the distance between examples in the embedding space is meaningful; the smaller the distance the more similar the examples are. \n", - "\n", - "Being able to use a distance as a meaningful proxy for how similar two examples are, is what enables the fast ANN (aproximate nearest neighbor) search. Using a sub-linear ANN search instead of a standard quadratic NN search is what allows deep similarity search to scale to millions of items. The built in memory index used in this notebook scales to a million indexed examples very easily... if you have enough RAM :)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a003c971", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "def get_model():\n", - " inputs = tf.keras.layers.Input(shape=(28, 28, 1))\n", - " x = tf.keras.layers.experimental.preprocessing.Rescaling(1 / 255)(inputs)\n", - " x = tf.keras.layers.Conv2D(32, 3, activation=\"relu\")(x)\n", - " x = tf.keras.layers.Conv2D(32, 3, activation=\"relu\")(x)\n", - " x = tf.keras.layers.MaxPool2D()(x)\n", - " x = tf.keras.layers.Conv2D(64, 3, activation=\"relu\")(x)\n", - " x = tf.keras.layers.Conv2D(64, 3, activation=\"relu\")(x)\n", - " x = tf.keras.layers.Flatten()(x)\n", - " # smaller embeddings will have faster lookup times while a larger embedding will improve the accuracy up to a point.\n", - " outputs = tfsim.layers.MetricEmbedding(64)(x)\n", - " return tfsim.models.SimilarityModel(inputs, outputs)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a2177b12", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "model = get_model()\n", - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "id": "defb3961", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### ArcFace Loss definition\n", - "\n", - "Overall what makes Metric losses different from tradional losses is that:\n", - "- **They expect different inputs.** Instead of having the prediction equal the true values, they expect embeddings as `y_preds` and the id (as an int32) of the class as `y_true`. \n", - "- **They require a distance.** You need to specify which `distance` function to use to compute the distance between embeddings. `cosine` is usually a great starting point and the default.\n", - "\n", - "ArcFace Loss takes inputs as number of classes which labels includes, and embedding size which we define in model `MetricEmbedding()` layers." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "13b0d745", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "distance = \"cosine\" " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c22d10cc", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "num_classes = np.unique(y_train).size\n", - "embedding_size = model.get_layer('metric_embedding').output.shape[1]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d5b8e426", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "loss = tfsim.losses.ArcFaceLoss(num_classes= num_classes, embedding_size=embedding_size, name=\"ArcFaceLoss\")" - ] - }, - { - "cell_type": "markdown", - "id": "b6eaf9c8", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Compilation\n", - "\n", - "Tensorflow similarity use an extended `compile()` method that allows you to optionally specify `distance_metrics` (metrics that are computed over the distance between the embeddings), and the distance to use for the indexer.\n", - "\n", - "By default the `compile()` method tries to infer what type of distance you are using by looking at the first loss specified. If you use multiple losses, and the distance loss is not the first one, then you need to specify the distance function used as `distance=` parameter in the compile function." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "673f986f", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "LR = 0.0005 # @param {type:\"number\"}\n", - "model.compile(optimizer=tf.keras.optimizers.SGD(LR), loss=loss, distance=distance)" - ] - }, - { - "cell_type": "markdown", - "id": "15961601", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "## Training\n", - "\n", - "Similarity models are trained like normal models. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "147a6863", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "EPOCHS = 10 # @param {type:\"integer\"}\n", - "history = model.fit(x_train, y_train, epochs=EPOCHS, validation_data=(x_test, y_test))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "88e1ee4d", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "plt.plot(history.history[\"loss\"])\n", - "plt.plot(history.history[\"val_loss\"])\n", - "plt.legend([\"loss\", \"val_loss\"])\n", - "plt.title(f\"Loss: {loss.name}\")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a5404906", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "5ad4ba20", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "## Prediction\n", - "\n", - "Let's predict some features and visualiza them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a1936264", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "embedded_features = model.predict(x_test, verbose=1)\n", - "embedded_features /= np.linalg.norm(embedded_features, axis=1, keepdims=True)" - ] - }, - { - "cell_type": "markdown", - "id": "7c0df63b", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### 3D-Visualization of ArcFace Loss" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5aac5d98", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "fig = plt.figure()\n", - "ax = Axes3D(fig)\n", - "for c in range(len(np.unique(y_test))):\n", - " ax.plot(embedded_features[y_test==c, 0], embedded_features[y_test==c, 1], embedded_features[y_test==c, 2], '.', alpha=0.1)\n", - "plt.title('ArcFace')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8889f840", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fef529d9", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file From 49a18f69455a5d7da984d2f74963a164ec50b52f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aylin=20Ayd=C4=B1n?= Date: Wed, 21 Sep 2022 23:38:55 +0300 Subject: [PATCH 04/25] Delete test_losses.py --- tensorflow_similarity/losses/test_losses.py | 275 -------------------- 1 file changed, 275 deletions(-) delete mode 100644 tensorflow_similarity/losses/test_losses.py diff --git a/tensorflow_similarity/losses/test_losses.py b/tensorflow_similarity/losses/test_losses.py deleted file mode 100644 index 1d4fef9c..00000000 --- a/tensorflow_similarity/losses/test_losses.py +++ /dev/null @@ -1,275 +0,0 @@ -import tensorflow as tf -import numpy as np -from tensorflow_similarity.losses import TripletLoss, MultiSimilarityLoss -from tensorflow_similarity.losses import PNLoss -from tensorflow_similarity.losses import SoftNearestNeighborLoss -from tensorflow_similarity.losses import ArcFaceLoss - -# [triplet loss] -from tensorflow_similarity.losses.xbm_loss import XBM - - -def test_triplet_loss_serialization(): - loss = TripletLoss() - config = loss.get_config() - print(config) - loss2 = TripletLoss.from_config(config) - assert loss.name == loss2.name - assert loss.distance == loss2.distance - - -def triplet_hard_loss_np(labels, embedding, margin, dist_func, soft=False): - num_data = embedding.shape[0] - # Reshape labels to compute adjacency matrix. - labels_reshaped = np.reshape(labels.astype(np.float32), - (labels.shape[0], 1)) - - adjacency = np.equal(labels_reshaped, labels_reshaped.T) - pdist_matrix = dist_func(embedding) - loss_np = 0.0 - for i in range(num_data): - pos_distances = [] - neg_distances = [] - for j in range(num_data): - if adjacency[i][j] == 0: - neg_distances.append(pdist_matrix[i][j]) - if adjacency[i][j] > 0.0 and i != j: - pos_distances.append(pdist_matrix[i][j]) - - # if there are no positive pairs, distance is 0 - if len(pos_distances) == 0: - pos_distances.append(0) - - # Sort by distance. - neg_distances.sort() - min_neg_distance = neg_distances[0] - pos_distances.sort(reverse=True) - max_pos_distance = pos_distances[0] - - if soft: - loss_np += np.log1p(np.exp(max_pos_distance - min_neg_distance)) - else: - loss_np += np.maximum(0.0, - max_pos_distance - min_neg_distance + margin) - - loss_np /= num_data - return loss_np - - -def test_triplet_loss(): - num_inputs = 10 - # y_true: labels - y_true = tf.random.uniform((num_inputs,), 0, 10, dtype=tf.int32) - # y_preds: embedding - y_preds = tf.random.uniform((num_inputs, 20), 0, 1) - tpl = TripletLoss() - # y_true, y_preds - loss = tpl(y_true, y_preds) - assert loss > 0.9 - - -def test_triplet_loss_easy(): - num_inputs = 10 - # y_true: labels - y_true = tf.random.uniform((num_inputs,), 0, 3, dtype=tf.int32) - # y_preds: embedding - y_preds = tf.random.uniform((num_inputs, 16), 0, 1) - tpl = TripletLoss(positive_mining_strategy='easy', - negative_mining_strategy='easy') - # y_true, y_preds - loss = tpl(y_true, y_preds) - assert loss > 0 - - -def test_triplet_loss_semi_hard(): - num_inputs = 10 - # y_true: labels - y_true = tf.random.uniform((num_inputs,), 0, 3, dtype=tf.int32) - # y_preds: embedding - y_preds = tf.random.uniform((num_inputs, 16), 0, 1) - tpl = TripletLoss(positive_mining_strategy='easy', - negative_mining_strategy='semi-hard') - # y_true, y_preds - loss = tpl(y_true, y_preds) - assert loss - - -def test_triplet_loss_hard(): - num_inputs = 10 - # y_true: labels - y_true = tf.random.uniform((num_inputs,), 0, 3, dtype=tf.int32) - # y_preds: embedding - y_preds = tf.random.uniform((num_inputs, 16), 0, 1) - tpl = TripletLoss(positive_mining_strategy='hard', - negative_mining_strategy='hard') - # y_true, y_preds - loss = tpl(y_true, y_preds) - assert loss - - -# [pn loss] -def test_pn_loss_serialization(): - loss = PNLoss() - config = loss.get_config() - print(config) - loss2 = PNLoss.from_config(config) - assert loss.name == loss2.name - assert loss.distance == loss2.distance - - -def test_np_loss(): - num_inputs = 10 - # y_true: labels - y_true = tf.random.uniform((num_inputs,), 0, 10, dtype=tf.int32) - # y_preds: embedding - y_preds = tf.random.uniform((num_inputs, 20), 0, 1) - pnl = PNLoss() - # y_true, y_preds - loss = pnl(y_true, y_preds) - assert loss > 0.9 - - -# [soft neasrest neighbor loss] -def test_softnn_loss_serialization(): - loss = SoftNearestNeighborLoss(distance="cosine", temperature=50) - config = loss.get_config() - loss2 = SoftNearestNeighborLoss.from_config(config) - assert loss.name == loss2.name - assert loss.distance == loss2.distance - assert loss.temperature == loss2.temperature - - -def softnn_util(y_true, x, temperature=1): - """ - A simple loop based implementation of soft - nearest neighbor loss to test the code. - https://arxiv.org/pdf/1902.01889.pdf - """ - - y_true = y_true.numpy() - x = x.numpy() - batch_size = y_true.shape[0] - loss = 0 - eps = 1e-9 - for i in range(batch_size): - numerator = 0 - denominator = 0 - for j in range(batch_size): - if i == j: continue - if y_true[i] == y_true[j]: - numerator += np.exp(-1 * - np.sum(np.square(x[i] - x[j])) / temperature) - denominator += np.exp(-1 * - np.sum(np.square(x[i] - x[j])) / temperature) - if numerator == 0: continue - loss += np.log(numerator / denominator) - return -loss / batch_size - - -def test_softnn_loss(): - num_inputs = 10 - n_classes = 10 - # y_true: labels - y_true = tf.random.uniform((num_inputs,), 0, n_classes, dtype=tf.int32) - # x: embeddings - x = tf.random.uniform((num_inputs, 20), 0, 1) - - temperature = np.random.uniform(0.1, 50) - softnn = SoftNearestNeighborLoss(temperature=temperature) - loss = softnn(y_true, x) - loss_check = softnn_util(y_true, x, temperature) - loss_diff = loss.numpy() - loss_check - assert np.abs(loss_diff) < 1e-3 - - -def test_xbm_loss(): - batch_size = 6 - embed_dim = 16 - - embeddings1 = tf.random.uniform(shape=[batch_size, embed_dim]) - labels1 = tf.constant( - [ - [1], - [1], - [2], - [2], - [3], - [3], - ], - dtype=tf.int32 - ) - - embeddings2 = tf.random.uniform(shape=[batch_size, embed_dim]) - labels2 = tf.constant( - [ - [4], - [4], - [5], - [5], - [6], - [6], - ], - dtype=tf.int32 - ) - - distance = "cosine" - loss = MultiSimilarityLoss(distance=distance) - loss_nowarm = XBM(loss, memory_size=12, warmup_steps=0) - - # test enqueue - loss_nowarm(labels1, embeddings1) - assert loss_nowarm._y_pred_memory.numpy().shape == (batch_size, embed_dim) - tf.assert_equal(loss_nowarm._y_true_memory, labels1) - - loss_nowarm(labels2, embeddings2) - assert loss_nowarm._y_pred_memory.numpy().shape == (2 * batch_size, embed_dim) - tf.assert_equal(loss_nowarm._y_true_memory, tf.concat([labels2, labels1], axis=0)) - - # test dequeue - loss_nowarm(labels2, embeddings2) - assert loss_nowarm._y_pred_memory.numpy().shape == (2 * batch_size, embed_dim) - tf.assert_equal(loss_nowarm._y_true_memory, tf.concat([labels2, labels2], axis=0)) - - # test warmup - loss_warm = XBM(loss, memory_size=12, warmup_steps=1) - - loss_warm(labels1, embeddings1) - assert loss_warm._y_pred_memory.numpy().shape == (0, embed_dim) - tf.assert_equal(loss_warm._y_true_memory, tf.constant([[]], dtype=tf.int32)) - - loss_warm(labels2, embeddings2) - assert loss_warm._y_pred_memory.numpy().shape == (batch_size, embed_dim) - tf.assert_equal(loss_warm._y_true_memory, labels2) - - -# arcface loss -""" -ArcFaceLoss - ArcFace: Additive Angular Margin Loss for Deep Face - Recognition. [online] arXiv.org. Available at: - . -""" - - -def test_arcface_loss_serialization(): - n_classes = 10 - embed_size = 16 - loss = ArcFaceLoss(num_classes=n_classes, embedding_size=embed_size) - config = loss.get_config() - loss2 = ArcFaceLoss.from_config(config) - assert loss.name == loss2.name - assert loss.margin == loss2.margin - assert loss.scale == loss2.scale - assert loss.num_classes == loss2.num_classes - assert loss.embedding_size == loss2.embedding_size - - - -def test_arcface_loss(): - tf.random.set_seed(128) - loss_fn = ArcFaceLoss(num_classes=4, embedding_size=5) - labels = tf.Variable([0, 1, 2, 3]) - embeddings = tf.Variable(tf.random.uniform(shape=[4, 5])) - loss = loss_fn(labels, embeddings) - - assert 60.4 < loss.numpy() < 60.5 From e0ea837a0a17e26f44044b31a97ec3f53279d7d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aylin=20Ayd=C4=B1n?= Date: Wed, 21 Sep 2022 23:39:40 +0300 Subject: [PATCH 05/25] Add files via upload --- examples/ArcFace Loss Sample Notebook.ipynb | 775 ++++++++++++++++++++ 1 file changed, 775 insertions(+) create mode 100644 examples/ArcFace Loss Sample Notebook.ipynb diff --git a/examples/ArcFace Loss Sample Notebook.ipynb b/examples/ArcFace Loss Sample Notebook.ipynb new file mode 100644 index 00000000..b481492e --- /dev/null +++ b/examples/ArcFace Loss Sample Notebook.ipynb @@ -0,0 +1,775 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "28956aa1", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "Copyright 2022 The TensorFlow Similarity Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24eda1a6", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# @title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "id": "7ca9d025", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "# TensorFlow Similarity ArcFace Loss Example" + ] + }, + { + "cell_type": "markdown", + "id": "d072628f", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "A Total Angular Margin Loss (ArcFace) calculates the geodetic distance in the hypersphere instead of the euclidean distance to improve the discriminatory strength of the facial recognition model and stabilize the training process. Rails are used to measure all distances in geodetic space. The geodetic trace is the path taken between two places. It specifies the geodetic distance, which is the shortest distance between two places.\n", + "\n", + "ArcFace loss determines the angle between the current feature and the target weight using the arc-cosine function since the dot product between the DCNN feature and the last fully connected layer after feature and weight normalization matches the cosine distance. The target logit is then returned by multiplying the goal angle by an additional angular margin and using the cosine function. After that, we continue as before and rescale all logits to a certain feature norm, just like with softmax loss." + ] + }, + { + "cell_type": "markdown", + "id": "808ac087", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Notebook goal\n", + "\n", + "This notebook demonstrates how to use ArcFaceLoss implementation of TensorFlow Similarity with standalone usage and to train a `SimilarityModel()` on a fraction of the MNIST classes.\n", + "\n", + "You are going to learn about the main features offered by the `ArcFaceLoss()` and will:\n", + "\n", + " 1. Standalone usage of ArcFaceLoss\n", + "\n", + " 2. Usage with `model.compile()`\n", + "\n", + " 3. 3D-Visualization of ArcFaceLoss \n", + "\n", + "### Things to try \n", + "\n", + "Along the way you can try the following things to improve the model performance:\n", + "- Adding more \"seen\" classes at training time.\n", + "- Use a larger embedding by increasing the size of the output.\n", + "- Add data augmentation pre-processing layers to the model.\n", + "- Include more examples in the index to give the models more points to choose from.\n", + "- Try a more challenging dataset, such as Fashion MNIST." + ] + }, + { + "cell_type": "markdown", + "id": "078c53c0", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Notebook goal\n", + "\n", + "This notebook demonstrates how to use ArcFaceLoss implementation of TensorFlow Similarity with standalone usage and to train a `SimilarityModel()` on a fraction of the MNIST classes.\n", + "\n", + "You are going to learn about the main features offered by the `ArcFaceLoss()` and will:\n", + "\n", + " 1. Standalone usage of ArcFaceLoss\n", + "\n", + " 2. Usage with `model.compile()`\n", + "\n", + " 3. 3D-Visualization of ArcFaceLoss \n", + "\n", + "### Things to try \n", + "\n", + "Along the way you can try the following things to improve the model performance:\n", + "- Adding more \"seen\" classes at training time.\n", + "- Use a larger embedding by increasing the size of the output.\n", + "- Add data augmentation pre-processing layers to the model.\n", + "- Include more examples in the index to give the models more points to choose from.\n", + "- Try a more challenging dataset, such as Fashion MNIST." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8fd63f16", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import gc\n", + "import os\n", + "\n", + "import numpy as np\n", + "from matplotlib import pyplot as plt\n", + "from tabulate import tabulate\n", + "from mpl_toolkits.mplot3d import Axes3D\n", + "\n", + "# INFO messages are not printed.\n", + "# This must be run before loading other modules.\n", + "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"1\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80af5fc0", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import tensorflow as tf" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ba8caf7", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# install TF similarity if needed\n", + "try:\n", + " import tensorflow_similarity as tfsim # main package\n", + "except ModuleNotFoundError:\n", + " !pip install tensorflow_similarity\n", + " import tensorflow_similarity as tfsim" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2484bd72", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "tfsim.utils.tf_cap_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3fe0344e", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Clear out any old model state.\n", + "gc.collect()\n", + "tf.keras.backend.clear_session()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "99d9bef9", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "print(\"TensorFlow:\", tf.__version__)\n", + "print(\"TensorFlow Similarity\", tfsim.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7137afbc", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "1d534ad3", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "# Standalone Usage of ArcFaceLoss\n", + "\n", + "ArcFace loss alone can be used as follows when it is desired to calculate the additive angular margin loss of the existing data set." + ] + }, + { + "cell_type": "markdown", + "id": "68d526da", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Initialize Loss function as ArcFaceLoss" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bebf6ef0", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "loss_fn = tfsim.losses.ArcFaceLoss(num_classes=8, embedding_size=10)" + ] + }, + { + "cell_type": "markdown", + "id": "d2ccfd7d", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Create own simple random dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1d1ec43a", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "labels = tf.Variable([0, 1, 2, 3, 4, 5, 6, 7])\n", + "embeddings = tf.Variable(tf.random.uniform(shape=[8, 10]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "73d0c1c6", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "print(\"\", embeddings)" + ] + }, + { + "cell_type": "markdown", + "id": "d65b3085", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Calculate loss" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cdf7c30c", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "loss = loss_fn(labels, embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16745b7d", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "print(\"loss : \" , loss)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "loss = loss_fn(labels, embeddings)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "print(\"loss : \" , loss)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## Data preparation\n", + "\n", + "We are going to load the MNIST dataset to showcase how the model is able to find similar examples from classes unseen during training. The model's ability to generalize the matching to unseen classes, without retraining, is one of the main reason you would want to use metric learning.\n", + "\n", + "\n", + "**WARNING**: Tensorflow similarity expects `y_train` to be an IntTensor containing the class ids for each example instead of the standard categorical encoding traditionally used for multi-class classification." + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8a9f8122", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"loss : \" , loss)" + ] + }, + { + "cell_type": "markdown", + "id": "11ef5236", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Data preparation\n", + "\n", + "We are going to load the MNIST dataset to showcase how the model is able to find similar examples from classes unseen during training. The model's ability to generalize the matching to unseen classes, without retraining, is one of the main reason you would want to use metric learning.\n", + "\n", + "\n", + "**WARNING**: Tensorflow similarity expects `y_train` to be an IntTensor containing the class ids for each example instead of the standard categorical encoding traditionally used for multi-class classification." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "97152229", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()" + ] + }, + { + "cell_type": "markdown", + "id": "08b766d8", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Model setup" + ] + }, + { + "cell_type": "markdown", + "id": "3eac2da7", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Model definition\n", + "\n", + "`SimilarityModel()` models extend `tensorflow.keras.model.Model` with additional features and functionality that allow you to index and search for similar looking examples.\n", + "\n", + "As visible in the model definition below, similarity models output a 64 dimensional float embedding using the `MetricEmbedding()` layers. This layer is a Dense layer with L2 normalization. Thanks to the loss, the model learns to minimize the distance between similar examples and maximize the distance between dissimilar examples. As a result, the distance between examples in the embedding space is meaningful; the smaller the distance the more similar the examples are. \n", + "\n", + "Being able to use a distance as a meaningful proxy for how similar two examples are, is what enables the fast ANN (aproximate nearest neighbor) search. Using a sub-linear ANN search instead of a standard quadratic NN search is what allows deep similarity search to scale to millions of items. The built in memory index used in this notebook scales to a million indexed examples very easily... if you have enough RAM :)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a003c971", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "def get_model():\n", + " inputs = tf.keras.layers.Input(shape=(28, 28, 1))\n", + " x = tf.keras.layers.experimental.preprocessing.Rescaling(1 / 255)(inputs)\n", + " x = tf.keras.layers.Conv2D(32, 3, activation=\"relu\")(x)\n", + " x = tf.keras.layers.Conv2D(32, 3, activation=\"relu\")(x)\n", + " x = tf.keras.layers.MaxPool2D()(x)\n", + " x = tf.keras.layers.Conv2D(64, 3, activation=\"relu\")(x)\n", + " x = tf.keras.layers.Conv2D(64, 3, activation=\"relu\")(x)\n", + " x = tf.keras.layers.Flatten()(x)\n", + " # smaller embeddings will have faster lookup times while a larger embedding will improve the accuracy up to a point.\n", + " outputs = tfsim.layers.MetricEmbedding(64)(x)\n", + " return tfsim.models.SimilarityModel(inputs, outputs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2177b12", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "model = get_model()\n", + "model.summary()" + ] + }, + { + "cell_type": "markdown", + "id": "defb3961", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### ArcFace Loss definition\n", + "\n", + "Overall what makes Metric losses different from tradional losses is that:\n", + "- **They expect different inputs.** Instead of having the prediction equal the true values, they expect embeddings as `y_preds` and the id (as an int32) of the class as `y_true`. \n", + "- **They require a distance.** You need to specify which `distance` function to use to compute the distance between embeddings. `cosine` is usually a great starting point and the default.\n", + "\n", + "ArcFace Loss takes inputs as number of classes which labels includes, and embedding size which we define in model `MetricEmbedding()` layers." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13b0d745", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "distance = \"cosine\" " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c22d10cc", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "num_classes = np.unique(y_train).size\n", + "embedding_size = model.get_layer('metric_embedding').output.shape[1]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5b8e426", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "loss = tfsim.losses.ArcFaceLoss(num_classes= num_classes, embedding_size=embedding_size, name=\"ArcFaceLoss\")" + ] + }, + { + "cell_type": "markdown", + "id": "b6eaf9c8", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Compilation\n", + "\n", + "Tensorflow similarity use an extended `compile()` method that allows you to optionally specify `distance_metrics` (metrics that are computed over the distance between the embeddings), and the distance to use for the indexer.\n", + "\n", + "By default the `compile()` method tries to infer what type of distance you are using by looking at the first loss specified. If you use multiple losses, and the distance loss is not the first one, then you need to specify the distance function used as `distance=` parameter in the compile function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "673f986f", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "LR = 0.0005 # @param {type:\"number\"}\n", + "model.compile(optimizer=tf.keras.optimizers.SGD(LR), loss=loss, distance=distance)" + ] + }, + { + "cell_type": "markdown", + "id": "15961601", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Training\n", + "\n", + "Similarity models are trained like normal models. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "147a6863", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "EPOCHS = 10 # @param {type:\"integer\"}\n", + "history = model.fit(x_train, y_train, epochs=EPOCHS, validation_data=(x_test, y_test))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "88e1ee4d", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "plt.plot(history.history[\"loss\"])\n", + "plt.plot(history.history[\"val_loss\"])\n", + "plt.legend([\"loss\", \"val_loss\"])\n", + "plt.title(f\"Loss: {loss.name}\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5404906", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "5ad4ba20", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Prediction\n", + "\n", + "Let's predict some features and visualiza them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1936264", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "embedded_features = model.predict(x_test, verbose=1)\n", + "embedded_features /= np.linalg.norm(embedded_features, axis=1, keepdims=True)" + ] + }, + { + "cell_type": "markdown", + "id": "7c0df63b", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### 3D-Visualization of ArcFace Loss" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5aac5d98", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "fig = plt.figure()\n", + "ax = Axes3D(fig)\n", + "for c in range(len(np.unique(y_test))):\n", + " ax.plot(embedded_features[y_test==c, 0], embedded_features[y_test==c, 1], embedded_features[y_test==c, 2], '.', alpha=0.1)\n", + "plt.title('ArcFace')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8889f840", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fef529d9", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file From 882e8d37208f8cdd55a0db9fe601a25a9bb65761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aylin=20Ayd=C4=B1n?= Date: Wed, 21 Sep 2022 23:40:12 +0300 Subject: [PATCH 06/25] Add files via upload --- tests/test_losses.py | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/tests/test_losses.py b/tests/test_losses.py index 78181eab..1169423f 100644 --- a/tests/test_losses.py +++ b/tests/test_losses.py @@ -2,6 +2,7 @@ import tensorflow as tf from tensorflow_similarity.losses import ( + ArcFaceLoss, MultiSimilarityLoss, PNLoss, SoftNearestNeighborLoss, @@ -22,7 +23,6 @@ def test_triplet_loss_serialization(): def triplet_hard_loss_np(labels, embedding, margin, dist_func, soft=False): - num_data = embedding.shape[0] # Reshape labels to compute adjacency matrix. labels_reshaped = np.reshape(labels.astype(np.float32), (labels.shape[0], 1)) @@ -88,7 +88,9 @@ def test_triplet_loss_semi_hard(): y_true = tf.random.uniform((num_inputs,), 0, 3, dtype=tf.int32) # y_preds: embedding y_preds = tf.random.uniform((num_inputs, 16), 0, 1) - tpl = TripletLoss(positive_mining_strategy="easy", negative_mining_strategy="semi-hard") + tpl = TripletLoss( + positive_mining_strategy="easy", negative_mining_strategy="semi-hard" + ) # y_true, y_preds loss = tpl(y_true, y_preds) assert loss @@ -239,3 +241,35 @@ def test_xbm_loss(): loss_warm(labels2, embeddings2) assert loss_warm._y_pred_memory.numpy().shape == (batch_size, embed_dim) tf.assert_equal(loss_warm._y_true_memory, labels2) + + +# arcface loss +""" +ArcFaceLoss + ArcFace: Additive Angular Margin Loss for Deep Face + Recognition. [online] arXiv.org. Available at: + . +""" + + +def test_arcface_loss_serialization(): + n_classes = 10 + embed_size = 16 + loss = ArcFaceLoss(num_classes=n_classes, embedding_size=embed_size) + config = loss.get_config() + loss2 = ArcFaceLoss.from_config(config) + assert loss.name == loss2.name + assert loss.margin == loss2.margin + assert loss.scale == loss2.scale + assert loss.num_classes == loss2.num_classes + assert loss.embedding_size == loss2.embedding_size + + +def test_arcface_loss(): + tf.random.set_seed(128) + loss_fn = ArcFaceLoss(num_classes=4, embedding_size=5) + labels = tf.Variable([0, 1, 2, 3]) + embeddings = tf.Variable(tf.random.uniform(shape=[4, 5])) + loss = loss_fn(labels, embeddings) + + assert 60.4 < loss.numpy() < 60.5 From 9eb3a34db80c0720d2bcf92f5623ff640f002df5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aylin=20Ayd=C4=B1n?= Date: Thu, 22 Sep 2022 12:16:40 +0300 Subject: [PATCH 07/25] Delete test_losses.py --- test_losses.py | 275 ------------------------------------------------- 1 file changed, 275 deletions(-) delete mode 100644 test_losses.py diff --git a/test_losses.py b/test_losses.py deleted file mode 100644 index 1169423f..00000000 --- a/test_losses.py +++ /dev/null @@ -1,275 +0,0 @@ -import numpy as np -import tensorflow as tf - -from tensorflow_similarity.losses import ( - ArcFaceLoss, - MultiSimilarityLoss, - PNLoss, - SoftNearestNeighborLoss, - TripletLoss, -) - -# [triplet loss] -from tensorflow_similarity.losses.xbm_loss import XBM - - -def test_triplet_loss_serialization(): - loss = TripletLoss() - config = loss.get_config() - print(config) - loss2 = TripletLoss.from_config(config) - assert loss.name == loss2.name - assert loss.distance == loss2.distance - - -def triplet_hard_loss_np(labels, embedding, margin, dist_func, soft=False): - num_data = embedding.shape[0] - # Reshape labels to compute adjacency matrix. - labels_reshaped = np.reshape(labels.astype(np.float32), (labels.shape[0], 1)) - - adjacency = np.equal(labels_reshaped, labels_reshaped.T) - pdist_matrix = dist_func(embedding) - loss_np = 0.0 - for i in range(num_data): - pos_distances = [] - neg_distances = [] - for j in range(num_data): - if adjacency[i][j] == 0: - neg_distances.append(pdist_matrix[i][j]) - if adjacency[i][j] > 0.0 and i != j: - pos_distances.append(pdist_matrix[i][j]) - - # if there are no positive pairs, distance is 0 - if len(pos_distances) == 0: - pos_distances.append(0) - - # Sort by distance. - neg_distances.sort() - min_neg_distance = neg_distances[0] - pos_distances.sort(reverse=True) - max_pos_distance = pos_distances[0] - - if soft: - loss_np += np.log1p(np.exp(max_pos_distance - min_neg_distance)) - else: - loss_np += np.maximum(0.0, max_pos_distance - min_neg_distance + margin) - - loss_np /= num_data - return loss_np - - -def test_triplet_loss(): - num_inputs = 10 - # y_true: labels - y_true = tf.random.uniform((num_inputs,), 0, 10, dtype=tf.int32) - # y_preds: embedding - y_preds = tf.random.uniform((num_inputs, 20), 0, 1) - tpl = TripletLoss() - # y_true, y_preds - loss = tpl(y_true, y_preds) - assert loss > 0.9 - - -def test_triplet_loss_easy(): - num_inputs = 10 - # y_true: labels - y_true = tf.random.uniform((num_inputs,), 0, 3, dtype=tf.int32) - # y_preds: embedding - y_preds = tf.random.uniform((num_inputs, 16), 0, 1) - tpl = TripletLoss(positive_mining_strategy="easy", negative_mining_strategy="easy") - # y_true, y_preds - loss = tpl(y_true, y_preds) - assert loss > 0 - - -def test_triplet_loss_semi_hard(): - num_inputs = 10 - # y_true: labels - y_true = tf.random.uniform((num_inputs,), 0, 3, dtype=tf.int32) - # y_preds: embedding - y_preds = tf.random.uniform((num_inputs, 16), 0, 1) - tpl = TripletLoss( - positive_mining_strategy="easy", negative_mining_strategy="semi-hard" - ) - # y_true, y_preds - loss = tpl(y_true, y_preds) - assert loss - - -def test_triplet_loss_hard(): - num_inputs = 10 - # y_true: labels - y_true = tf.random.uniform((num_inputs,), 0, 3, dtype=tf.int32) - # y_preds: embedding - y_preds = tf.random.uniform((num_inputs, 16), 0, 1) - tpl = TripletLoss(positive_mining_strategy="hard", negative_mining_strategy="hard") - # y_true, y_preds - loss = tpl(y_true, y_preds) - assert loss - - -# [pn loss] -def test_pn_loss_serialization(): - loss = PNLoss() - config = loss.get_config() - print(config) - loss2 = PNLoss.from_config(config) - assert loss.name == loss2.name - assert loss.distance == loss2.distance - - -def test_np_loss(): - num_inputs = 10 - # y_true: labels - y_true = tf.random.uniform((num_inputs,), 0, 10, dtype=tf.int32) - # y_preds: embedding - y_preds = tf.random.uniform((num_inputs, 20), 0, 1) - pnl = PNLoss() - # y_true, y_preds - loss = pnl(y_true, y_preds) - assert loss > 0.9 - - -# [soft neasrest neighbor loss] -def test_softnn_loss_serialization(): - loss = SoftNearestNeighborLoss(distance="cosine", temperature=50) - config = loss.get_config() - loss2 = SoftNearestNeighborLoss.from_config(config) - assert loss.name == loss2.name - assert loss.distance == loss2.distance - assert loss.temperature == loss2.temperature - - -def softnn_util(y_true, x, temperature=1): - """ - A simple loop based implementation of soft - nearest neighbor loss to test the code. - https://arxiv.org/pdf/1902.01889.pdf - """ - - y_true = y_true.numpy() - x = x.numpy() - batch_size = y_true.shape[0] - loss = 0 - eps = 1e-9 - for i in range(batch_size): - numerator = 0 - denominator = 0 - for j in range(batch_size): - if i == j: - continue - if y_true[i] == y_true[j]: - numerator += np.exp(-1 * np.sum(np.square(x[i] - x[j])) / temperature) - denominator += np.exp(-1 * np.sum(np.square(x[i] - x[j])) / temperature) - if numerator == 0: - continue - loss += np.log(numerator / denominator) - return -loss / batch_size - - -def test_softnn_loss(): - num_inputs = 10 - n_classes = 10 - # y_true: labels - y_true = tf.random.uniform((num_inputs,), 0, n_classes, dtype=tf.int32) - # x: embeddings - x = tf.random.uniform((num_inputs, 20), 0, 1) - - temperature = np.random.uniform(0.1, 50) - softnn = SoftNearestNeighborLoss(temperature=temperature) - loss = softnn(y_true, x) - loss_check = softnn_util(y_true, x, temperature) - loss_diff = loss.numpy() - loss_check - assert np.abs(loss_diff) < 1e-3 - - -def test_xbm_loss(): - batch_size = 6 - embed_dim = 16 - - embeddings1 = tf.random.uniform(shape=[batch_size, embed_dim]) - labels1 = tf.constant( - [ - [1], - [1], - [2], - [2], - [3], - [3], - ], - dtype=tf.int32, - ) - - embeddings2 = tf.random.uniform(shape=[batch_size, embed_dim]) - labels2 = tf.constant( - [ - [4], - [4], - [5], - [5], - [6], - [6], - ], - dtype=tf.int32, - ) - - distance = "cosine" - loss = MultiSimilarityLoss(distance=distance) - loss_nowarm = XBM(loss, memory_size=12, warmup_steps=0) - - # test enqueue - loss_nowarm(labels1, embeddings1) - assert loss_nowarm._y_pred_memory.numpy().shape == (batch_size, embed_dim) - tf.assert_equal(loss_nowarm._y_true_memory, labels1) - - loss_nowarm(labels2, embeddings2) - assert loss_nowarm._y_pred_memory.numpy().shape == (2 * batch_size, embed_dim) - tf.assert_equal(loss_nowarm._y_true_memory, tf.concat([labels2, labels1], axis=0)) - - # test dequeue - loss_nowarm(labels2, embeddings2) - assert loss_nowarm._y_pred_memory.numpy().shape == (2 * batch_size, embed_dim) - tf.assert_equal(loss_nowarm._y_true_memory, tf.concat([labels2, labels2], axis=0)) - - # test warmup - loss_warm = XBM(loss, memory_size=12, warmup_steps=1) - - loss_warm(labels1, embeddings1) - assert loss_warm._y_pred_memory.numpy().shape == (0, embed_dim) - tf.assert_equal(loss_warm._y_true_memory, tf.constant([[]], dtype=tf.int32)) - - loss_warm(labels2, embeddings2) - assert loss_warm._y_pred_memory.numpy().shape == (batch_size, embed_dim) - tf.assert_equal(loss_warm._y_true_memory, labels2) - - -# arcface loss -""" -ArcFaceLoss - ArcFace: Additive Angular Margin Loss for Deep Face - Recognition. [online] arXiv.org. Available at: - . -""" - - -def test_arcface_loss_serialization(): - n_classes = 10 - embed_size = 16 - loss = ArcFaceLoss(num_classes=n_classes, embedding_size=embed_size) - config = loss.get_config() - loss2 = ArcFaceLoss.from_config(config) - assert loss.name == loss2.name - assert loss.margin == loss2.margin - assert loss.scale == loss2.scale - assert loss.num_classes == loss2.num_classes - assert loss.embedding_size == loss2.embedding_size - - -def test_arcface_loss(): - tf.random.set_seed(128) - loss_fn = ArcFaceLoss(num_classes=4, embedding_size=5) - labels = tf.Variable([0, 1, 2, 3]) - embeddings = tf.Variable(tf.random.uniform(shape=[4, 5])) - loss = loss_fn(labels, embeddings) - - assert 60.4 < loss.numpy() < 60.5 From 8db6bc9a171dd20de260ac0f511653f0ff74ddd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aylin=20Ayd=C4=B1n?= Date: Thu, 22 Sep 2022 12:16:52 +0300 Subject: [PATCH 08/25] Delete arcface_loss.py --- arcface_loss.py | 120 ------------------------------------------------ 1 file changed, 120 deletions(-) delete mode 100644 arcface_loss.py diff --git a/arcface_loss.py b/arcface_loss.py deleted file mode 100644 index e93a98fd..00000000 --- a/arcface_loss.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2022 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""ArcFace losses base class. - -ArcFace: Additive Angular Margin Loss for Deep Face -Recognition. [online] arXiv.org. Available at: -. - -""" - -from typing import Any, Callable, Dict, Optional, Tuple, Union - -import tensorflow as tf - -from tensorflow_similarity.algebra import build_masks -from tensorflow_similarity.distances import Distance, distance_canonicalizer -from tensorflow_similarity.types import FloatTensor, IntTensor -from tensorflow_similarity.utils import is_tensor_or_variable - -from .metric_loss import MetricLoss -from .utils import logsumexp - - -@tf.keras.utils.register_keras_serializable(package="Similarity") -class ArcFaceLoss(tf.keras.losses.Loss): - """Implement of ArcFace: Additive Angular Margin Loss: - Step 1: Create a trainable kernel matrix with the shape of [embedding_size, num_classes]. - Step 2: Normalize the kernel and prediction vectors. - Step 3: Calculate the cosine similarity between the normalized prediction vector and the kernel. - Step 4: Create a one-hot vector include the margin value for the ground truth class. - Step 5: Add margin_hot to the cosine similarity and multiply it by scale. - Step 6: Calculate the cross-entropy loss. - - ArcFace: Additive Angular Margin Loss for Deep Face - Recognition. [online] arXiv.org. Available at: - . - - Standalone usage: - >>> loss_fn = tfsim.losses.ArcFaceLoss(num_classes=2, embedding_size=3) - >>> labels = tf.Variable([1, 0]) - >>> embeddings = tf.Variable([[0.2, 0.3, 0.1], [0.4, 0.5, 0.5]]) - >>> loss = loss_fn(labels, embeddings) - Args: - num_classes: Number of classes. - embedding_size: The size of the embedding vectors. - margin: The margin value. - scale: s in the paper, feature scale - name: Optional name for the operation. - reduction: Type of loss reduction to apply to the loss. - """ - - def __init__( - self, - num_classes: int, - embedding_size: int, - margin: float = 0.50, # margin in radians - scale: float = 64.0, # feature scale - name: Optional[str] = None, - reduction: Callable = tf.keras.losses.Reduction.AUTO, - **kwargs - ): - - super().__init__(reduction=reduction, name=name, **kwargs) - - self.num_classes = num_classes - self.embedding_size = embedding_size - self.margin = margin - self.scale = scale - self.name = name - self.kernel = tf.Variable(tf.random.normal([embedding_size, num_classes])) - - def call(self, y_true: FloatTensor, y_pred: FloatTensor) -> FloatTensor: - - y_pred_norm = tf.math.l2_normalize(y_pred, axis=1) - kernel_norm = tf.math.l2_normalize(self.kernel, axis=0) - - cos_theta = tf.matmul(y_pred_norm, kernel_norm) - cos_theta = tf.clip_by_value(cos_theta, -1.0, 1.0) - - m_hot = tf.one_hot(y_true, self.num_classes, on_value=self.margin, axis=1) - m_hot = tf.reshape(m_hot, [-1, self.num_classes]) - - cos_theta = tf.acos(cos_theta) - cos_theta += m_hot - cos_theta = tf.math.cos(cos_theta) - cos_theta = tf.math.multiply(cos_theta, self.scale) - - cce = tf.keras.losses.SparseCategoricalCrossentropy( - from_logits=True, reduction=self.reduction - ) - loss: FloatTensor = cce(y_true, cos_theta) - - return loss - - def get_config(self) -> Dict[str, Any]: - """Contains the loss configuration. - Returns: - A Python dict containing the configuration of the loss. - """ - config = { - "num_classes": self.num_classes, - "embedding_size": self.embedding_size, - "margin": self.margin, - "scale": self.scale, - "name": self.name, - } - base_config = super().get_config() - return {**base_config, **config} From d6c1d344fceea0e0494fb45a1afa64b6340c416a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aylin=20Ayd=C4=B1n?= Date: Thu, 22 Sep 2022 12:25:38 +0300 Subject: [PATCH 09/25] Add files via upload --- tensorflow_similarity/losses/arcface_loss.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tensorflow_similarity/losses/arcface_loss.py b/tensorflow_similarity/losses/arcface_loss.py index e0252d5c..e20bf06e 100644 --- a/tensorflow_similarity/losses/arcface_loss.py +++ b/tensorflow_similarity/losses/arcface_loss.py @@ -13,11 +13,9 @@ # limitations under the License. # ============================================================================== """ArcFace losses base class. - ArcFace: Additive Angular Margin Loss for Deep Face Recognition. [online] arXiv.org. Available at: . - """ from typing import Any, Callable, Dict, Optional, Tuple, Union @@ -41,11 +39,9 @@ class ArcFaceLoss(tf.keras.losses.Loss): Step 4: Create a one-hot vector include the margin value for the ground truth class. Step 5: Add margin_hot to the cosine similarity and multiply it by scale. Step 6: Calculate the cross-entropy loss. - ArcFace: Additive Angular Margin Loss for Deep Face Recognition. [online] arXiv.org. Available at: . - Standalone usage: >>> loss_fn = tfsim.losses.ArcFaceLoss(num_classes=2, embedding_size=3) >>> labels = tf.Variable([1, 0]) @@ -106,7 +102,7 @@ def call(self, y_true: FloatTensor, y_pred: FloatTensor) -> FloatTensor: def get_config(self) -> Dict[str, Any]: """Contains the loss configuration. Returns: - A Python dict containing the configuration of the loss. + The configuration of the ArcFace loss. """ config = { "num_classes": self.num_classes, From 0274c3d654ffc4f5354621e5e0f886d05d849991 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aylin=20Ayd=C4=B1n?= Date: Thu, 22 Sep 2022 12:26:51 +0300 Subject: [PATCH 10/25] Add files via upload --- tests/test_losses.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/test_losses.py b/tests/test_losses.py index 1169423f..ce958cb5 100644 --- a/tests/test_losses.py +++ b/tests/test_losses.py @@ -1,14 +1,8 @@ import numpy as np import tensorflow as tf - -from tensorflow_similarity.losses import ( - ArcFaceLoss, - MultiSimilarityLoss, - PNLoss, - SoftNearestNeighborLoss, - TripletLoss, -) - +from tensorflow_similarity.losses import (ArcFaceLoss, MultiSimilarityLoss, + PNLoss, SoftNearestNeighborLoss, + TripletLoss) # [triplet loss] from tensorflow_similarity.losses.xbm_loss import XBM @@ -270,6 +264,9 @@ def test_arcface_loss(): loss_fn = ArcFaceLoss(num_classes=4, embedding_size=5) labels = tf.Variable([0, 1, 2, 3]) embeddings = tf.Variable(tf.random.uniform(shape=[4, 5])) + print(embeddings) + loss = loss_fn(labels, embeddings) + print(loss) assert 60.4 < loss.numpy() < 60.5 From bc108c45156b142ef71320c039aacee055ffd8c8 Mon Sep 17 00:00:00 2001 From: Abhishar Sinha <24841841+abhisharsinha@users.noreply.github.com> Date: Tue, 20 Sep 2022 23:49:16 +0530 Subject: [PATCH 11/25] Fixed typo to resolve #284 The function should be tf.concat instead of tf.constant, according to the description given above. This also resolves issue #284 --- tensorflow_similarity/classification_metrics/precision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_similarity/classification_metrics/precision.py b/tensorflow_similarity/classification_metrics/precision.py index 4569cb61..4938d74f 100644 --- a/tensorflow_similarity/classification_metrics/precision.py +++ b/tensorflow_similarity/classification_metrics/precision.py @@ -84,7 +84,7 @@ def compute( # The following accounts for the and sets the first precision value to # 1.0 if the first recall and precision are both zero. if (tp + fp)[0] == 0.0 and len(p) > 1: - initial_precision = tf.constant([tf.constant([1.0]), tf.zeros(len(p) - 1)], axis=0) + initial_precision = tf.concat([tf.constant([1.0]), tf.zeros(len(p) - 1)], axis=0) p = p + initial_precision return p From 7cbbdcf8841c0e48c5602ec4c34064e86f87223f Mon Sep 17 00:00:00 2001 From: Owen Vallis Date: Fri, 23 Sep 2022 20:13:00 +0000 Subject: [PATCH 12/25] Patch bump to 0.16.8 --- tensorflow_similarity/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_similarity/__init__.py b/tensorflow_similarity/__init__.py index 863c4b00..d8fbc9c2 100644 --- a/tensorflow_similarity/__init__.py +++ b/tensorflow_similarity/__init__.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.16.7" +__version__ = "0.16.8" from . import algebra # noqa From 2ff3b7291d71f031621276a536794f6d5e05e9ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aylin=20Ayd=C4=B1n?= Date: Wed, 28 Sep 2022 10:31:19 +0300 Subject: [PATCH 13/25] Add files via upload --- tests/test_losses.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_losses.py b/tests/test_losses.py index ce958cb5..3a96a32f 100644 --- a/tests/test_losses.py +++ b/tests/test_losses.py @@ -1,5 +1,6 @@ import numpy as np import tensorflow as tf + from tensorflow_similarity.losses import (ArcFaceLoss, MultiSimilarityLoss, PNLoss, SoftNearestNeighborLoss, TripletLoss) From 17f7051eeafdd209f112202727c210f5fcfb9c28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aylin=20Ayd=C4=B1n?= Date: Wed, 28 Sep 2022 10:31:38 +0300 Subject: [PATCH 14/25] Add files via upload --- tensorflow_similarity/losses/arcface_loss.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tensorflow_similarity/losses/arcface_loss.py b/tensorflow_similarity/losses/arcface_loss.py index e20bf06e..9ac55c60 100644 --- a/tensorflow_similarity/losses/arcface_loss.py +++ b/tensorflow_similarity/losses/arcface_loss.py @@ -18,16 +18,12 @@ . """ -from typing import Any, Callable, Dict, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional import tensorflow as tf -from tensorflow_similarity.algebra import build_masks -from tensorflow_similarity.distances import Distance, distance_canonicalizer -from tensorflow_similarity.types import FloatTensor, IntTensor -from tensorflow_similarity.utils import is_tensor_or_variable -from .metric_loss import MetricLoss -from .utils import logsumexp +from tensorflow_similarity.distances import Distance +from tensorflow_similarity.types import FloatTensor @tf.keras.utils.register_keras_serializable(package="Similarity") @@ -74,7 +70,9 @@ def __init__( self.margin = margin self.scale = scale self.name = name - self.kernel = tf.Variable(tf.random.normal([embedding_size, num_classes])) + self.kernel = tf.Variable( + tf.random.normal([embedding_size, num_classes]), trainable=True + ) def call(self, y_true: FloatTensor, y_pred: FloatTensor) -> FloatTensor: From f7de1d827de96d9af6dac860e52283b696b6725a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aylin=20Ayd=C4=B1n?= Date: Thu, 6 Oct 2022 21:25:32 +0300 Subject: [PATCH 15/25] Add files via upload --- tensorflow_similarity/losses/__init__.py | 11 +++++------ tensorflow_similarity/losses/arcface_loss.py | 1 - 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tensorflow_similarity/losses/__init__.py b/tensorflow_similarity/losses/__init__.py index 8829e09f..0fdb573d 100644 --- a/tensorflow_similarity/losses/__init__.py +++ b/tensorflow_similarity/losses/__init__.py @@ -15,14 +15,13 @@ """ Contrastive learning specialized losses. """ -from .pn_loss import PNLoss # noqa -from .triplet_loss import TripletLoss # noqa -from .metric_loss import MetricLoss # noqa +from .arcface_loss import ArcFaceLoss # noqa from .circle_loss import CircleLoss # noqa +from .metric_loss import MetricLoss # noqa from .multisim_loss import MultiSimilarityLoss # noqa -from .simsiam import SimSiamLoss # noqa +from .pn_loss import PNLoss # noqa from .simclr import SimCLRLoss # noqa +from .simsiam import SimSiamLoss # noqa +from .triplet_loss import TripletLoss # noqa from .vicreg import VicReg # noqa -from .arcface_loss import ArcFaceLoss # noqa from .xbm_loss import XBM # noqa - diff --git a/tensorflow_similarity/losses/arcface_loss.py b/tensorflow_similarity/losses/arcface_loss.py index 9ac55c60..bf4c4842 100644 --- a/tensorflow_similarity/losses/arcface_loss.py +++ b/tensorflow_similarity/losses/arcface_loss.py @@ -21,7 +21,6 @@ from typing import Any, Callable, Dict, Optional import tensorflow as tf - from tensorflow_similarity.distances import Distance from tensorflow_similarity.types import FloatTensor From 455dc3566b8e3d4cad371d18ea76b5b98e63a8ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aylin=20Ayd=C4=B1n?= Date: Thu, 6 Oct 2022 21:25:55 +0300 Subject: [PATCH 16/25] Add files via upload --- tests/test_losses.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_losses.py b/tests/test_losses.py index 3a96a32f..ff149c94 100644 --- a/tests/test_losses.py +++ b/tests/test_losses.py @@ -1,9 +1,13 @@ import numpy as np import tensorflow as tf +from tensorflow_similarity.losses import ( + ArcFaceLoss, + MultiSimilarityLoss, + PNLoss, + SoftNearestNeighborLoss, + TripletLoss, +) -from tensorflow_similarity.losses import (ArcFaceLoss, MultiSimilarityLoss, - PNLoss, SoftNearestNeighborLoss, - TripletLoss) # [triplet loss] from tensorflow_similarity.losses.xbm_loss import XBM From d67630c61afdc7925ce080e72467336a0d566cd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aylin=20Ayd=C4=B1n?= Date: Fri, 7 Oct 2022 10:12:52 +0300 Subject: [PATCH 17/25] Add files via upload From d8c99f5f19bb83719a5fa1fcc074993947e78269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aylin=20Ayd=C4=B1n?= Date: Fri, 7 Oct 2022 10:13:10 +0300 Subject: [PATCH 18/25] Add files via upload --- tensorflow_similarity/__init__.py | 37 ++++++++++++------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/tensorflow_similarity/__init__.py b/tensorflow_similarity/__init__.py index 00573fe4..7ccee62d 100644 --- a/tensorflow_similarity/__init__.py +++ b/tensorflow_similarity/__init__.py @@ -11,27 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.17.0.dev10" - -from . import algebra # noqa -from . import architectures # noqa -from . import augmenters # noqa -from . import callbacks # noqa -from . import classification_metrics # noqa -from . import distances # noqa -from . import evaluators # noqa -from . import indexer # noqa -from . import layers # noqa -from . import losses # noqa -from . import matchers # noqa -from . import models # noqa -from . import retrieval_metrics # noqa -from . import samplers # noqa -from . import schedules # noqa -from . import search # noqa -from . import stores # noqa -from . import training_metrics # noqa -from . import types # noqa -from . import utils # noqa -from . import visualization # noqa +""" +Contrastive learning specialized losses. +""" +from .arcface_loss import ArcFaceLoss # noqa +from .barlow import Barlow # noqa +from .circle_loss import CircleLoss # noqa +from .metric_loss import MetricLoss # noqa +from .multisim_loss import MultiSimilarityLoss # noqa +from .pn_loss import PNLoss # noqa +from .simclr import SimCLRLoss # noqa +from .simsiam import SimSiamLoss # noqa +from .softnn_loss import SoftNearestNeighborLoss # noqa +from .triplet_loss import TripletLoss # noqa +from .vicreg import VicReg # noqa From 1980579912566bd0efbd2ba980b26f58cbd1fcf7 Mon Sep 17 00:00:00 2001 From: Owen Vallis Date: Fri, 7 Oct 2022 09:35:50 -0700 Subject: [PATCH 19/25] Delete ArcFace Loss Sample Notebook.ipynb --- ArcFace Loss Sample Notebook.ipynb | 775 ----------------------------- 1 file changed, 775 deletions(-) delete mode 100644 ArcFace Loss Sample Notebook.ipynb diff --git a/ArcFace Loss Sample Notebook.ipynb b/ArcFace Loss Sample Notebook.ipynb deleted file mode 100644 index b481492e..00000000 --- a/ArcFace Loss Sample Notebook.ipynb +++ /dev/null @@ -1,775 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "28956aa1", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "Copyright 2022 The TensorFlow Similarity Authors." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24eda1a6", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "# @title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "id": "7ca9d025", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "# TensorFlow Similarity ArcFace Loss Example" - ] - }, - { - "cell_type": "markdown", - "id": "d072628f", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "A Total Angular Margin Loss (ArcFace) calculates the geodetic distance in the hypersphere instead of the euclidean distance to improve the discriminatory strength of the facial recognition model and stabilize the training process. Rails are used to measure all distances in geodetic space. The geodetic trace is the path taken between two places. It specifies the geodetic distance, which is the shortest distance between two places.\n", - "\n", - "ArcFace loss determines the angle between the current feature and the target weight using the arc-cosine function since the dot product between the DCNN feature and the last fully connected layer after feature and weight normalization matches the cosine distance. The target logit is then returned by multiplying the goal angle by an additional angular margin and using the cosine function. After that, we continue as before and rescale all logits to a certain feature norm, just like with softmax loss." - ] - }, - { - "cell_type": "markdown", - "id": "808ac087", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Notebook goal\n", - "\n", - "This notebook demonstrates how to use ArcFaceLoss implementation of TensorFlow Similarity with standalone usage and to train a `SimilarityModel()` on a fraction of the MNIST classes.\n", - "\n", - "You are going to learn about the main features offered by the `ArcFaceLoss()` and will:\n", - "\n", - " 1. Standalone usage of ArcFaceLoss\n", - "\n", - " 2. Usage with `model.compile()`\n", - "\n", - " 3. 3D-Visualization of ArcFaceLoss \n", - "\n", - "### Things to try \n", - "\n", - "Along the way you can try the following things to improve the model performance:\n", - "- Adding more \"seen\" classes at training time.\n", - "- Use a larger embedding by increasing the size of the output.\n", - "- Add data augmentation pre-processing layers to the model.\n", - "- Include more examples in the index to give the models more points to choose from.\n", - "- Try a more challenging dataset, such as Fashion MNIST." - ] - }, - { - "cell_type": "markdown", - "id": "078c53c0", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Notebook goal\n", - "\n", - "This notebook demonstrates how to use ArcFaceLoss implementation of TensorFlow Similarity with standalone usage and to train a `SimilarityModel()` on a fraction of the MNIST classes.\n", - "\n", - "You are going to learn about the main features offered by the `ArcFaceLoss()` and will:\n", - "\n", - " 1. Standalone usage of ArcFaceLoss\n", - "\n", - " 2. Usage with `model.compile()`\n", - "\n", - " 3. 3D-Visualization of ArcFaceLoss \n", - "\n", - "### Things to try \n", - "\n", - "Along the way you can try the following things to improve the model performance:\n", - "- Adding more \"seen\" classes at training time.\n", - "- Use a larger embedding by increasing the size of the output.\n", - "- Add data augmentation pre-processing layers to the model.\n", - "- Include more examples in the index to give the models more points to choose from.\n", - "- Try a more challenging dataset, such as Fashion MNIST." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8fd63f16", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "import gc\n", - "import os\n", - "\n", - "import numpy as np\n", - "from matplotlib import pyplot as plt\n", - "from tabulate import tabulate\n", - "from mpl_toolkits.mplot3d import Axes3D\n", - "\n", - "# INFO messages are not printed.\n", - "# This must be run before loading other modules.\n", - "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"1\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "80af5fc0", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "import tensorflow as tf" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8ba8caf7", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "# install TF similarity if needed\n", - "try:\n", - " import tensorflow_similarity as tfsim # main package\n", - "except ModuleNotFoundError:\n", - " !pip install tensorflow_similarity\n", - " import tensorflow_similarity as tfsim" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2484bd72", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "tfsim.utils.tf_cap_memory()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3fe0344e", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "# Clear out any old model state.\n", - "gc.collect()\n", - "tf.keras.backend.clear_session()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "99d9bef9", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "print(\"TensorFlow:\", tf.__version__)\n", - "print(\"TensorFlow Similarity\", tfsim.__version__)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7137afbc", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "1d534ad3", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "# Standalone Usage of ArcFaceLoss\n", - "\n", - "ArcFace loss alone can be used as follows when it is desired to calculate the additive angular margin loss of the existing data set." - ] - }, - { - "cell_type": "markdown", - "id": "68d526da", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Initialize Loss function as ArcFaceLoss" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bebf6ef0", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "loss_fn = tfsim.losses.ArcFaceLoss(num_classes=8, embedding_size=10)" - ] - }, - { - "cell_type": "markdown", - "id": "d2ccfd7d", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Create own simple random dataset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1d1ec43a", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "labels = tf.Variable([0, 1, 2, 3, 4, 5, 6, 7])\n", - "embeddings = tf.Variable(tf.random.uniform(shape=[8, 10]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "73d0c1c6", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "print(\"\", embeddings)" - ] - }, - { - "cell_type": "markdown", - "id": "d65b3085", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Calculate loss" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cdf7c30c", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "loss = loss_fn(labels, embeddings)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "16745b7d", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "print(\"loss : \" , loss)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "loss = loss_fn(labels, embeddings)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "print(\"loss : \" , loss)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "markdown", - "source": [ - "## Data preparation\n", - "\n", - "We are going to load the MNIST dataset to showcase how the model is able to find similar examples from classes unseen during training. The model's ability to generalize the matching to unseen classes, without retraining, is one of the main reason you would want to use metric learning.\n", - "\n", - "\n", - "**WARNING**: Tensorflow similarity expects `y_train` to be an IntTensor containing the class ids for each example instead of the standard categorical encoding traditionally used for multi-class classification." - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8a9f8122", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"loss : \" , loss)" - ] - }, - { - "cell_type": "markdown", - "id": "11ef5236", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "## Data preparation\n", - "\n", - "We are going to load the MNIST dataset to showcase how the model is able to find similar examples from classes unseen during training. The model's ability to generalize the matching to unseen classes, without retraining, is one of the main reason you would want to use metric learning.\n", - "\n", - "\n", - "**WARNING**: Tensorflow similarity expects `y_train` to be an IntTensor containing the class ids for each example instead of the standard categorical encoding traditionally used for multi-class classification." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "97152229", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()" - ] - }, - { - "cell_type": "markdown", - "id": "08b766d8", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "## Model setup" - ] - }, - { - "cell_type": "markdown", - "id": "3eac2da7", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Model definition\n", - "\n", - "`SimilarityModel()` models extend `tensorflow.keras.model.Model` with additional features and functionality that allow you to index and search for similar looking examples.\n", - "\n", - "As visible in the model definition below, similarity models output a 64 dimensional float embedding using the `MetricEmbedding()` layers. This layer is a Dense layer with L2 normalization. Thanks to the loss, the model learns to minimize the distance between similar examples and maximize the distance between dissimilar examples. As a result, the distance between examples in the embedding space is meaningful; the smaller the distance the more similar the examples are. \n", - "\n", - "Being able to use a distance as a meaningful proxy for how similar two examples are, is what enables the fast ANN (aproximate nearest neighbor) search. Using a sub-linear ANN search instead of a standard quadratic NN search is what allows deep similarity search to scale to millions of items. The built in memory index used in this notebook scales to a million indexed examples very easily... if you have enough RAM :)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a003c971", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "def get_model():\n", - " inputs = tf.keras.layers.Input(shape=(28, 28, 1))\n", - " x = tf.keras.layers.experimental.preprocessing.Rescaling(1 / 255)(inputs)\n", - " x = tf.keras.layers.Conv2D(32, 3, activation=\"relu\")(x)\n", - " x = tf.keras.layers.Conv2D(32, 3, activation=\"relu\")(x)\n", - " x = tf.keras.layers.MaxPool2D()(x)\n", - " x = tf.keras.layers.Conv2D(64, 3, activation=\"relu\")(x)\n", - " x = tf.keras.layers.Conv2D(64, 3, activation=\"relu\")(x)\n", - " x = tf.keras.layers.Flatten()(x)\n", - " # smaller embeddings will have faster lookup times while a larger embedding will improve the accuracy up to a point.\n", - " outputs = tfsim.layers.MetricEmbedding(64)(x)\n", - " return tfsim.models.SimilarityModel(inputs, outputs)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a2177b12", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "model = get_model()\n", - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "id": "defb3961", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### ArcFace Loss definition\n", - "\n", - "Overall what makes Metric losses different from tradional losses is that:\n", - "- **They expect different inputs.** Instead of having the prediction equal the true values, they expect embeddings as `y_preds` and the id (as an int32) of the class as `y_true`. \n", - "- **They require a distance.** You need to specify which `distance` function to use to compute the distance between embeddings. `cosine` is usually a great starting point and the default.\n", - "\n", - "ArcFace Loss takes inputs as number of classes which labels includes, and embedding size which we define in model `MetricEmbedding()` layers." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "13b0d745", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "distance = \"cosine\" " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c22d10cc", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "num_classes = np.unique(y_train).size\n", - "embedding_size = model.get_layer('metric_embedding').output.shape[1]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d5b8e426", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "loss = tfsim.losses.ArcFaceLoss(num_classes= num_classes, embedding_size=embedding_size, name=\"ArcFaceLoss\")" - ] - }, - { - "cell_type": "markdown", - "id": "b6eaf9c8", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Compilation\n", - "\n", - "Tensorflow similarity use an extended `compile()` method that allows you to optionally specify `distance_metrics` (metrics that are computed over the distance between the embeddings), and the distance to use for the indexer.\n", - "\n", - "By default the `compile()` method tries to infer what type of distance you are using by looking at the first loss specified. If you use multiple losses, and the distance loss is not the first one, then you need to specify the distance function used as `distance=` parameter in the compile function." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "673f986f", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "LR = 0.0005 # @param {type:\"number\"}\n", - "model.compile(optimizer=tf.keras.optimizers.SGD(LR), loss=loss, distance=distance)" - ] - }, - { - "cell_type": "markdown", - "id": "15961601", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "## Training\n", - "\n", - "Similarity models are trained like normal models. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "147a6863", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "EPOCHS = 10 # @param {type:\"integer\"}\n", - "history = model.fit(x_train, y_train, epochs=EPOCHS, validation_data=(x_test, y_test))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "88e1ee4d", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "plt.plot(history.history[\"loss\"])\n", - "plt.plot(history.history[\"val_loss\"])\n", - "plt.legend([\"loss\", \"val_loss\"])\n", - "plt.title(f\"Loss: {loss.name}\")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a5404906", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "5ad4ba20", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "## Prediction\n", - "\n", - "Let's predict some features and visualiza them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a1936264", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "embedded_features = model.predict(x_test, verbose=1)\n", - "embedded_features /= np.linalg.norm(embedded_features, axis=1, keepdims=True)" - ] - }, - { - "cell_type": "markdown", - "id": "7c0df63b", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### 3D-Visualization of ArcFace Loss" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5aac5d98", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "fig = plt.figure()\n", - "ax = Axes3D(fig)\n", - "for c in range(len(np.unique(y_test))):\n", - " ax.plot(embedded_features[y_test==c, 0], embedded_features[y_test==c, 1], embedded_features[y_test==c, 2], '.', alpha=0.1)\n", - "plt.title('ArcFace')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8889f840", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fef529d9", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file From 347a7e9f9acfe39a7770d10c644fa679c164a1ee Mon Sep 17 00:00:00 2001 From: Owen Vallis Date: Fri, 7 Oct 2022 09:37:34 -0700 Subject: [PATCH 20/25] Update __init__.py --- tensorflow_similarity/__init__.py | 38 +++++++++++++++++++------------ 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/tensorflow_similarity/__init__.py b/tensorflow_similarity/__init__.py index 7ccee62d..15eaae2d 100644 --- a/tensorflow_similarity/__init__.py +++ b/tensorflow_similarity/__init__.py @@ -12,17 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -Contrastive learning specialized losses. -""" -from .arcface_loss import ArcFaceLoss # noqa -from .barlow import Barlow # noqa -from .circle_loss import CircleLoss # noqa -from .metric_loss import MetricLoss # noqa -from .multisim_loss import MultiSimilarityLoss # noqa -from .pn_loss import PNLoss # noqa -from .simclr import SimCLRLoss # noqa -from .simsiam import SimSiamLoss # noqa -from .softnn_loss import SoftNearestNeighborLoss # noqa -from .triplet_loss import TripletLoss # noqa -from .vicreg import VicReg # noqa +__version__ = "0.17.0.dev10" + + +from . import algebra # noqa +from . import architectures # noqa +from . import augmenters # noqa +from . import callbacks # noqa +from . import classification_metrics # noqa +from . import distances # noqa +from . import evaluators # noqa +from . import indexer # noqa +from . import layers # noqa +from . import losses # noqa +from . import matchers # noqa +from . import models # noqa +from . import retrieval_metrics # noqa +from . import samplers # noqa +from . import schedules # noqa +from . import search # noqa +from . import stores # noqa +from . import training_metrics # noqa +from . import types # noqa +from . import utils # noqa +from . import visualization # noqa From 140c837c1970ce4c7ea4212c7bf9c52e106e4ddb Mon Sep 17 00:00:00 2001 From: Owen Vallis Date: Fri, 7 Oct 2022 09:37:54 -0700 Subject: [PATCH 21/25] Update __init__.py --- tensorflow_similarity/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow_similarity/__init__.py b/tensorflow_similarity/__init__.py index 15eaae2d..00573fe4 100644 --- a/tensorflow_similarity/__init__.py +++ b/tensorflow_similarity/__init__.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - __version__ = "0.17.0.dev10" From 5a80c3260422a76c745bd074e2cdf9a43442cc52 Mon Sep 17 00:00:00 2001 From: Owen Vallis Date: Fri, 7 Oct 2022 09:38:34 -0700 Subject: [PATCH 22/25] Update __init__.py --- tensorflow_similarity/losses/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow_similarity/losses/__init__.py b/tensorflow_similarity/losses/__init__.py index 0fdb573d..1a3d82ff 100644 --- a/tensorflow_similarity/losses/__init__.py +++ b/tensorflow_similarity/losses/__init__.py @@ -16,12 +16,14 @@ Contrastive learning specialized losses. """ from .arcface_loss import ArcFaceLoss # noqa +from .barlow import Barlow # noqa from .circle_loss import CircleLoss # noqa from .metric_loss import MetricLoss # noqa from .multisim_loss import MultiSimilarityLoss # noqa from .pn_loss import PNLoss # noqa from .simclr import SimCLRLoss # noqa from .simsiam import SimSiamLoss # noqa +from .softnn_loss import SoftNearestNeighborLoss # noqa from .triplet_loss import TripletLoss # noqa from .vicreg import VicReg # noqa from .xbm_loss import XBM # noqa From ffc325ce8ef70a193eb6599d46f6eba933e5da5d Mon Sep 17 00:00:00 2001 From: Owen Vallis Date: Wed, 12 Oct 2022 05:13:38 +0000 Subject: [PATCH 23/25] Fix formatting errors. --- tensorflow_similarity/losses/arcface_loss.py | 212 +++++++++---------- tests/test_losses.py | 5 +- 2 files changed, 106 insertions(+), 111 deletions(-) diff --git a/tensorflow_similarity/losses/arcface_loss.py b/tensorflow_similarity/losses/arcface_loss.py index bf4c4842..2362bba3 100644 --- a/tensorflow_similarity/losses/arcface_loss.py +++ b/tensorflow_similarity/losses/arcface_loss.py @@ -1,112 +1,108 @@ -# Copyright 2022 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== +# Copyright 2022 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== """ArcFace losses base class. ArcFace: Additive Angular Margin Loss for Deep Face Recognition. [online] arXiv.org. Available at: . -""" - -from typing import Any, Callable, Dict, Optional - -import tensorflow as tf -from tensorflow_similarity.distances import Distance -from tensorflow_similarity.types import FloatTensor - - -@tf.keras.utils.register_keras_serializable(package="Similarity") -class ArcFaceLoss(tf.keras.losses.Loss): - """Implement of ArcFace: Additive Angular Margin Loss: - Step 1: Create a trainable kernel matrix with the shape of [embedding_size, num_classes]. - Step 2: Normalize the kernel and prediction vectors. - Step 3: Calculate the cosine similarity between the normalized prediction vector and the kernel. - Step 4: Create a one-hot vector include the margin value for the ground truth class. - Step 5: Add margin_hot to the cosine similarity and multiply it by scale. - Step 6: Calculate the cross-entropy loss. - ArcFace: Additive Angular Margin Loss for Deep Face - Recognition. [online] arXiv.org. Available at: - . - Standalone usage: - >>> loss_fn = tfsim.losses.ArcFaceLoss(num_classes=2, embedding_size=3) - >>> labels = tf.Variable([1, 0]) - >>> embeddings = tf.Variable([[0.2, 0.3, 0.1], [0.4, 0.5, 0.5]]) - >>> loss = loss_fn(labels, embeddings) - Args: - num_classes: Number of classes. - embedding_size: The size of the embedding vectors. - margin: The margin value. - scale: s in the paper, feature scale - name: Optional name for the operation. - reduction: Type of loss reduction to apply to the loss. - """ - - def __init__( - self, - num_classes: int, - embedding_size: int, - margin: float = 0.50, # margin in radians - scale: float = 64.0, # feature scale - name: Optional[str] = None, - reduction: Callable = tf.keras.losses.Reduction.AUTO, - **kwargs - ): - - super().__init__(reduction=reduction, name=name, **kwargs) - - self.num_classes = num_classes - self.embedding_size = embedding_size - self.margin = margin - self.scale = scale - self.name = name - self.kernel = tf.Variable( - tf.random.normal([embedding_size, num_classes]), trainable=True - ) - - def call(self, y_true: FloatTensor, y_pred: FloatTensor) -> FloatTensor: - - y_pred_norm = tf.math.l2_normalize(y_pred, axis=1) - kernel_norm = tf.math.l2_normalize(self.kernel, axis=0) - - cos_theta = tf.matmul(y_pred_norm, kernel_norm) - cos_theta = tf.clip_by_value(cos_theta, -1.0, 1.0) - - m_hot = tf.one_hot(y_true, self.num_classes, on_value=self.margin, axis=1) - m_hot = tf.reshape(m_hot, [-1, self.num_classes]) - - cos_theta = tf.acos(cos_theta) - cos_theta += m_hot - cos_theta = tf.math.cos(cos_theta) - cos_theta = tf.math.multiply(cos_theta, self.scale) - - cce = tf.keras.losses.SparseCategoricalCrossentropy( - from_logits=True, reduction=self.reduction - ) - loss: FloatTensor = cce(y_true, cos_theta) - - return loss - - def get_config(self) -> Dict[str, Any]: - """Contains the loss configuration. - Returns: - The configuration of the ArcFace loss. - """ - config = { - "num_classes": self.num_classes, - "embedding_size": self.embedding_size, - "margin": self.margin, - "scale": self.scale, - "name": self.name, - } - base_config = super().get_config() - return {**base_config, **config} +""" + +from typing import Any, Callable, Dict, Optional + +import tensorflow as tf + +from tensorflow_similarity.types import FloatTensor + + +@tf.keras.utils.register_keras_serializable(package="Similarity") +class ArcFaceLoss(tf.keras.losses.Loss): + """Implement of ArcFace: Additive Angular Margin Loss: + Step 1: Create a trainable kernel matrix with the shape of [embedding_size, num_classes]. + Step 2: Normalize the kernel and prediction vectors. + Step 3: Calculate the cosine similarity between the normalized prediction vector and the kernel. + Step 4: Create a one-hot vector include the margin value for the ground truth class. + Step 5: Add margin_hot to the cosine similarity and multiply it by scale. + Step 6: Calculate the cross-entropy loss. + ArcFace: Additive Angular Margin Loss for Deep Face + Recognition. [online] arXiv.org. Available at: + . + Standalone usage: + >>> loss_fn = tfsim.losses.ArcFaceLoss(num_classes=2, embedding_size=3) + >>> labels = tf.Variable([1, 0]) + >>> embeddings = tf.Variable([[0.2, 0.3, 0.1], [0.4, 0.5, 0.5]]) + >>> loss = loss_fn(labels, embeddings) + Args: + num_classes: Number of classes. + embedding_size: The size of the embedding vectors. + margin: The margin value. + scale: s in the paper, feature scale + name: Optional name for the operation. + reduction: Type of loss reduction to apply to the loss. + """ + + def __init__( + self, + num_classes: int, + embedding_size: int, + margin: float = 0.50, # margin in radians + scale: float = 64.0, # feature scale + name: Optional[str] = None, + reduction: Callable = tf.keras.losses.Reduction.AUTO, + **kwargs + ): + + super().__init__(reduction=reduction, name=name, **kwargs) + + self.num_classes = num_classes + self.embedding_size = embedding_size + self.margin = margin + self.scale = scale + self.name = name + self.kernel = tf.Variable(tf.random.normal([embedding_size, num_classes]), trainable=True) + + def call(self, y_true: FloatTensor, y_pred: FloatTensor) -> FloatTensor: + + y_pred_norm = tf.math.l2_normalize(y_pred, axis=1) + kernel_norm = tf.math.l2_normalize(self.kernel, axis=0) + + cos_theta = tf.matmul(y_pred_norm, kernel_norm) + cos_theta = tf.clip_by_value(cos_theta, -1.0, 1.0) + + m_hot = tf.one_hot(y_true, self.num_classes, on_value=self.margin, axis=1) + m_hot = tf.reshape(m_hot, [-1, self.num_classes]) + + cos_theta = tf.acos(cos_theta) + cos_theta += m_hot + cos_theta = tf.math.cos(cos_theta) + cos_theta = tf.math.multiply(cos_theta, self.scale) + + cce = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction=self.reduction) + loss: FloatTensor = cce(y_true, cos_theta) + + return loss + + def get_config(self) -> Dict[str, Any]: + """Contains the loss configuration. + Returns: + The configuration of the ArcFace loss. + """ + config = { + "num_classes": self.num_classes, + "embedding_size": self.embedding_size, + "margin": self.margin, + "scale": self.scale, + "name": self.name, + } + base_config = super().get_config() + return {**base_config, **config} diff --git a/tests/test_losses.py b/tests/test_losses.py index ff149c94..86b340b0 100644 --- a/tests/test_losses.py +++ b/tests/test_losses.py @@ -1,5 +1,6 @@ import numpy as np import tensorflow as tf + from tensorflow_similarity.losses import ( ArcFaceLoss, MultiSimilarityLoss, @@ -87,9 +88,7 @@ def test_triplet_loss_semi_hard(): y_true = tf.random.uniform((num_inputs,), 0, 3, dtype=tf.int32) # y_preds: embedding y_preds = tf.random.uniform((num_inputs, 16), 0, 1) - tpl = TripletLoss( - positive_mining_strategy="easy", negative_mining_strategy="semi-hard" - ) + tpl = TripletLoss(positive_mining_strategy="easy", negative_mining_strategy="semi-hard") # y_true, y_preds loss = tpl(y_true, y_preds) assert loss From b832f95fcb773b76a18332fef10324f06a7a48a0 Mon Sep 17 00:00:00 2001 From: Owen Vallis Date: Wed, 12 Oct 2022 05:28:33 +0000 Subject: [PATCH 24/25] Update Arcface notebook. Initial review of notebook. Removed empty cells and verified that everything WAI. --- examples/ArcFace Loss Sample Notebook.ipynb | 1608 ++++++++++--------- 1 file changed, 833 insertions(+), 775 deletions(-) diff --git a/examples/ArcFace Loss Sample Notebook.ipynb b/examples/ArcFace Loss Sample Notebook.ipynb index b481492e..9eb1c21d 100644 --- a/examples/ArcFace Loss Sample Notebook.ipynb +++ b/examples/ArcFace Loss Sample Notebook.ipynb @@ -1,775 +1,833 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "28956aa1", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "Copyright 2022 The TensorFlow Similarity Authors." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24eda1a6", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "# @title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "id": "7ca9d025", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "# TensorFlow Similarity ArcFace Loss Example" - ] - }, - { - "cell_type": "markdown", - "id": "d072628f", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "A Total Angular Margin Loss (ArcFace) calculates the geodetic distance in the hypersphere instead of the euclidean distance to improve the discriminatory strength of the facial recognition model and stabilize the training process. Rails are used to measure all distances in geodetic space. The geodetic trace is the path taken between two places. It specifies the geodetic distance, which is the shortest distance between two places.\n", - "\n", - "ArcFace loss determines the angle between the current feature and the target weight using the arc-cosine function since the dot product between the DCNN feature and the last fully connected layer after feature and weight normalization matches the cosine distance. The target logit is then returned by multiplying the goal angle by an additional angular margin and using the cosine function. After that, we continue as before and rescale all logits to a certain feature norm, just like with softmax loss." - ] - }, - { - "cell_type": "markdown", - "id": "808ac087", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Notebook goal\n", - "\n", - "This notebook demonstrates how to use ArcFaceLoss implementation of TensorFlow Similarity with standalone usage and to train a `SimilarityModel()` on a fraction of the MNIST classes.\n", - "\n", - "You are going to learn about the main features offered by the `ArcFaceLoss()` and will:\n", - "\n", - " 1. Standalone usage of ArcFaceLoss\n", - "\n", - " 2. Usage with `model.compile()`\n", - "\n", - " 3. 3D-Visualization of ArcFaceLoss \n", - "\n", - "### Things to try \n", - "\n", - "Along the way you can try the following things to improve the model performance:\n", - "- Adding more \"seen\" classes at training time.\n", - "- Use a larger embedding by increasing the size of the output.\n", - "- Add data augmentation pre-processing layers to the model.\n", - "- Include more examples in the index to give the models more points to choose from.\n", - "- Try a more challenging dataset, such as Fashion MNIST." - ] - }, - { - "cell_type": "markdown", - "id": "078c53c0", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Notebook goal\n", - "\n", - "This notebook demonstrates how to use ArcFaceLoss implementation of TensorFlow Similarity with standalone usage and to train a `SimilarityModel()` on a fraction of the MNIST classes.\n", - "\n", - "You are going to learn about the main features offered by the `ArcFaceLoss()` and will:\n", - "\n", - " 1. Standalone usage of ArcFaceLoss\n", - "\n", - " 2. Usage with `model.compile()`\n", - "\n", - " 3. 3D-Visualization of ArcFaceLoss \n", - "\n", - "### Things to try \n", - "\n", - "Along the way you can try the following things to improve the model performance:\n", - "- Adding more \"seen\" classes at training time.\n", - "- Use a larger embedding by increasing the size of the output.\n", - "- Add data augmentation pre-processing layers to the model.\n", - "- Include more examples in the index to give the models more points to choose from.\n", - "- Try a more challenging dataset, such as Fashion MNIST." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8fd63f16", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "import gc\n", - "import os\n", - "\n", - "import numpy as np\n", - "from matplotlib import pyplot as plt\n", - "from tabulate import tabulate\n", - "from mpl_toolkits.mplot3d import Axes3D\n", - "\n", - "# INFO messages are not printed.\n", - "# This must be run before loading other modules.\n", - "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"1\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "80af5fc0", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "import tensorflow as tf" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8ba8caf7", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "# install TF similarity if needed\n", - "try:\n", - " import tensorflow_similarity as tfsim # main package\n", - "except ModuleNotFoundError:\n", - " !pip install tensorflow_similarity\n", - " import tensorflow_similarity as tfsim" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2484bd72", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "tfsim.utils.tf_cap_memory()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3fe0344e", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "# Clear out any old model state.\n", - "gc.collect()\n", - "tf.keras.backend.clear_session()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "99d9bef9", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "print(\"TensorFlow:\", tf.__version__)\n", - "print(\"TensorFlow Similarity\", tfsim.__version__)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7137afbc", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "1d534ad3", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "# Standalone Usage of ArcFaceLoss\n", - "\n", - "ArcFace loss alone can be used as follows when it is desired to calculate the additive angular margin loss of the existing data set." - ] - }, - { - "cell_type": "markdown", - "id": "68d526da", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Initialize Loss function as ArcFaceLoss" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bebf6ef0", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "loss_fn = tfsim.losses.ArcFaceLoss(num_classes=8, embedding_size=10)" - ] - }, - { - "cell_type": "markdown", - "id": "d2ccfd7d", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Create own simple random dataset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1d1ec43a", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "labels = tf.Variable([0, 1, 2, 3, 4, 5, 6, 7])\n", - "embeddings = tf.Variable(tf.random.uniform(shape=[8, 10]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "73d0c1c6", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "print(\"\", embeddings)" - ] - }, - { - "cell_type": "markdown", - "id": "d65b3085", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Calculate loss" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cdf7c30c", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "loss = loss_fn(labels, embeddings)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "16745b7d", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "print(\"loss : \" , loss)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "loss = loss_fn(labels, embeddings)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "print(\"loss : \" , loss)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } - }, - { - "cell_type": "markdown", - "source": [ - "## Data preparation\n", - "\n", - "We are going to load the MNIST dataset to showcase how the model is able to find similar examples from classes unseen during training. The model's ability to generalize the matching to unseen classes, without retraining, is one of the main reason you would want to use metric learning.\n", - "\n", - "\n", - "**WARNING**: Tensorflow similarity expects `y_train` to be an IntTensor containing the class ids for each example instead of the standard categorical encoding traditionally used for multi-class classification." - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8a9f8122", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"loss : \" , loss)" - ] - }, - { - "cell_type": "markdown", - "id": "11ef5236", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "## Data preparation\n", - "\n", - "We are going to load the MNIST dataset to showcase how the model is able to find similar examples from classes unseen during training. The model's ability to generalize the matching to unseen classes, without retraining, is one of the main reason you would want to use metric learning.\n", - "\n", - "\n", - "**WARNING**: Tensorflow similarity expects `y_train` to be an IntTensor containing the class ids for each example instead of the standard categorical encoding traditionally used for multi-class classification." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "97152229", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()" - ] - }, - { - "cell_type": "markdown", - "id": "08b766d8", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "## Model setup" - ] - }, - { - "cell_type": "markdown", - "id": "3eac2da7", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Model definition\n", - "\n", - "`SimilarityModel()` models extend `tensorflow.keras.model.Model` with additional features and functionality that allow you to index and search for similar looking examples.\n", - "\n", - "As visible in the model definition below, similarity models output a 64 dimensional float embedding using the `MetricEmbedding()` layers. This layer is a Dense layer with L2 normalization. Thanks to the loss, the model learns to minimize the distance between similar examples and maximize the distance between dissimilar examples. As a result, the distance between examples in the embedding space is meaningful; the smaller the distance the more similar the examples are. \n", - "\n", - "Being able to use a distance as a meaningful proxy for how similar two examples are, is what enables the fast ANN (aproximate nearest neighbor) search. Using a sub-linear ANN search instead of a standard quadratic NN search is what allows deep similarity search to scale to millions of items. The built in memory index used in this notebook scales to a million indexed examples very easily... if you have enough RAM :)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a003c971", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "def get_model():\n", - " inputs = tf.keras.layers.Input(shape=(28, 28, 1))\n", - " x = tf.keras.layers.experimental.preprocessing.Rescaling(1 / 255)(inputs)\n", - " x = tf.keras.layers.Conv2D(32, 3, activation=\"relu\")(x)\n", - " x = tf.keras.layers.Conv2D(32, 3, activation=\"relu\")(x)\n", - " x = tf.keras.layers.MaxPool2D()(x)\n", - " x = tf.keras.layers.Conv2D(64, 3, activation=\"relu\")(x)\n", - " x = tf.keras.layers.Conv2D(64, 3, activation=\"relu\")(x)\n", - " x = tf.keras.layers.Flatten()(x)\n", - " # smaller embeddings will have faster lookup times while a larger embedding will improve the accuracy up to a point.\n", - " outputs = tfsim.layers.MetricEmbedding(64)(x)\n", - " return tfsim.models.SimilarityModel(inputs, outputs)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a2177b12", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "model = get_model()\n", - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "id": "defb3961", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### ArcFace Loss definition\n", - "\n", - "Overall what makes Metric losses different from tradional losses is that:\n", - "- **They expect different inputs.** Instead of having the prediction equal the true values, they expect embeddings as `y_preds` and the id (as an int32) of the class as `y_true`. \n", - "- **They require a distance.** You need to specify which `distance` function to use to compute the distance between embeddings. `cosine` is usually a great starting point and the default.\n", - "\n", - "ArcFace Loss takes inputs as number of classes which labels includes, and embedding size which we define in model `MetricEmbedding()` layers." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "13b0d745", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "distance = \"cosine\" " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c22d10cc", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "num_classes = np.unique(y_train).size\n", - "embedding_size = model.get_layer('metric_embedding').output.shape[1]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d5b8e426", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "loss = tfsim.losses.ArcFaceLoss(num_classes= num_classes, embedding_size=embedding_size, name=\"ArcFaceLoss\")" - ] - }, - { - "cell_type": "markdown", - "id": "b6eaf9c8", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### Compilation\n", - "\n", - "Tensorflow similarity use an extended `compile()` method that allows you to optionally specify `distance_metrics` (metrics that are computed over the distance between the embeddings), and the distance to use for the indexer.\n", - "\n", - "By default the `compile()` method tries to infer what type of distance you are using by looking at the first loss specified. If you use multiple losses, and the distance loss is not the first one, then you need to specify the distance function used as `distance=` parameter in the compile function." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "673f986f", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "LR = 0.0005 # @param {type:\"number\"}\n", - "model.compile(optimizer=tf.keras.optimizers.SGD(LR), loss=loss, distance=distance)" - ] - }, - { - "cell_type": "markdown", - "id": "15961601", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "## Training\n", - "\n", - "Similarity models are trained like normal models. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "147a6863", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "EPOCHS = 10 # @param {type:\"integer\"}\n", - "history = model.fit(x_train, y_train, epochs=EPOCHS, validation_data=(x_test, y_test))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "88e1ee4d", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "plt.plot(history.history[\"loss\"])\n", - "plt.plot(history.history[\"val_loss\"])\n", - "plt.legend([\"loss\", \"val_loss\"])\n", - "plt.title(f\"Loss: {loss.name}\")\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a5404906", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "5ad4ba20", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "## Prediction\n", - "\n", - "Let's predict some features and visualiza them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a1936264", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "embedded_features = model.predict(x_test, verbose=1)\n", - "embedded_features /= np.linalg.norm(embedded_features, axis=1, keepdims=True)" - ] - }, - { - "cell_type": "markdown", - "id": "7c0df63b", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "### 3D-Visualization of ArcFace Loss" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5aac5d98", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "fig = plt.figure()\n", - "ax = Axes3D(fig)\n", - "for c in range(len(np.unique(y_test))):\n", - " ax.plot(embedded_features[y_test==c, 0], embedded_features[y_test==c, 1], embedded_features[y_test==c, 2], '.', alpha=0.1)\n", - "plt.title('ArcFace')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8889f840", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fef529d9", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file +{ + "cells": [ + { + "cell_type": "markdown", + "id": "28956aa1", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "Copyright 2022 The TensorFlow Similarity Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24eda1a6", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# @title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "id": "7ca9d025", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "# TensorFlow Similarity ArcFace Loss Example" + ] + }, + { + "cell_type": "markdown", + "id": "d072628f", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "A Total Angular Margin Loss (ArcFace) calculates the geodetic distance in the hypersphere instead of the euclidean distance to improve the discriminatory strength of the facial recognition model and stabilize the training process. Rails are used to measure all distances in geodetic space. The geodetic trace is the path taken between two places. It specifies the geodetic distance, which is the shortest distance between two places.\n", + "\n", + "ArcFace loss determines the angle between the current feature and the target weight using the arc-cosine function since the dot product between the DCNN feature and the last fully connected layer after feature and weight normalization matches the cosine distance. The target logit is then returned by multiplying the goal angle by an additional angular margin and using the cosine function. After that, we continue as before and rescale all logits to a certain feature norm, just like with softmax loss." + ] + }, + { + "cell_type": "markdown", + "id": "808ac087", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Notebook goal\n", + "\n", + "This notebook demonstrates how to use ArcFaceLoss implementation of TensorFlow Similarity with standalone usage and to train a `SimilarityModel()` on a fraction of the MNIST classes.\n", + "\n", + "You are going to learn about the main features offered by the `ArcFaceLoss()` and will:\n", + "\n", + " 1. Standalone usage of ArcFaceLoss\n", + "\n", + " 2. Usage with `model.compile()`\n", + "\n", + " 3. 3D-Visualization of ArcFaceLoss \n", + "\n", + "### Things to try \n", + "\n", + "Along the way you can try the following things to improve the model performance:\n", + "- Adding more \"seen\" classes at training time.\n", + "- Use a larger embedding by increasing the size of the output.\n", + "- Add data augmentation pre-processing layers to the model.\n", + "- Include more examples in the index to give the models more points to choose from.\n", + "- Try a more challenging dataset, such as Fashion MNIST." + ] + }, + { + "cell_type": "markdown", + "id": "078c53c0", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Notebook goal\n", + "\n", + "This notebook demonstrates how to use ArcFaceLoss implementation of TensorFlow Similarity with standalone usage and to train a `SimilarityModel()` on a fraction of the MNIST classes.\n", + "\n", + "You are going to learn about the main features offered by the `ArcFaceLoss()` and will:\n", + "\n", + " 1. Standalone usage of ArcFaceLoss\n", + "\n", + " 2. Usage with `model.compile()`\n", + "\n", + " 3. 3D-Visualization of ArcFaceLoss \n", + "\n", + "### Things to try \n", + "\n", + "Along the way you can try the following things to improve the model performance:\n", + "- Adding more \"seen\" classes at training time.\n", + "- Use a larger embedding by increasing the size of the output.\n", + "- Add data augmentation pre-processing layers to the model.\n", + "- Include more examples in the index to give the models more points to choose from.\n", + "- Try a more challenging dataset, such as Fashion MNIST." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "8fd63f16", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import gc\n", + "import os\n", + "\n", + "import numpy as np\n", + "from matplotlib import pyplot as plt\n", + "from tabulate import tabulate\n", + "from mpl_toolkits.mplot3d import Axes3D\n", + "\n", + "# INFO messages are not printed.\n", + "# This must be run before loading other modules.\n", + "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"1\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "80af5fc0", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import tensorflow as tf" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "8ba8caf7", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Your CPU supports instructions that this binary was not compiled to use: SSE3 SSE4.1 SSE4.2 AVX AVX2\n", + "For maximum performance, you can install NMSLIB from sources \n", + "pip install --no-binary :all: nmslib\n" + ] + } + ], + "source": [ + "# install TF similarity if needed\n", + "try:\n", + " import tensorflow_similarity as tfsim # main package\n", + "except ModuleNotFoundError:\n", + " !pip install tensorflow_similarity\n", + " import tensorflow_similarity as tfsim" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "2484bd72", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "tfsim.utils.tf_cap_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "3fe0344e", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Clear out any old model state.\n", + "gc.collect()\n", + "tf.keras.backend.clear_session()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "99d9bef9", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TensorFlow: 2.8.0\n", + "TensorFlow Similarity 0.17.0.dev10\n" + ] + } + ], + "source": [ + "print(\"TensorFlow:\", tf.__version__)\n", + "print(\"TensorFlow Similarity\", tfsim.__version__)" + ] + }, + { + "cell_type": "markdown", + "id": "1d534ad3", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "# Standalone Usage of ArcFaceLoss\n", + "\n", + "ArcFace loss alone can be used as follows when it is desired to calculate the additive angular margin loss of the existing data set." + ] + }, + { + "cell_type": "markdown", + "id": "68d526da", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Initialize Loss function as ArcFaceLoss" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "bebf6ef0", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "loss_fn = tfsim.losses.ArcFaceLoss(num_classes=8, embedding_size=10)" + ] + }, + { + "cell_type": "markdown", + "id": "d2ccfd7d", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Create own simple random dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "1d1ec43a", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "labels = tf.Variable([0, 1, 2, 3, 4, 5, 6, 7])\n", + "embeddings = tf.Variable(tf.random.uniform(shape=[8, 10]))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "73d0c1c6", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " \n" + ] + } + ], + "source": [ + "print(\"\", embeddings)" + ] + }, + { + "cell_type": "markdown", + "id": "d65b3085", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Calculate loss" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "cdf7c30c", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "loss = loss_fn(labels, embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "16745b7d", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loss : tf.Tensor(48.764076, shape=(), dtype=float32)\n" + ] + } + ], + "source": [ + "print(\"loss : \" , loss)" + ] + }, + { + "cell_type": "markdown", + "id": "11ef5236", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Data preparation\n", + "\n", + "We are going to load the MNIST dataset to showcase how the model is able to find similar examples from classes unseen during training. The model's ability to generalize the matching to unseen classes, without retraining, is one of the main reason you would want to use metric learning.\n", + "\n", + "\n", + "**WARNING**: Tensorflow similarity expects `y_train` to be an IntTensor containing the class ids for each example instead of the standard categorical encoding traditionally used for multi-class classification." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "97152229", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()" + ] + }, + { + "cell_type": "markdown", + "id": "08b766d8", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Model setup" + ] + }, + { + "cell_type": "markdown", + "id": "3eac2da7", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Model definition\n", + "\n", + "`SimilarityModel()` models extend `tensorflow.keras.model.Model` with additional features and functionality that allow you to index and search for similar looking examples.\n", + "\n", + "As visible in the model definition below, similarity models output a 64 dimensional float embedding using the `MetricEmbedding()` layers. This layer is a Dense layer with L2 normalization. Thanks to the loss, the model learns to minimize the distance between similar examples and maximize the distance between dissimilar examples. As a result, the distance between examples in the embedding space is meaningful; the smaller the distance the more similar the examples are. \n", + "\n", + "Being able to use a distance as a meaningful proxy for how similar two examples are, is what enables the fast ANN (aproximate nearest neighbor) search. Using a sub-linear ANN search instead of a standard quadratic NN search is what allows deep similarity search to scale to millions of items. The built in memory index used in this notebook scales to a million indexed examples very easily... if you have enough RAM :)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "a003c971", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "def get_model():\n", + " inputs = tf.keras.layers.Input(shape=(28, 28, 1))\n", + " x = tf.keras.layers.experimental.preprocessing.Rescaling(1 / 255)(inputs)\n", + " x = tf.keras.layers.Conv2D(32, 3, activation=\"relu\")(x)\n", + " x = tf.keras.layers.Conv2D(32, 3, activation=\"relu\")(x)\n", + " x = tf.keras.layers.MaxPool2D()(x)\n", + " x = tf.keras.layers.Conv2D(64, 3, activation=\"relu\")(x)\n", + " x = tf.keras.layers.Conv2D(64, 3, activation=\"relu\")(x)\n", + " x = tf.keras.layers.Flatten()(x)\n", + " # smaller embeddings will have faster lookup times while a larger embedding will improve the accuracy up to a point.\n", + " outputs = tfsim.layers.MetricEmbedding(64)(x)\n", + " return tfsim.models.SimilarityModel(inputs, outputs)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "a2177b12", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"similarity_model\"\n", + "_________________________________________________________________\n", + " Layer (type) Output Shape Param # \n", + "=================================================================\n", + " input_1 (InputLayer) [(None, 28, 28, 1)] 0 \n", + " \n", + " rescaling (Rescaling) (None, 28, 28, 1) 0 \n", + " \n", + " conv2d (Conv2D) (None, 26, 26, 32) 320 \n", + " \n", + " conv2d_1 (Conv2D) (None, 24, 24, 32) 9248 \n", + " \n", + " max_pooling2d (MaxPooling2D (None, 12, 12, 32) 0 \n", + " ) \n", + " \n", + " conv2d_2 (Conv2D) (None, 10, 10, 64) 18496 \n", + " \n", + " conv2d_3 (Conv2D) (None, 8, 8, 64) 36928 \n", + " \n", + " flatten (Flatten) (None, 4096) 0 \n", + " \n", + " metric_embedding (MetricEmb (None, 64) 262208 \n", + " edding) \n", + " \n", + "=================================================================\n", + "Total params: 327,200\n", + "Trainable params: 327,200\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ] + } + ], + "source": [ + "model = get_model()\n", + "model.summary()" + ] + }, + { + "cell_type": "markdown", + "id": "defb3961", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### ArcFace Loss definition\n", + "\n", + "Overall what makes Metric losses different from tradional losses is that:\n", + "- **They expect different inputs.** Instead of having the prediction equal the true values, they expect embeddings as `y_preds` and the id (as an int32) of the class as `y_true`. \n", + "- **They require a distance.** You need to specify which `distance` function to use to compute the distance between embeddings. `cosine` is usually a great starting point and the default.\n", + "\n", + "ArcFace Loss takes inputs as number of classes which labels includes, and embedding size which we define in model `MetricEmbedding()` layers." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "13b0d745", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "distance = \"cosine\" " + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "c22d10cc", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "num_classes = np.unique(y_train).size\n", + "embedding_size = model.get_layer('metric_embedding').output.shape[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "d5b8e426", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "loss = tfsim.losses.ArcFaceLoss(num_classes=num_classes, embedding_size=embedding_size, name=\"ArcFaceLoss\")" + ] + }, + { + "cell_type": "markdown", + "id": "b6eaf9c8", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### Compilation\n", + "\n", + "Tensorflow similarity use an extended `compile()` method that allows you to optionally specify `distance_metrics` (metrics that are computed over the distance between the embeddings), and the distance to use for the indexer.\n", + "\n", + "By default the `compile()` method tries to infer what type of distance you are using by looking at the first loss specified. If you use multiple losses, and the distance loss is not the first one, then you need to specify the distance function used as `distance=` parameter in the compile function." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "673f986f", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "LR = 0.0005 # @param {type:\"number\"}\n", + "model.compile(optimizer=tf.keras.optimizers.SGD(LR), loss=loss, distance=distance)" + ] + }, + { + "cell_type": "markdown", + "id": "15961601", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Training\n", + "\n", + "Similarity models are trained like normal models. " + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "147a6863", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/10\n", + "1875/1875 [==============================] - 11s 4ms/step - loss: 5.2161 - val_loss: 1.8907\n", + "Epoch 2/10\n", + "1875/1875 [==============================] - 7s 4ms/step - loss: 1.8353 - val_loss: 1.6826\n", + "Epoch 3/10\n", + "1875/1875 [==============================] - 7s 4ms/step - loss: 1.3566 - val_loss: 1.1404\n", + "Epoch 4/10\n", + "1875/1875 [==============================] - 7s 4ms/step - loss: 1.1160 - val_loss: 1.0936\n", + "Epoch 5/10\n", + "1875/1875 [==============================] - 7s 4ms/step - loss: 0.9555 - val_loss: 1.0854\n", + "Epoch 6/10\n", + "1875/1875 [==============================] - 7s 4ms/step - loss: 0.8343 - val_loss: 1.0062\n", + "Epoch 7/10\n", + "1875/1875 [==============================] - 7s 4ms/step - loss: 0.7546 - val_loss: 0.9062\n", + "Epoch 8/10\n", + "1875/1875 [==============================] - 7s 4ms/step - loss: 0.6776 - val_loss: 0.8000\n", + "Epoch 9/10\n", + "1875/1875 [==============================] - 7s 4ms/step - loss: 0.6194 - val_loss: 0.8160\n", + "Epoch 10/10\n", + "1875/1875 [==============================] - 7s 4ms/step - loss: 0.5676 - val_loss: 0.7515\n" + ] + } + ], + "source": [ + "EPOCHS = 10 # @param {type:\"integer\"}\n", + "history = model.fit(x_train, y_train, epochs=EPOCHS, validation_data=(x_test, y_test))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "88e1ee4d", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAEICAYAAAB25L6yAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAna0lEQVR4nO3de3xcdZ3/8dcnyeR+adImvaVtWihQaLmZVlikXBVRLrr6U1BRWRYWcBFYRFF/KN5ddRFdWVx/LoKCCgu4IiAgFymoC02hUEpLgV6Tltza3JvbzPf3xzmTTNIknbRJ5szM+/l4nEdmzpw585khvPvNd845H3POISIiwZWR6AJERGRsCmoRkYBTUIuIBJyCWkQk4BTUIiIBp6AWEQk4BbWISMApqGVUZrbVzM5MdB1RZnaTmTkzW3EQ+6jy99ERs7w8kXWO8FpZk7F/SR/6BZKkYGYGXATsBj4FvDDGtlnOuf797HJaHNuIBIJG1DJuZpZjZreY2U5/ucXMcvzHZpjZQ2bWYma7zexZM8vwH/uCmdWZWbuZvW5mZ4zjZU8G5gBXAxeYWXZMPZ82s7+Y2Q/NbDdwk5nlmdm/mdk2M2s1s+fMLG+M97TCzP7m173LzH4y7DWOMrM/+e+p3sy+5K/PMLMbzOwtM2s2s3vNrCyOz3COmT3o7+9NM7t0WC01Ztbmv9bN/vpcM7vLf50WM1ttZjPH8RlKklJQy4H4MnACcCxwDLAC+L/+Y9cBtUA5MBP4EuDM7HDgn4Hlzrki4CxgK4CZvcvMWvbzmp8C/gDc498/Z9jj7wQ2AxXAt4AfAO8A/g4oAz4PRMbYfxi4FpgBnAicAVzp11cEPAE8ivePxaHAk/7zPgt8ADjFf2wPcOt+3gvAb/A+pznAh4Fvx/zD9SPgR865YuAQ4N6Yz6AEmAdMBy4H9sbxWpLsnHNatIy44AXpmSOsfwt4X8z9s4Ct/u2vA78HDh32nEOBBuBMIDTOOvKBNuAD/v3/BH4f8/inge0x9zPwAuyYEfZVBTigJWb53AjbXQP8zr99IfDSKLVtAM6IuT8b6MObVoy+Vtaw58zD+4ehKGbdd4A7/NurgK8BM4Y97x+AvwJHJ/p3Q8vULhpRy4GYA2yLub/NXwfwfeBN4HEz22xmNwA4597EC7+bgAYz+62ZzSE+HwT6gUf8+3cDZ5tZecw2O2JuzwBy8f5BGc0M59w0f/mBmR3mT9m8bWZtwLf9/YAXrKPtawHwO38qogUvuMN4f02MZg6w2znXHrNuGzDXv30JcBiw0Z/eiP718CvgMeC3/pTT98wsNMbrSIpQUMuB2IkXUFHz/XU459qdc9c55xYB5wL/Ev2T3jn3a+fcu/znOuBf43y9TwGFwHYzexv4byCEN9KNir0MZBPQjTdtEK/bgI3AYudNOXwJMP+xHWPsawdwdkzoT3PO5Trn6sZ4rZ1AmT+lEjUfqANwzr3hnLsQbxrnX4H7zKzAOdfnnPuac+5IvCmdc4BPjuM9SpJSUMv+hPwvsaJLFt786v81s3IzmwF8BbgLwMzOMbND/aM02vBGl2EzO9zMTve/dOzGm5oI7+/FzWwu3nzxOXhz4sfizYv/K16A78M5FwFuB272v7TLNLMTo194jqLIr7fDzI4Aroh57CFglpld43+RWmRm7/Qf+ynwLTNb4NdbbmbnD9t3TuxniBfIfwW+4687Gm8Ufbe/j0+YWbn/Plr8fYTN7DQzW2ZmmX6tfcTxGUoKSPTci5bgLnhz1G7Y8k28aYUfA7v85cdArv+ca/3ndeJ9WXajv/5ovEPq2vEOsXsImOM/djLQMUoNNwBrRlg/By+oluLNUT837PE84Ba8UGzFm/fNY/R545V4I+oO4Fm8ufbnYh5fivcF4h7gbeAGf30G8C/A6/57ewv4tv9Y9LWGL2cClf5nsNt/zuUxr3UX3nx+B7Cewbn5C/3X6QTq/c89a6z/hlpSYzH/F0BERAJKUx8iIgGnoBYRCTgFtYhIwCmoRUQCblIuyjRjxgxXVVU1GbsWEUlJa9asaXLOlY/02KQEdVVVFTU1NZOxaxGRlGRm20Z7TFMfIiIBp6AWEQk4BbWISMCpw4uITIi+vj5qa2vp7u5OdCmBlpubS2VlJaFQ/Bc+VFCLyISora2lqKiIqqoqvGtyyXDOOZqbm6mtrWXhwoVxP09THyIyIbq7u5k+fbpCegxmxvTp08f9V4eCWkQmjEJ6/w7kMwpMUPf0h/npM2/x7BuNiS5FRCRQAhPU2ZkZ/GzVZn6/dmeiSxGRJFVYWJjoEiZFYILazKheUErN1t2JLkVEJFACE9QAy6vK2NrcRUO7Du8RkQPnnOP6669n6dKlLFu2jHvuuQeAXbt2sXLlSo499liWLl3Ks88+Szgc5tOf/vTAtj/84Q8TXP2+AnV4XnVVKQA1W/fwvmWzE1yNiByor/1hPa/tbJvQfR45p5ivnntUXNs+8MADrF27lpdffpmmpiaWL1/OypUr+fWvf81ZZ53Fl7/8ZcLhMF1dXaxdu5a6ujpeffVVAFpaWia07okQqBH1UXNKyA1lsFrTHyJyEJ577jkuvPBCMjMzmTlzJqeccgqrV69m+fLl/OIXv+Cmm25i3bp1FBUVsWjRIjZv3sxVV13Fo48+SnFxcaLL30egRtTZWRkcN6+Umq17El2KiByEeEe+k2W0XrArV65k1apVPPzww1x00UVcf/31fPKTn+Tll1/mscce49Zbb+Xee+/l9ttvn+KKxxbXiNrMtprZOjNba2aTev3S5VWlrN/ZSkdP/2S+jIiksJUrV3LPPfcQDodpbGxk1apVrFixgm3btlFRUcGll17KJZdcwosvvkhTUxORSIQPfehDfOMb3+DFF19MdPn7GM+I+jTnXNOkVeKrrioj4uCl7Xs4efGI19AWERnTBz/4Qf72t79xzDHHYGZ873vfY9asWdx55518//vfJxQKUVhYyC9/+Uvq6uq4+OKLiUQiAHznO99JcPX7stH+RBiykdlWoDreoK6urnYH2jigvbuPY772OP98+mL+5d2HHdA+RGTqbdiwgSVLliS6jKQw0mdlZmucc9UjbR/vl4kOeNzM1pjZZSNtYGaXmVmNmdU0Nh742YVFuSGOnFOs46lFRHzxBvVJzrnjgbOBz5jZyuEbOOd+5pyrds5Vl5cf3JRF9YIyXtreQl84clD7ERFJBXEFtXNup/+zAfgdsGIyi1peVcbevjDrJ/g4TBGRZLTfoDazAjMrit4G3gO8OplFDZ74oukPEZF4RtQzgefM7GXgBeBh59yjk1nUzOJc5pfl68QXERHiODzPObcZOGYKahlieVUZf369AeecrnErImktUKeQx1peVUpzZy+bmzoTXYqISEIFNqirq8oAzVOLyOQY69rVW7duZenSpVNYzdgCG9SHlBdQVpDNal33Q0TSXKAuyhRLjQREktgfb4C3103sPmctg7O/O+rDX/jCF1iwYAFXXnklADfddBNmxqpVq9izZw99fX1885vf5Pzzzx/Xy3Z3d3PFFVdQU1NDVlYWN998M6eddhrr16/n4osvpre3l0gkwv3338+cOXP4yEc+Qm1tLeFwmBtvvJGPfvSjB/W2IcBBDd4Xio+/Vk9DezcVRbmJLkdEAuyCCy7gmmuuGQjqe++9l0cffZRrr72W4uJimpqaOOGEEzjvvPPGdYDCrbfeCsC6devYuHEj73nPe9i0aRM//elPufrqq/n4xz9Ob28v4XCYRx55hDlz5vDwww8D0NraOiHvLdBBrUYCIklqjJHvZDnuuONoaGhg586dNDY2UlpayuzZs7n22mtZtWoVGRkZ1NXVUV9fz6xZs+Le73PPPcdVV10FwBFHHMGCBQvYtGkTJ554It/61reora3l7//+71m8eDHLli3jc5/7HF/4whc455xzOPnkkyfkvQV2jhrUSEBExufDH/4w9913H/fccw8XXHABd999N42NjaxZs4a1a9cyc+ZMurvH1+pvtAvXfexjH+PBBx8kLy+Ps846i6eeeorDDjuMNWvWsGzZMr74xS/y9a9/fSLeVrBH1NlZGRw7b5oaCYhIXC644AIuvfRSmpqaeOaZZ7j33nupqKggFArx9NNPs23btnHvc+XKldx9992cfvrpbNq0ie3bt3P44YezefNmFi1axGc/+1k2b97MK6+8whFHHEFZWRmf+MQnKCws5I477piQ9xXooAZYUVXGT55+k46efgpzAl+uiCTQUUcdRXt7O3PnzmX27Nl8/OMf59xzz6W6uppjjz2WI444Ytz7vPLKK7n88stZtmwZWVlZ3HHHHeTk5HDPPfdw1113EQqFmDVrFl/5yldYvXo1119/PRkZGYRCIW677bYJeV9xXY96vA7metTDrdrUyCdvf4FfXbJCjQREAkzXo47fZF2POmGOmz+NDEPHU4tI2gr8XEJRbogls9VIQEQm3rp167jooouGrMvJyeH5559PUEUjC3xQg3c89T2rd9AXjhDKDPwfASJpK9kuorZs2TLWrl07pa95INPNSZF60UYCr6mRgEhg5ebm0tzcfEBBlC6cczQ3N5ObO74T+JJiRB098WX11t0cM29aYosRkRFVVlZSW1vLwfRMTQe5ublUVlaO6zlJEdSxjQT+8eRFiS5HREYQCoVYuHBhostISUkx9QHeqLpm6x79WSUiaSdpgnpFVRnNnb1sUSMBEUkzSRPU0UYCuu6HiKSbpAnqQ8oLKM0P6cQXEUk7SRPUZkZ1VZlOfBGRtJM0QQ3ePPXW5i4a2sd3mUIRkWSWVEEdPZ56jaY/RCSNJFVQRxsJvKDpDxFJI0kV1GokICLpKKmCGrzrfqzf2UpHT3+iSxERmRJJGdQRB2u3tyS6FBGRKZF0QR1tJKB5ahFJF0kX1GokICLpJumCGrzpj5e2t9AXjiS6FBGRSZe0Qa1GAiKSLpIyqGMbCYiIpLqkDOrYRgIiIqkuKYMa1EhARNJH3EFtZplm9pKZPTSZBcVruRoJiEiaGM+I+mpgw2QVMl7L/UYCOp1cRFJdXEFtZpXA+4GfT2458Ys2EtCJLyKS6uIdUd8CfB4Y9cBlM7vMzGrMrGYq2sWrkYCIpIv9BrWZnQM0OOfWjLWdc+5nzrlq51x1eXn5hBU4luVVpWokICIpL54R9UnAeWa2FfgtcLqZ3TWpVcUpOk+tRgIiksr2G9TOuS865yqdc1XABcBTzrlPTHplcYg2ElDDWxFJZUl7HDUMNhLQiS8iksrGFdTOuT87586ZrGIOhBoJiEiqS+oRNUC1GgmISIpL+qA+3m8koOkPEUlVSR/U0UYCCmoRSVVJH9SgRgIiktpSIqirq0rVSEBEUlZKBHX0xBdNf4hIKkqJoI42EtCV9EQkFaVEUIM3/bF66241EhCRlJMyQa1GAiKSqlIqqEGNBEQk9aRMUEcbCegLRRFJNSkT1NFGAgpqEUk1KRPUoEYCIpKaUiqoq9VIQERSUEoF9VI1EhCRFJRSQR1tJFCzTfPUIpI6UiqoIdpIoI1ONRIQkRSRckFdXVVGOOJ4SY0ERCRFpFxQq5GAiKSalAvqaCMBzVOLSKpIuaAGb576xW1qJCAiqSElg1qNBEQklaRkUKuRgIikkpQMajUSEJFUkpJBDd70R802NRIQkeSXskG9vKqMpg41EhCR5JfCQV0KqJGAiCS/lA3qQ8oL1UhARFJCygZ1tJFAzTaNqEUkuaVsUIM3/bGlqVONBEQkqaV0UKuRgIikgpQOajUSEJFUkNJBrUYCIpIK9hvUZpZrZi+Y2ctmtt7MvjYVhU0UNRIQkWQXz4i6BzjdOXcMcCzwXjM7YVKrmkBqJCAiyW6/Qe08Hf7dkL8kzXnZaiQgIskurjlqM8s0s7VAA/An59zzI2xzmZnVmFlNY2PjBJd54NRIQESSXVxB7ZwLO+eOBSqBFWa2dIRtfuacq3bOVZeXl09wmQdneVUZL21XIwERSU7jOurDOdcC/Bl472QUM1mqq0rp6lUjARFJTvEc9VFuZtP823nAmcDGSa5rQlUvUCMBEUle8YyoZwNPm9krwGq8OeqHJresiTWrJJd5ZXm6kp6IJKWs/W3gnHsFOG4KaplUy6vKWLWpEeccZpbockRE4pbSZybGijYS2NrclehSRETGJY2C2msksHqL5qlFJLmkTVCrkYCIJKu0CWo1EhCRZJU2QQ2DjQQa23sSXYqISNzSKqijjQRqNP0hIkkkrYJ66ZwScrLUSEBEkktaBbUaCYhIMkqroAZYsVCNBEQkuaRdUEcbCazd0ZLoUkRE4pJ2QR1tJPCCTnwRkSSRdkFdlBviiFlqJCAiySPtghq8eWo1EhCRZJGWQR1tJLBhlxoJiEjwpWdQ+40ENE8tIskgLYNajQREJJmkZVADLF9QRs223TjnEl2KiMiY0jeoF6qRgIgkh/QN6mgjAV2gSUQCLm2DeqCRgL5QFJGAS9ugNjPesUCNBEQk+NI2qAFWLFQjAREJvrQO6mgjgTU6nVxEAiytgzraSOCFLZr+EJHgSuugViMBEUkGaR3UAMur1EhARIJNQb1QjQREJNjSPqijjQR04ouIBFXaB3W0kYCCWkSCKu2DGrzTydVIQESCSkGNN0+tRgIiElQKagYbCazW9alFJIAU1Aw2EtAFmkQkiBTUPjUSEJGg2m9Qm9k8M3vazDaY2Xozu3oqCptq1VVqJCAiwRTPiLofuM45twQ4AfiMmR05uWVNvRUL1UhARIJpv0HtnNvlnHvRv90ObADmTnZhUy3aSKBGQS0iATOuOWozqwKOA54f4bHLzKzGzGoaGxsnqLypE20koCM/RCRo4g5qMysE7geucc7tc8Cxc+5nzrlq51x1eXn5RNY4ZZZXqZGAiARPXEFtZiG8kL7bOffA5JaUOMsXqpGAiARPPEd9GPBfwAbn3M2TWs3uLRAJT+pLjCXaSEDTHyISJFlxbHMScBGwzszW+uu+5Jx7ZEIriYThZ6cABgtPhoWnwKJTYfqhYDahLzWaaCMBHfkhIkGy36B2zj0HTH5SRsLwvh/A5mdgyzOw4Q/e+qI5sOgUP7hPgeI5k1rG8qoybnvmLTp7+inIieffMRGRyRWcJMrKhqM/4i3Owe7NXmBvfgY2PQYv/8bbbvriweCuehfkl01oGdVVpYSfdvx3zQ4+9XdV2BSN5kVERmOTccp0dXW1q6mpmbgdRiJQ/6oX3FtWwda/QF8nYDD7mMHgnn8iZOcf1Et194X56H/+jZdrWznp0Ol87bylHFpRODHvQ0RkFGa2xjlXPeJjSRHUw4X7oG7N4DTJjhcg0geZ2VC5YjC45x4PmaFx774/HOHu57fzg8dfp7svzCXvWsRnzziU/Ozg/AEiIqkl9YJ6uN5O2P63weDe9QrgILsQFpw0GNwVR0JG/Of4NLb38N0/buT+F2uZU5LLjeccyXuXztJ0iIhMuNQP6uG6dsPWZweDu/lNb33+jJgjSk6B0oVxHVGyeutubvyfV9n4djsnL57B1847ikXlmg4RkYmTfkE9XGvd4BeTW56B9l3e+pL5sGglLDwVFq6Eopmj7qI/HOFX/7uNmx/fRHd/mMtWLuIzp2k6REQmhoI6lnPQ9IYf3H/2Rt7drd5j5Uu8kfaiU+HQd0PmviHc0N7Ndx/ZyAMv1TF3Wh43nnMkZx01U9MhInJQFNRjiYRh18uDI+7t/wv9e2HO8fDB/4Tyw0Z82vObm/nK79fzen07px5ezk3nHkXVjIIpLl5EUoWCejz6e+C1B+GPn4e+Ljjjq/DOy0f8ErIvHOHOv27llifeoLc/wuWnLOKKUw8lLzszAYWLSDJTUB+I9nr4w2dh06Ow4F3wgVuhtGrETRvauvnWIxv4/dqdVJbm8dVzj+LMJRWaDhGRuI0V1OqZOJqimXDhb+H8W72pkdtOgjV3eHPcw1QU5/KjC47jN5eeQF4ok0t/WcMld9awrblz6usWkZSjEXU8WrbD/1zpffG4+D1w7o+hePaIm/aFI9zxl63c8sQm+iKOK045hCtOPYTckKZDRGR0GlEfrGnz4ZMPwtnfgy3Pwn+cAOvuG3F0HcrM4NKVi3jyulM566hZ/OjJN3j3D5/hyQ31CShcRFKBgjpeGRnwzn+Cy5+DGYvh/kvgvz8Nnc0jbj6rJJd/v/A4fv2P7yQ7M4NL7qzhH+9czY7d6nIuIuOjqY8DEe6Hv/4Inv4O5JXCeT+Gw88edfPe/gi/+MsWfvTkG4Qjjs+cdiiXrVyk6RARGaCpj4mWmQUnXweX/RkKZ8JvLvDmsKMnzgyTnZXBP51yCE9edwpnHjmTm/+0ibNuWcXTrzdMbd0ikpQU1Adj1lK49Ck4+XPe9bJvO8k723EUs0vyuPVjx/OrS1aQacbFv1jNZb+s0XSIiIxJQX2wsrLhjBvhkj9BVg788nx45HroHT18T15czh+vOZnPv/dwnn2jiXf/8Bl+8tQb9PQnrl+kiASX5qgnUm8XPPl1eP42KDsEPvhTmLdizKfUtezlmw+9xh9ffZuFMwq46byjOOWw8ikqWESCQnPUUyU7H87+LnzqD15zg9vPgidu8k5LH8XcaXnc9ol3cOc/eIH+qdtf4Iq71lDXsneKihaRoNOIerJ0t8FjX4KXfgUVR3mj69lHj/mUnv4wP392C//+1BsYxpWnHsL/qZ7HrJLcKSpaRBJF1/pIpE2PwYNXQVcznHIDvOvaES+fGqt2TxffeOg1HlvvnSSzbG4JZy6ZyRlLKjhqTrGuISKSghTUida1Gx75HLx6P8x9B3zgp6NePjXWpvp2/vRaPU9uqOelHS04B7NLcjljSQVnLpnJCYum61hskRShoA6KVx+Ah6/b7+VTR9LU0cNTGxt44rV6nn2jib19YfKzM1m5uJwzllRw+hEVTC/MmeQ3ICKTRUEdJPtcPvU/oHTBuHbR3Rfmb28188SGep7c0MDbbd2YwfHzSzlzyUzOXFLBoRWFmiIRSSIK6qBxDtbeDX+8AXBw1rfh+E/G1Wh331051u9s44kN9TyxoZ5X69oAWDA9nzOO8EJ7+cIyQpk6wEckyBTUQTWOy6fGa1frXp7c0MCTG+r5y1vN9PZHKMrN4tTDKzhzSQWnHlZBSX5ogt6AiEwUBXWQRSKw+v/Bn77qndn4/n+DpR86oNH1cJ09/Tz3ZhNPvFbPUxsbaO7sJTPDWFFVxhlLKnj3kTNZMF19HkWCQEGdDJrehP+5HGpXw5EfgPffDAXTJ2z34Yhj7Y4WnvSnSDbVdwCwuKKQM/x57ePml5KZoXltkURQUCeLcD/89cfw9Le9y6e+73sw+1jILYGc4v0efz0e25u7vC8jN9bz/Obd9EccZQXZnHZ4Be8+soKTF5dTkDNxryciY1NQJ5u3X4XfXQ7164auDxVAbrEX2iP+LBnj8dHDvq27j2deb+TJDfU8/XojrXv7yM7M4MRDpnvz2odXUFmap6NIRCaRgjoZ9ffCW095ZzT2tHmnpPe0ede8HnI/5md/HNcH2U/Yh7OL2d6ZxSuNEV7Y1c9b7Zl0ulyK8rJZVFHCobOKWTyzhMWzSygvyscyMsAywTIgIzPm9mjrMydk/l0k1YwV1PrbNqiysuHw947vOf29+w/z4Y/v3QMt2wYey+zvZiGwEDgfINvfdwR4218mgmX6oZ0Rc9v2XR8b+tkFUDQLCmd5PweW2V4Dh8KZ3ucmkmL2G9RmdjtwDtDgnFs6+SXJAcvKhqwZUDDjwPcxUtj3doKLgAvT299P3e4OdjR3UNvcyc49HTS1dQGOTCKU5GRQOS2HuSXZzCnJYXZxNoUh858fgUh4YF9D70dvh4fddoPb9LRDx9tQvx46GrzHh8ufPhjcRbOhaOa+9wtnekfYiCSJeEbUdwA/AX45uaVIIOwn7LNhYMQd1dXbz2s721hX18q62lYer2vlrY0dA03aZxXnsqyyhKPnlrDU/3nQp7tHwtDZ5AV3e8wSe79hA3TUjxzoeWWjB/nA/VkKdAmE/Qa1c26VmVVNQS2SpPKzs6iuKqO6qmxgXWdPP+sHwruFV+paeWJD/UB4z52Wx9K5xRxdOY1lc0tYNreE0oJxTFtkZPqhOhNmHzP6dpGwN88/UpBH7ze+7gV6pH/f5+eV7hvkJZUwbQFMmw8l87zrkItMIs1Ry6QoyMlixcIyViwcDO/27j4vvGtbvQCvax24lCtAZWkeR1eWsHRuCUfP9QL8oM+izMiEwgpvGet64JGIF+hjjdCb3vADvW/Ymy33Qjsa3kNuz4NQ3sG9B0l7cR314Y+oHxprjtrMLgMuA5g/f/47tm3bNlE1Sgpr3dvH+p3elMkrda28WtfKtubBfpPzy/JZVumNuI+cXcyi8gLmlOSRkagTcyIR6GzwTv9v2Q57tg7ebtkOrTsg3Dv0OYUzYwJ8/tAwL5kHITWGkAk4PC+eoI6lw/PkYLR09fJqnT9tUtfCK7Wt1O4ZPPQwN5RB1fQCDikvZFF5gbfM8G4X5Sb4OiaRiDcKHwjvbX6g+z9ba/cdkRfO2jfESxf4QV6pefI0oaCWpLe7s5dN9e1sbuxkc2MHm5u8n9t3dxGJ+RUuL8ph0YwCFpUXckhMiFeW5pEVhCsIRsLeNEpsiA/8jAb5sLnyotkjTKn4YV4yDzJ1ka1UcFBBbWa/AU4FZgD1wFedc/811nMU1DJVevsjbN/dyVuNnfuE+J6uwZFrdmYG86fnD4T4ovICL8hnFI7vS8zJFglD286h0ykDYb4NWuuGHsVimd6ou7TKW8oWDt4urfK+DJWkoDMTJS3t6exlc1PHkBB/yx+F94UHf+9L80NeeA8L8fllBWRnBWAUHivcD+07/amUbd4c+e4t3s89W6Graej2uSVQOiy8o4FeXDmh14+Rg6OgFonRH46wY89eb/Td2DkkzJs6ega2y8ww5pXm7RPi88ryKS/MCV6Ig3dS0J5tsCcmvAeWbUPnxy3TOyplIMCHj8anTXX1aU1BLRKntu6+wSkUP8Q3N3aypamTnv7IkG3LCrKpKMqhojiXmUU5zCzOpaI4h4qiXGYWe+sDFejRaZV9AtwP9a7modvnle47Eo+GefHcgxuNO+fNxYf7vH88wv3e0TKRPn/daI/1x2zj/wz7//jkl0G+f7JWfhnkTkuq68ooqEUOUiTiqGvZy+amTna17KW+rYeG9u6Bnw1tPTR29BCO7Pv/0/SCbMr9IJ8ZE+Tl/s+ZxbmUF+Ukvl1ad9vI0ynRQxBjR+MZWd4XmSWVXhgOCVA/WEcK3EifH7ojnFw00TKyvEsK5PvBXTAjJsine0vsuryyhE4F6aJMIgcpI8OYV5bPvLLRz0IMRxzNnT00xIR3fVsP9f7thvZuNr7dRlNH76iBXlGcS0VRzkCAD4zY/duTGui5xTBrmbcMNzAaHxbgrbWAeUeeZOV4PzOzvZDMDEFGyAu/DH/9wO39PZYVs6/QGI/5t52Dvbuhs9mbp+9s8n52NQ+u2/WKd7+7ZYzPYNpgeOdP95p3DIT7COum6GQmjahFptjwQK9v66G+rZuG9h4a2gZH6Y3tPQzPc7PoCN0P8aIcKopzKC/MGQj5iiJvCiY3lJmYNxh04T7o2h0T5LE/R1rXPPL1YgBC+X5o++FdUgnn3nJAZWlELRIgmRnmhWlRLlAy6nbhiKO5o4eG9sEgr48GeVs3jR09vP52+6hTLkU5WZQX5wyGtz8ij86jR++X5IXSqylEZmjwOjHxcM4bhUdH5rGh3tns3e9qgs5G74qTk0BBLRJQmRnmjZKLc1k6d/RAj0Qcu7t6B+bJG/xQb/SXhvZuXq5toaGth719+44Ms7My/BF5zmCYR0fsfqiXF+UwvSA7GCcNTTUz74vVvFLg0ISUoKAWSXIZGcaMwhxm7OfSsc45Onr6/fDuGZhqaezoobHNu7+lqZPnt+ympatvn+dnGJQVDJ9uyaE0P5tp+dmU5oeYlp/NtPwQpfnZlOSF1Cx5giioRdKEmVGUG6Io1zvBZyw9/WGaOnqHjM69n4NHuGzcNfq0S1RxbhalBTFBnhfyb3uBHg312PuFOVnpNRUTBwW1iOwjJyuTudPymDtt7KManHO09/TT0tnHnq5e9nT10rq3jz2dvezp6qOlq5eWvX3s6epjd2cvbzV20NLVR3v36IfnhTKNkrzoyNwfpeeF/MD3gn0g8AtCTPO3TeUvTxXUInLAzIzi3BDFuSHmT4+/gUJfOELrXj/Iu7wg39PVO+R+9PaO3V2s8x8fftJRrJysDIrzQpT4S3Fu1uDtmJ/FuTHb5HnbBH0Ur6AWkSkXysyIa159uO6+sDdy74wdrXuB3ra3j1Z/aevuo7GjhzcbO2jb209bdx9jHYmcYQwL+Zhgz8vaZ/2Q8M/NmvQvWRXUIpI0ckOZzC7JY3bJ+E40iUS8KZpomLf5Yd4aG+57+4cE/c7WvQPbx17EayQF2ZmU5IWoLM3n3stPPJi3OCIFtYikvIwMGxgJzxvnc51zdPdFBgK8dW8frV0jB30oc3KmTxTUIiJjMDPysjPJy85kVkli2qal4dHrIiLJRUEtIhJwCmoRkYBTUIuIBJyCWkQk4BTUIiIBp6AWEQk4BbWISMBNSisuM2sEth3g02cATRNYTjLTZzGUPo+h9HkMSoXPYoFzrnykByYlqA+GmdWM1jcs3eizGEqfx1D6PAal+mehqQ8RkYBTUIuIBFwQg/pniS4gQPRZDKXPYyh9HoNS+rMI3By1iIgMFcQRtYiIxFBQi4gEXGCC2szea2avm9mbZnZDoutJJDObZ2ZPm9kGM1tvZlcnuqZEM7NMM3vJzB5KdC2JZmbTzOw+M9vo/45MfO+nJGJm1/r/n7xqZr8xs8Rc3X8SBSKozSwTuBU4GzgSuNDMjkxsVQnVD1znnFsCnAB8Js0/D4CrgQ2JLiIgfgQ86pw7AjiGNP5czGwu8Fmg2jm3FMgELkhsVRMvEEENrADedM5tds71Ar8Fzk9wTQnjnNvlnHvRv92O9z/i3MRWlThmVgm8H/h5omtJNDMrBlYC/wXgnOt1zrUktKjEywLyzCwLyAd2JrieCReUoJ4L7Ii5X0saB1MsM6sCjgOeT3ApiXQL8HkgkuA6gmAR0Aj8wp8K+rmZFSS6qERxztUBPwC2A7uAVufc44mtauIFJahHat2b9scNmlkhcD9wjXOuLdH1JIKZnQM0OOfWJLqWgMgCjgduc84dB3QCafudjpmV4v31vRCYAxSY2ScSW9XEC0pQ18KQLu6VpOCfL+NhZiG8kL7bOfdAoutJoJOA88xsK96U2OlmdldiS0qoWqDWORf9C+s+vOBOV2cCW5xzjc65PuAB4O8SXNOEC0pQrwYWm9lCM8vG+zLgwQTXlDBmZnhzkBucczcnup5Ecs590TlX6Zyrwvu9eMo5l3Ijpng5594GdpjZ4f6qM4DXElhSom0HTjCzfP//mzNIwS9XsxJdAIBzrt/M/hl4DO9b29udc+sTXFYinQRcBKwzs7X+ui855x5JXEkSIFcBd/uDms3AxQmuJ2Gcc8+b2X3Ai3hHS71ECp5OrlPIRUQCLihTHyIiMgoFtYhIwCmoRUQCTkEtIhJwCmoRkYBTUIuIBJyCWkQk4P4//qb3CcHrGcgAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(history.history[\"loss\"])\n", + "plt.plot(history.history[\"val_loss\"])\n", + "plt.legend([\"loss\", \"val_loss\"])\n", + "plt.title(f\"Loss: {loss.name}\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "5ad4ba20", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Prediction\n", + "\n", + "Let's predict some features and visualize them." + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "a1936264", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "313/313 [==============================] - 1s 2ms/step\n" + ] + } + ], + "source": [ + "embedded_features = model.predict(x_test, verbose=1)" + ] + }, + { + "cell_type": "markdown", + "id": "7c0df63b", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "### 3D-Visualization of ArcFace Loss" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "5aac5d98", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATwAAAFACAYAAAAh5cC8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAEAAElEQVR4nOz9eYxk6ZrWCf6+7zub7eZbuMeSGZEZud+8W97MvFV3qGqYQkAVTEGJUQPdggbRYgbRND38AyrEDFKBhhIMTUul6QHR3dWA2NRboRYUTXVR04y6qrh1697MyMjMiIx98912O/v3ffOHxbFr4WHubu7pERm3wh4plOnu5xw7x+ycx973e5/3eYW1ljnmmGOO5wHyiz6BOeaYY46nhTnhzTHHHM8N5oQ3xxxzPDeYE94cc8zx3GBOeHPMMcdzgznhzTHHHM8N5oQ3xxxzPDeYE94cYwghfkUI0RZC+Mfc/48LIbQQYjDx7+dO+jznmOO4mBPeHAAIIS4APwJY4CcP2E4dcqhftdZWJ/79Jyd4mnPM8bkwJ7w5Cvwx4NeAnwf+o+KXQoifF0L8l0KIfyGEGAK/QwjxghDifxBCbAshdg+L4oQQf0II8YkQoi+EuCGE+L/s+fvvF0J8TwjRE0JcF0L8noe/bwgh/ishxLoQ4r4Q4q/OQLhzzLEvnC/6BOZ4ZvDHgL8F/Drwa0KIVWvt5sO//QfATwC/DygB/zvwy8AfBTTw7iHH3nq47w3gR4F/KYT4trX2N4UQ7wN/H/g/A/8rcBqoPdzvvwU2gVeACvA/A3eBv/O5r3aO5xJi3ks7hxDitwH/Bjhtrd0RQnwK/B1r7X8uhPh5QFpr/9jDbX8Y+OcPt833HOePA38PGEz8+vdYa39tz3b/E/BvrLX/hRDi7wChtfb/tmebVeAO0LTWRg9/90eAP2Wt/R0ndOlzPGeYR3hzwCiF/V+stTsPf/5HD3/3nz/8+e7Eti8At/eS3QR+zVr72yZ/IYT4ceD/AbzGaBmlDFyaON6/mHKc84ALrAshit/JPecyxxxHwpzwnnMIIUrAvw8oIcTGw1/7QFMI8dWHP0+mAXeBF4UQzgGkN3l8H/jvGaXMv2CtzR5GeAWL3QUuTtn1LpAAy7O8zhxzzIJ50WKOP8BoHe4t4GsP/70J/FtGJLUX/w5YB/66EKIihAiEEP+HA47vMSLQbSB/GO39rom//1fAnxBC/JgQQgohzgoh3rDWrgP/C/D/EkLUH/7tohDi3/s8FzvH84054c3xHwH/jbX2jrV2o/gH/BzwH7InC7DWauD/xKiQcAe4B/yh/Q5ure0D/ynwz4A2owLIP5/4+78D/gSj9LkL/H8ZpbMwIlwP+Pjhvv8do6LGHHMcC/OixRxzzPHcYB7hzTHHHM8N5oQ3xxxzPDeYE94cc8zx3GBOeHPMMcdzgznhzTHHHM8NDhMez0u4c8wxxxcFcfgmR8M8wptjjjmeG8wJb4455nhuMCe8OeaY47nBnPDmmGOO5wZzwptjjjmeG8wJb4455nhuMCe8OeaY47nBnPDmmGOO5wZzwptjjjmeG8wJb4455nhuMCe8OeaY47nBnPDmmGOO5wZzwptjjjmeG8wJb4455nhuMCe8OeaY47nBnPDmmGOO5wZzwptjjjmeG8wJb4455nhuMCe8OeaY47nBnPDmmGOO5wZzwntOoLUmz3Osnc9lmuP5xWFTy+b4AYe1ljzPieOYPM+RUuI4zviflBIhTnw41BxzPJMQh3zjz8OBH2AYY8iyDGMMWmuMMcCIBIvPXQgxJ8A5nlWc+I04J7zfgrDWorUmyzIApJTkeY7W+jEyK8hvToBzPIOYE94cB8NaS5ZlY3IryKpYwzuMvKy140gQvk+AruuilJoT4BxPE3PCm2N/GGNI0xRr7SNkB+wb4R2GaQTouu44Atz7OnPMcYKYE94cj8Nay3A4pN/vs7CwMJWAjkt4017LGDNOgaWUuK47jgDnBDjHCeLEb6R5lfYHHNZa0jSl1+uxubnJ4uLiE309IQRKqfFrA6RpSpqmAOMq8GQKPMcczwrmhPcDjKIwYa1FSrmvxs5ay8bGBlmWsbi4SKlUOpHXLyK5/Qjwzp07nD9//pEIcE6Ac3yRmBPeDyAKbV1RhJBS7kt4WZZx+fJllFIEQcDVq1dJkoRarcbCwgILCwv4vn8i57WXAHd2djh//vw8ApzjmcGc8H7AMKmtm1wvE0I8RnidTofLly9z8eJFlpeXyfOc8+fPY4yh3+/Tbrf5+OOPyfOcer3OwsICzWYTz/NO7HwLQobvS2D2EmBRBJkT4BxPGnPC+wHBNG3dJCYJz1rLrVu32Nzc5Otf/zrlcpk8z8fbSilpNBo0Gg0uXLiAMYZer0e73ebevXtorWk0GmMCdF33RK5hb0GjIMAkSUiSZHxue6vAc8xxUpgT3g8A9tPWTaIgvDRNuXTpEpVKhffff3+miElKSbPZpNls8tJLL6G1ptvt0m63uXPnDtZams0mCwsLNBoNHOdkbptZCFAp9UgEOCfAOT4P5oT3jOMgbd0khBDEccy3v/1tXnvtNVZWVo79mkopFhcXxxXfPM/pdDq0221u3ryJEOIRAizW7D4vphGgMYY4jh85tzkBznFczAnvGcVkCju5DjYNxhhu377NcDjkW9/6FkEQPLbN5yEGx3FYXl5meXkZGBVCOp0OOzs7XL9+HaXUuABSr9dPbB1uToBznDTmhPcMokhN9xYmpiGOYz788EPq9TqNRmMq2Z00XNdlZWVlHEWmaUq73WZjY4OrV6/ieR4LCwtjw4InHQFGUcStW7c4d+4cQRDMCXCOfTEnvGcMWmvu3LmD7/ssLi4e+MBubW3x2Wef8eabb1Iul7l8+fJTPNPvw/M8VldXWV1dBSBJEtrtNmma8p3vfAff98cRYLVaPTESKghQSslwOEQIMSbASYnMnADnKDAnvGcEk9q64oHd7+E0xnDlyhXCMOS9997D8zySJHlmzD1932dtbY27d+/y7rvvEsfxuAAyHA4plUpjAiyXyydCQsUaZ6FJLH63lwAnnWDmBPj8YU54zwD2ausO6poIw5APP/yQtbU13njjjQN1eF80inMqlUqUSiXOnDmDtZYwDOl0Oty8eZPhcEi1Wh0XQUql0rFIqCC8va+/VwdYuMYUKETQcyus5wNzwvsCsVdbN/mATjqUFFhfX+fmzZt86UtfotFoPPK3Z5HwpkEIQaVSoVKpcPbs2bHxQbvd5tq1a8RxTLVaHUeAR1mTPIyspq0BThJg0Sc8J8DfupgT3heEve1hkw/WXvLSWvPJJ5+Q5znvv//+VB3cDwrh7YUQgmq1SrVa5YUXXsBaO+4CuXLlCkmSUK/XxxHgfm1wx7n2WQhwbob6WwtzwvsCsF97WIFJ8ur3+1y6dIkXX3yRs2fPHqjDO8g84ObNmwwGAxYXF0+0f/akIYSgXq9Tr9cPbYNbWFh4pAvk85LRNALM8/yRCHxOgD/YmBPeU8Ss2jopJWmacvfuXe7du8eXv/xlarXagcfej/CSJBnLVs6cOUOn0zmUOJ4l7G2D01o/1gbXbDZJ05Q8z09MAgPTCTDLMgaDARsbG1y4cGHuBv0DhjnhPSUcRVtnjOHBgwc0Gg3ef//9mR7iaYS3u7vLp59+yuuvv87CwgJZltFsNsf9s0X72N27dx9pH2s2mydKHCeJSZEzjNL9TqfD1tYWly5dAnhi11Gs8Vlrx5XfLMseiQD3GiHMCfDZwpzwngJmbQ8D6Ha73Lhxg0ajwdtvvz3za+yNRK5du0an0+Eb3/gGQRA8VgSRUj5CHEX7WKvV4ubNm+P+2sXFxRPtnjhpKKVYWlrC933eeecdjDFTr6PoAjkJApz8HCePV3yp7TVCmLtBPzuYE94TxDTfuoO2LRxOXnrppbF90lERxzGXLl2i2Wzy7rvvzvyA7W0fS9OUTqfD5ubmI90TCwsL1Gq1Z+7BLUhoWhtcu91me3uba9eu4TjOmMhrtdqxiHyaBAY4kACLz38yBZ4T4NPHnPCeEIq05/Lly3z5y18+8Mbe63Cys7MzjhKOgjzP+c53vsPrr78+fuCPC8/zOHXqFKdOnQIYi4fv3btHv9+nXC6fuHj4ScB13UeuI0kSOp0O6+vrXLly5VhEvh/h7cXcDv/Zw5zwngCKyl6hMTvo4SjW2V599dXxQ7mfDm8/FClskiT8yI/8yBPppw2CgNOnT3P69OmxeLjdbnPjxg3CMBxr5xYXF59KP+80zEJCvu8/0gY3SeSDwYAgCMYEWKlUph6zsNQ/zrnNCfCLxZzwThB7U9higXu/ba9du0a73R6vsxU4iqZuMoUtl8tPhWwmxcPnzp3DWstgMHhMO5dlGWmafmEEOAv2EnkURXQ6HW7fvs1gMJgayRaFp8+DaQQ4d4N+8pgT3gnhMG3dJAqHk8XFRd57770DdXgHYWdnhytXrvDGG2+wtLTE9vb2576O40AIQa1Wo1ar8eKLL44dlHd3d/n444/H0pGicnpSBqInDSEE5XKZcrn8SBvc3kjW8zy01jOntrO+9n5mqMPhkPX19fFApDkBHh/P5p33A4TDrNf3YtLhZL+RioeltJNV2HffffeZExEXlVHf9/nqV78KMDYQvXXr1hMzED1p7BfJ3r9/n3a7zbe//e1jt8HN8toFAeZ5Trfb3dcNWik1t8OfEXPC+xyYxXq9gDGGq1evMhwOxw4n++GgCG8yOjxKFfaLwGTatrS0xNLSEvC4gajjOI8UDp7VyKWIZJeXl/E8jwsXLoxT+U8//ZQ0TR8Rc5/UMCRjzCMuMDA3Qz0u5oR3TBxFW1c4nKyurvL6668feiPuF+HtTWF/ULHXQLTwz3vw4AH9fn+mwsEXiUlXm71tcEUXyIMHD8jz/ESGIRWEN4m5G/TxMCe8I+Io2joYRTPf+973pjqc7Ie9EZ4xhuvXr3+uFLZYE3oWUfjnra2tYa0ljmNarRa3bt1iOBxSqVQeqQB/0Q/uflXayWFIwHgYUqfTGQ9DmiTAWdcypxHeXswJcDbMCe8IOEp7WOFwkmUZ3/rWt4707T5JeCeRwhY6vzAMx07KJ+0+fFIQQlAqlTh79uxj9lGfffYZcRw/kSHiR8GsxYppw5CKdr6jrGXOQnh7sR8BPu9u0HPCmxFFYWKWFLbf7/PRRx9x7tw5ut3ukVOZwgD0JFLYyWHcCwsLJElCq9Xizp07DAYDKpXKmABLpdKxXuNJYq991DT3lEajMe5pfRomCMetzjqOM3Utc3d3dzwMaZIAC5I7DuHtxTQz1OfRDXpOeIegSGE/+eQTXnvttUPbw+7du8fdu3fHDidFY/5Rbp7CE+7mzZufK4W9c+cO6+vrfP3rXycIArIsIwgCzpw5M5ZdFNHT1atXx/q5k150P0ns556ytbXFhx9++FRMEE5ChwfThyEVRgifffYZruuysLDwRIo40whw0smn3+9TKpWo1Wq/payw5oR3ACa1dbu7uwd+4Hme89FHH+E4Dt/85jfHD1qRns56sxQpLHCsFLa4cT/66CNc1+W9995DKTW1CDIteioW3e/fv48x5pl3UCncU3zf5xvf+MYjJgg3btx4IiMkj9NpMQv2tvMVxZz19XUGgwFhGD6RYUjwOAFub2+zuLiI4zjj6/2t4AY9J7wpmGa9fhC63S6XL1/mpZde4vTp04/87aD5FHuxvb3N1atXeeWVV7h9+/axbqginb5w4QJnzpx55G+HEe/kovtLL700XnPaSx6fp/H+SWOaCcLkCMmTmKB2koLjg1AUc6SUhGHI6uoqnU6Hu3fvjiOwJ1XNNsaMU1v4reMGPSe8PTiKtm7S4eRrX/sa5XL5sW2KVqSDoiNjDNeuXaPb7fLuu++ilOLWrVtHPvcsy/joo4/4yle+QrVanXouR8HeNaeCPPbKRxYXF59ZA4G9IySjKBpPUNuvdewwPC3CK1Cs4RXDkCbb4IoCyGQ1+/MMQ9r7mgWmFUF+EN2g54Q3gaNo6/Y6nBzkXnxQ10SRwi4tLY1TWGPMkcwDtNbjFq4f/uEfnrr2dhI33yR5TD5wk9PHigLIs9o/O22C2mTr2GQFeL9r+KIIbxKTbXB7q9nXr19/xNDhOAUprfWBX9LTCHCvGeqz6AY9Jzxmt14v0Gq1+OSTTx5xONkPB6W0RQq7twp7FPOA4XDIhx9+yLlz5xgOh08tzZz2wA0GA1qt1rjrIE1TdnZ2WFxcfCYt5Ke1jhUV4OIaJnVzxRfJSRUtZsWsOry9w5CKLpCiIHUUOc9hWcm019/rBVgQ4C//8i/zySef8Jf+0l+a+XhPCs894R3Vev3GjRu0Wq3HHE72QxGx7T3OtWvX6PV6U6uwsz5MxdjGt99+m3q9zsbGxhcmLp40ECi6Dr797W/T7/e5d+8e1trxw/as9s8K8fgAoV6vR6vV4t69exhjaDQaJEkydfniSaFYTzsKphk6TMp5siybSuYFtNaf68tzkgCLPuBnAc814R0lhQX4jd/4DRYWFqY6nOyHvSntZAr7jW9841iRgjGGTz/9lCRJeO+998bR01EiwyeNwtqoGHQzrX+2SH+fRQdlmN45UQwQv3nzJvfu3XuExJ9UdH0SOry9cp7JinwxDGmSAA9LaY+CYn3xWcBzSXhHbQ/b3t5mOBzy1a9+9dAUdi8mCa9IYQ9ySjkMURTxwQcfsLa2xptvvvkIUUyLJp8VTOufLSKnSQflxcXFz73g/qRQmCC02+1xpbrdbrO1tTW2j5+sYp/UNTyJFHpvRb5og5ss6Fy/fp3FxUUajcbnsvQaDAaf24H7pPDcEd5RfOsmHU6azSb1ev3Ir1cQ3pUrV+j3+5/LzqkQpL711lvj4TuTeBZJYj/4vj/VQfnatWuPtI8tLi4+cwLoIiOYZh9/VPfkWV/vSa/N7m2D+/Vf/3UWFhbGRanPY+lVFFCeBTw3hFc0pRfh9VEdTr73ve8dK3rSWnP58mXW1tY+Vwr72WefMRgMDrSWOijCi+OYBw8e0Gw2n7ke2r3Fg8n1psuXL5Pn+VMxEE2NJbMWVwg8ebAcaRoB7TVB2CsbOW7V9CRS2qNCCPHYMKTJJYmjCLqLa38W8FwQXpHCFsNbvvSlLx24fVEMmHQ4OeqcCRilsNvb21y8eJELFy4c69wn1/zeeeedA4lqvzW8nZ0dPv30U9bW1sbpyrMwg2I/TGsfKwTQRdN9ce4n1T2RGsv9NAMLCDjrufuS3iwp5n6ykVar9VjV9LAo9osgvL2Y1gY3Kegu2uCmeRrO1/CeIiZT2P1arAoUDid5nj9SDICjEV4RkfX7fc6cOXPsD7sY8DOrecBewrPWjqvKhcav2GavcWURQS0sLDxzFux7061i9GIxQtL3fdI0HZshHCd6zawFC2UlCbUhsxaP6cc5jg5vUjayt2paRLH7eec9C4S3F3sF3Xs9DYuOlu3t7fEX7FHwi7/4i/z4j//4FUABf89a+9enbSeEeA/4NeAPWWv/u8OO+2zd2SeIado6x3HQWk/dfjAYcOnSJc6dO8e5c+ceu6FnJbw4jvnggw9YXl7mG9/4BteuXTty5bSw8r5+/frM8hd4lPCyLOPDDz+kWq2OU+liOMxeyULRgN9qtcYtbScdQZ0k9q6dRVHEd7/73fHgneM4wLhCgIBQGxAPf94HJyE83i+KLYoGkzKezysROSqOc32T6TyMPpPt7W1+9md/lu9973tsbW3xkz/5k/zYj/0Yb7755oHH0lrzZ/7MnwH4ceAe8G0hxD+31n48uZ0QQgE/C/yrWc/ztyTh7aetm0Za1lru37/PnTt3xg4n0zAL4U2rwh41FU7TdOz68e677x7pRi8Ir9vt8tFHH/HKK6+Mv4EPIt3J9Rj4fgRVpCvPegtZEAS4rsuXvvSlqaljvV5ncXFxqt6sgCcFZz135jW8k34PpnnnFWtmrVaLOI5ZWlp6Kn3MJxFRlkolXnzxRf7ZP/tn/ME/+Af56Z/+aT755BP+0T/6R/zMz/zMgfv+u3/373jllVe4fv36DQAhxD8Bfj/w8Z5N/yzw3wPvzXpev+UI7yBtnVLqkQhvP4eTaTiIuCZT2L1FhaMQXiEKffXVV7l27dqxTB83NjbY3d3la1/72rFT6WkRVGEgULRfFRHUszZAaFrqOKk3O8gBxpNi3zR2Ek+j02LSBCGKIl5++WXCMBwPED8JE4T9cJIaPBgVAL/2ta/x23/7b59p+/v37/PCCy9M/uoe8M3JXwghzgI/BfwfeR4JbxZt3ST5FA4n01xFpmE/4oqiiA8//HCcwu698WbRxllruX37NhsbG7zzzjuUSqVxKjzrjay1pt1uUyqVeP/990/0ht3rQNzv92m1WmMDzmazOY6gnmYHRVFVVQe8T9McYKbZRx0lcnoaMpFJGGPwfZ9arbavCcJJGgc8CcI7ypfvPtnI3l/+beAvWGv1Ua71twThzdoeppQiz3Nu3brFxsYGX/3qV2f+IKYR3ixC4sPsoQqHE9/3HzEhOIqPXhiGfPDBB/i+z8svv/xESWey/apYe9pLIEVq9iS7PiarqtZa8hn3m2Yf1Wq1juQA87TNA6YR7DQThFarNdYxTlbhjxqFn3SR5KitcefOnePu3buP/Ap4sGezd4F/8vBzWAZ+QgiRW2v/p4OO/QNPeHstag66EbMsI4oioig60OFkGiYJ76AUdtp++xVKer0ely5d4uLFi+PF3r2vN+uc27fffpsHD/beE08ee0cwFgRSdFB8/PHHLC0tje2XTgqTVdV+lqFnSEWnwfO8qdq5ve4pk8TxLLilTGJSx1gYBxQV4E8++eQRE4SFhYVDjRxOMsI7zvCo9957j88++wwhxEvAfeAPA//BnuO+VPy/EOLngf/5MLKDH2DC25vCHnYDFg4nruseWiWahoKAihR2ZWVlJiGxlHJMyJPnfu/ePe7du7fvWtthfbGFAcEk6a6vr3/hvbSTBBJFERcuXKDb7Y4jj6KAMMuDdxAmq6oCcSI38kEOMEXDfbPZJI7jfb/EngSOSrDTTBCKCnAxcmCya2Jv9PUkZDBHSjsdh5/7uZ/j9/7e3/uvGMlS/mtr7WUhxP8VwFr7/znuefxAEt5R2sOstVy/fn3scPKbv/mbx3pNpRSdTofbt2/v29o1DXtT2jzP+fjjjxFCHLjWdhDhJUnChx9+yMLCwiNi5GfJPAAYm1bW6/VHLORbrdb4wTuug8pkVVVoTfeAqupxsdcBppDvbG9vc/ny5XG7VSHfedJLCceFlPKRKvykk/XNmzfH65zF53DSa3jHOfef+ImfwFr72uTv9iM6a+0fn/W4P1CEV2jrPvvsMy5cuHDot1Acx1y6dIlms3lkicckjDFsbm4ShiHf/OY3j9TbOVm0GAwGfPjhh7z44oucO3fuwP32K5IUldzXX3/9sYbsZ43w9mKv+0ie57Tb7XG7kuu6RxohWVRVc/v5DBOsNqAtKIFQ+98jk/Mz3nnnnfH65fb29tg84Fl3gIHp09MKE4TPPvsMGFXqe73e576ONE2fqV7oHxjCm7ReX19f5+WXXz5w+/3MNY+KIoUNgoDTp08f+cMriOvBgwfcunXrQK3fJKZ1Teyt5B62z1487bWnw+A4ziPtSsUA7qO2v32e67LaYLopGECCbHgHkt7k683iADMpgH6W3vtJ7JUh3b17l263+4gJQnEdR9VhFjb6zwp+IAhvmrZuv5t80uFkP2eSWR+QSXeSLMvodrvHOv/t7W3CMOT999+fuVo1SV6FXtDzvAOLLfsRnrWWmzdvcufOnbF+a2lp6ZkbxLN3hORR2t+OTSbaggHhK2yiH0Z6h+827fWmOcBMVk6fZQeYSRSReOECvbeQcxQThOO0lT1JPNOEt5+2rhAQ773p9zqcTLspi4jrsKE6BWkWBYGdnZ0jmweEYciVK1fwPI+vfvWrR3ooi/Ms0uBZ9ILTCC/Pcy5dukQQBPzQD/3QWK9XyDCKyVeFDONZwVHa3z6X+YESIBmRnXz48wmd/2TldLJ39qOPPkJr/VQcYI6DSRnJfoWcWWcZP0vWUPAME95B2rpphLexscH169f50pe+NF4jmobDCG+yCjtJmkdtEdvc3OTatWvjKuVxms23trbY2Ng4dho8SZanT58myzKklOP0pfj2noxCTqqKetI4qP2t2+2Spil37949cvubUBLZ8GZaw/s8mNY72+l0xvZRRVT1LPQva6331e7t/SIqiLzVanH//v2xc3JhHDocDo/1RXqYeYAQ4vcDP8NoMSIH/jNr7f/vsOM+k4RXNP3vZ70+2SKmtebTTz8lyzLef//9Qx/SYt9p2x1ksDkr4e2NDuM4pt1uH7rf3mP0ej3iOD52GryxscGNGzfGZDkt1Z389i586CarqMAzayIwue6UJAmXL19GSnms9jeh5Exp7Elir35xb//yZOvY0y5EHUWWMknke52T/+pf/av86q/+KvV6nV/+5V/mh3/4h2cyc5jRPOB/Bf65tdYKIb4C/DPgjcOO/UwR3qzW64XN02EOJ9MwjbimpbCz7LcXhVPKZHRYrD3OiuIYjuPw6quvHinVKSrCV65cGV/LUaK0vVXUaSYCRRfFcRfhZzXZPAqstTiO80y2v82Kaf3LRfQXhiGXL18+9sjFo+LzyFImO23+9t/+2/zTf/pP+cVf/EV+4Rd+gb/wF/4C/+Af/APeeONgXprFPMBaO5jYpcLjrWdT8cwQ3lG0dUII1tfX2d7enjndK7DXQKBIYU+dOrXvuh8cTng7OztcuXLlsTazWXppCxT+d2+++SYbGxszXtH3obXm7t27nDlzhq9//euPXctRq5mTD+FJpL9HMdk8Kiav6xHh7QsvotOcbr83tf3tacpHjkL2RevYysoKly5d4vz581MdYJ7E0sNJCo+FELzzzjv89E//9Mz7zGIe8PDYPwX8P4FTwO+d5dhfOOFN+tbB4e1hhXarGIB91MXeSeI6bEbEfvvtPf9r167R6XSmVoVniQwLo87d3d3xMba2to60ZtjpdLhz5w5ra2tcvHhx5v1mxUmkv0cx2TwK9ougC8mJMNB0qlQvLHBGCmyWMei0p8pHnlTh5rhkX6w3T3OAmRRwFxHsSYzA/KInls1oHoC19n8E/kchxI8yWs/7nYcd+wslvElt3SztYYXDSTFs+DiVLaUUWZbx6aefHpjC7sU04io6Hgph835V4cPMAwqjzklx9FFExHfv3uXevXu8+OKLT63QMEv6G8cxURThOM5It3YEk82jYuq9MyE5SeKcjSgDV4IQnD21Ou6f3SsfqVdrLDYWaC4t4AUHr//NKlo+LtlPi7amCbg7nQ67u7vjEZj72a3PgpOM8AaDwWN94odhRvOAMay1/5sQ4qIQYtlau3PQsb8wwjvKTNhJ0e1Xv/pVNjY2jt3LWBQ5zp49e2AKuxd77eGL3tzXXnttLDydhoNS2sI8YNKoc3K/wwivsKS31vL++++zvr7+VHs8JzEt/f3ggw+4efPmIynYqUYTXOdYa3j7kcve96lIHR0BRliyKCMTFqsUnhB0tWaoDZ5UZBYISqyePccLL7yAznL699u0tzusX79PFlgWlqa3v1lt0K0EcguOQC36+5Leccl+FvLZ6wCz1279qMLhk4zwjiNLmcU8QAjxCnD9YdHiHcADdg879lMnvCKFvXLlChcvXjz0w0zTlI8++mjs8yalfGwdblZsbW2xtbXF+fPneemllw7fYQKF60kh4t3e3t6342HvftMI7969e9y9e3df84DDUuGCUM6cOcMLL7zwiCB7Gg4TbJ8kivS3VCrx1ltvoZR6LAUr1tBmrf7u7YjIay65HEWNk9c0mTrm1mJ9gWMgl5LMCu7FKYIR4bhCsJ3no2TJWM4ohWsttWqN+lITm2hMWdAZ9kbtb59dx1MuWZyOUmHlj1JmT2KHBll1EKXp13IUR+VJHMdsdL/paQc5wEzipFPaoxLejOYBfxD4Y0KIDIgYzbQ4NCV6qoQ3qa3b2trilVdeOXD7Iop69dVXHxmAfVTCm6zCvvDCC8dapykI77vf/S7lcpn33ntvpgd1L3Fprfn444/HUdlxzAOKAslezeGz2EtbeLlNS3+LITwzVX/3TU9hYeKai9TRFYJWlmOAhqOIjKEpJShBXUhCY9jNcjJjqWEZtBJiT6GkBAQ8FCI7vstKeYXlxSVMNyWOEj7+5DJ3bt1m2BmykAbUFupUS1UcDr6vZnVUnvb+HRfThMOFALqoYE9aRzmOc6Ip7XEnlh1mHmCt/VlG8yyOhKdGeHtTWNi/ajjpcDItilJKkSTJTK8bhiGXLl0aV2Hv3LlzrOiw1+sxHA6npp8HYZKECqPOWWQ0+83fmJxCtvfb+VkkvGnYK8EohnBfv36dKIqo1WpjDz3XdUmNJcWihMVL9MP0VFB5uB6WTkRBrhDk1nI3yUiMYSPNcJFkwnDOcWgkhrYR3NM5i1Wf3Tyn2svwEstKzYeyRFYdMiFIrcHLDB6AttjM4ioX17q8+crrpErQvt8mbPfY2b5P2L1JY6ExTh9PonvipK2a9lpHTermig6WOI7pdrs0Go3P/drP0kxaeAqEd9T2sFkcTmYVARfdDpNV2KN2TFhruXv3Lvfv36dcLh+J7OD7i+mTRp3FrNvD9ttrK3Xp0iVKpRLf+MY3pr4vBxFeHMdcuXKFUqnE0tLSFzqMe688YzICKZT7m7u7fHbnLhaLri/QbDSoliuccx18JRBaT10P86RgxXXIjMWRo/9qLOuJ5W6W088MdV/RSQyDKOF2L6IaC2oGykJywVUYAdt5jhxk5MbSdCQEEtmJ8QcZ/lASdRK26i72VA2xVOPVkosjodfu0tppcfvmbYSaPv3tKPKUJz2icdoIzG9/+9vj+9XzvPE1HOeeea5ayw7S1k0jvFkdTg5LaQvxbRiGj1VhjxIdFk37ruvy/vvv8+u//usz7TcJay1xHHPnzp2ZK8LwKHn1+30uXbrEyy+/fGDFq9hH5wajLVIJlCPH6cuFCxfI8/wRN5LFxUWWlpaeWjP7YfIMKSWlWh3XL3Hm9Dn6aUo06DNs7XLz9k22PY+zS4tU6w1wA1wpSYwhn0gVy0pSciSdJCNNDYkwZMCap8gzTZoafCno5IYYwemywkug7VpSJ8eLJd0s54y23NOa64lmmEjqUrPgWZQv6KY5OzEslzwyx9KzlroR1GWZ2lLA+eYZdE3SHfYe6Z6oLSwQV+sED1P3w+QpT3smreu6OI7D66+/DjDuFLp79+6xHGCeG/MAa+2YWA5rD5u0TN/P4WS/ffdi0kDgjTfeeOx1Z43wCpKZdcjPNBSyFSHETO7I085zfX2dmzdv8pWvfOXQG0cIgc4Ng3YMRmCFoTPcYWNznXfeeQelFNbasZvHYDBgd3eXjz76aDzNq+hGeFIP2aQ8o5fl9JKchqdILETGUJISIRhvkzkuTnORoLnAaQ2LYmR+8N2PbzCIY+JKhTMLFcLcsJaMaM+RgoqGq90YV0NkNMtlh6Ex5IEkNdAylsxYBkZzD/AUKCHJQ4ObxvQFmJ2EoYBACuSCZMvCbpKgtGSnH9F1fK5nGUsWjO/Rl4LVROOGFmsMSngsLyyy0lwCJYjThHs7u9y5exfiGFWpYBcXOLu0/9yJL3oId2GLdpADTEGA0740h8PhkRoDnjSeGOEd1BoGjIdiT66xzUoK+xFekcIeZCAwS8Hj/v373L59eyaS2Q9FVPXaa68VJfYjH2NjY2McXc6yHiSEwOQWjEC68Okn13BLgvfeew+l1HgQd7Ft0QReRH7FHNRr167h+/441TnyLFqdgs4QJnvsT4U8o5flbPUTUIptAfeFxQpwhOA136edj6QjFUdyRkhMP8W1Ak8pnJVVslKTVFo+a/XohV1u9ofcuPQhotTghWqZZT8gyjSpp0gjOKUtuSsIjWDDaPq5piYFVd/lfMnDEZblTKJ9ya1BjJtBnGWUEXRcRTfTrEvNUuDQriu+XHJ4NXC53UkpJYaqC6ELaWhQkUX4ApNraKcjqYqEoOFz/uxZnJWRdGc4GKIH3e+3v9UaLDYXaCw2cbyRnvKLJrxJ7OcAM2kcsNcBJoqiIxcJf/EXf5E/9+f+HFevXr3GdOOA/xD4Cw9/HAB/2lr7wSzHfqIp7UFrSkopNjc32dzcPNThZNq+k6RVpLBRFM00VGe/CK/QtWmtj9XFAdONOgsX2VmRJAm3bt2iXC7zta99bXbnDyFAWuI04ZMPPmb11CqvfukCWEGWaLQ2yH3sj/ZquYo2spvXrpCEfaqNRRaWVw9vZdIponsPsPjx9oj8+P7nUcgzekkOStEIXG4OYjaNZqXkcj9J6eWamlJk2vBl18Gk4BkLZYdhnKO14EGe08s0t6Viqb5AahVvnFllo91jfb3FzUHIXRXg1MqUEHi2xO1U80BaBBbHkXQyzdAIVhCsVjy6GHSo2Uo0TSHYxLAaadxc0PDLRIHkdCCIHNBAL86pGUvNk0TCIiy4ZQ/pjwoc5AaLGHntZRob5bgl5/vylKUFvJWHg7fTjO7dXbr3d7n76S1MVbG4vIjW+ql65x1FBjPNOKBwgPm3//bf8rf+1t9Ca82v/dqv8c1vfnMmUXxhHPCv//W/5uLFi28x3TjgJvDvWWvbQogfB/4uU1rPpuELER4XnmyO48zkcLIXk4RXpLBra2tTU9i92I/whsMhH3744aEV1IN0bHvX/I7zzdzpdLh8+TKrq6vjDoVZIYQgSWM+u3mZL33lDRaXR4WaIsVN84xK08VxD9dYlUolzq6tcK6UYEyNwTBkp9ceK+CXmjUWG1VqzSWEM5GO6Qyw4FXA2oc/PwpPCspKsGMs3TDFVQIlJLGxpIxs6ZYdxb12wu3OkEUtUAr0qQDlKoYaMk/iOgIwuDbHWljPch5YxdCr0Vhr4mcJC9oyiIZca+2SiwC9UGagQKYCYSzSkXw0jJHuaN3vrIGSFNwPU3YjzXc1yJIlCCPq1kW5kkUPvlUOyKxBaUPZCnAVXqAwQDu3lFxBoBQ2ysnDhyuM5uEXT83Fc9Qja3cKSbPRZOHUEjbR5AG0Bx3u3btHHMd0Op0n3v4G329lOw4mHWBeeeUVvvnNb/JTP/VT/P2///f5s3/2z/JH/+gf5c//+T9/4DEK44CXX34Za226j3HA/z6xy68x6sSYCU89wiscToIg4OzZs8dqhSoIb5YUdr99J1FYKX3pS186sIJ6kHC38J47f/48Z8+ePdL1wPerwQ8ePOCdd96h2+0yHA6PtP/W1hbtdpsf/uEfHptiZokGI3ACSTqwGG3hoABtsuBhRuQlgxp1KalV13jJq5DFQ3p3LrNz8xa3hyFy8TwLy6sjM07XBQSkQxAC1MSLPUx1MxQP+poUEMZycaFEoDW9XLOkFJE17CYZeZxQzxM8v0ykJMZVNJo+w0xjhMB3JFIKMIKKFKw6CqUcHkgNccKmAFP2qCM5a8vciHOibkRLGFJpSaRLvewjAdkesFj2uZvDQGg20PSBWMGihhjDWm4pZTkvG03TcVgXlqQGsSs5W/HJleDD1hBjDVLDV5SgtOBjwwyEJC8pHgwTRAzCM48WLPYYkXolj7XqGlmWoZSi0Wg8Fd/CkxQdnz59Gtd1+bt/9+8CowLIYZjVOGACfxL4l7Oe01OL8Ky13L9/nzt37vDlL3+Z3d3dIzsIT2IwGHD//v0jVT7h0TmxRSocx/FMVkr7zYotCgsHObccFBlqrcd+bsV6W7/fn1lTp7Xmg48+IM5iFpcfdQCWapTm5rEBxL4prc4NWaKJ+tnDbQTVusIpyIvvk5crYXl5ieUzL2KjDqH12Q2jsS9htVahVvEJnYn1z4lUN8ksm1kTvxQQpxoQvOh73CHFWvCtYEFmBGGHrJ2i6WJPryCDMtHDYsY5C2GeUsVwJ8nYziz3O0OiLOderrGAtKDiDAdLyQU3cFiKHbwytIA7CRBFGGtxLKymhqEUDMUoCW8FI2fJTEAdKCWaitaU2xCfytCuxXMUfQ961hKlmtRaTvke7fUhA5Pj+Rmi6SE9SRrnWAEVTxFZHumn3c+I1BiD53mPrZ2d1PS3vTjJNcO99/wsztSzGgcACCF+ByPC+22zntNTIbw8z8cPdLE21ul0yPNZ58V/H0UKC0y1QDoMRU/spC3ULKlwse+klGZWwjwoMpwUI09+s81qKxVFEd/53ndwF13qy3U2NzfJdIb7kJyUI6kuBKOoLYPEWKw2uBM9n6PKbkqe5sSDnPrKaHuDh22cG6WlygX18ItFjaI4Pexgu5uoUoNTrmTt9ZfJnTJXtrbY3bhGq7ON+s6vsHjuTRarPpV8iCg10XmMIIfMYIwlyzVCgIOgZwyhtpTynKW6Iqo36O90CSpgNSwoSVloRLTFTg43U/gshVtookFChVEvUvFOd42ln4YMQ+hL6DpltgGpQfpgKOECAwtX+xmhztBGkzgOQ1dSET5BCc4Cq0BVufSMoJtptpKctgKnL9hyU+oS1lND0g8JehotJYPM4Nccyo0A34BMM8JUI5R8rJ92mhGpMQZpBTbVYyKcZfrbcbVzJxnhJUlyqOJiL2Y1Dnho+vn3gB+31h7aQ1vgiae0hcPJXnnHUfRwBSZT2GK261EhpSSOY37zN3/zSPNli30LEiqMOmchzP0iw0J3OE2MPEvXROGf9/JrL5P5GTrWWGPJzPcJD0akpzFstlO0MUgpONMIxqRntAUsfskhHuSkkcbx1CjSU973iW4Cudtk0Omh0zrdVo+SN4RhSvDCWzRqZc64Z/lQw9nVJplO2fjsE3T3PqWghLv6CtWVRUIhkCFknRSUIq0IQm0oOxKLyz0hUTqhX1KcbVbJLHgI0iwl1mA9HzuI6EaSfmYYuLADlElpAm6YE5QFAy/nUyUJlGIYQhSBD1AateZ6wDCBbS1RsgSOwAgITAQmpG0ttbLHNvCS0Vx1JV2bM9geoB14MYfwbI3MCMrC0ncVzTJcTS0rDgTG8KIZRcZrQ0MmLZ4DbmBJDQeKkE2mkcZgnGzfqWrTpr+12+0jT3+Dk3dKOWpbWWEccPPmTV5++WWP6cYBLwL/A/BHrbVXj3L8J0p4d+/e5c6dO3z1q1997MIdxyEMw5mOM1mFPU6RY/I4N2/eJAxDfvRHf/TYIxcnjTonzT4P2m/vyMXr16/Tbrf3TckPivCKSvDm5uaoG8WRPAgfEOsYYw2ufPz9ybTBWkvZU0SpJtOWonZRpLBaW8oNj1LNxfUVyply4z9MTe1wiOgPkFJj0wi3ViNFIfNRNBjmOU60S9m4lJunEZXz5OYi/d0d1pOAKzdvs5VaSrpCVivzUlCmWi4TSkEgBNLzUKfWqKQh7UFKP9M4nkMn11wZ5KghdE3Iej9j2yjamSV1R8uTDlDBo1TO6JETIRkowSCCnJwSDlUbsR5abFkSExAF0DYWhaGZwUJZkWclBj7k7igF/oCMB9EApKC23aaWKETgoMOcZDekKgWB51CNDS4CjcHzFbijNjXVznEGGutDCuhEs20NVhuEkpwteY+Rns0N0hFHmqq2Vzu3d/pbMXNiWvvbSUZ4xxEdF8YBv/t3/26AT5huHPB/B5aA//fDQCO31r470/GPdDZHxMLCAmtra1O/MZRSM6W0R63C7odJ77pyuXysUr8Qgjt37tDv9/nGN74x87SsSfLKsoxLly5RqVT2bRGD/X30tNbjSvCkgcGZ8hnauk3sxY9EdwVcJRFCEKYaKQXuxFreKO31RgUO2J/sYJTeZiFO9zq0MzIZoJ0msREoPaRkQs4iyNMWOryHv9tFhhuYQZs+5whDF6fZ5PRSnaojuLvZ4Uq/z6WtbSoDl3q9yov1JueqHptZRJ51WAgkNZuwK1dYH2asd0IuehW20oxcKpZsn67R7EYCWSoRYrCEGJPQkooukNvR++mbjISUUI4EXDKyZGhAocTILiryBKdcWHBHa3g9RpFjhkulusCijbifCWwWsnIrpeV4vJi5yLKHH2vIACmRFZe8JCnVPDwxMiXIPMGDMAMx0kXKSNMQipCc1FV43h4CEgap1LGnqk3qLQ+b/lav1080wjtuW9lP/MRP8BM/8RMAYyfbPcYB/zHwHx/nnJ4o4VWr1X1FvrMIgGedRHYYioisaFnb3t4+8jEKh4+FhYV9e3z3QxEZztoiBtNT2mK974UXXuDcuUcr8a5yKTtlpN3Hj00JZNxFSMlKo4KrI2BiXU6nJN0QpEMSelQXvO+Tnk4fFi4ALERtSAdYbwHHKJrlMvXsBjZOyS99jAqqBLoD6S4ZVby0i7Ejp2HHZjh5i1YvolVrkFR8LtZr5Ap8T2B2etzbuM6lpINTBhEILq68xAZwJelzKkvYijXSSEICEqCdWhIzEvcyTLD4bJiIzA1oWYF2LFpDSYhRK1qek2HRUqFhJJ8RoAOBH42ixE4OmYJ2aEgk+Eh6Jdi0wAACKam4ZXTFxXUVJAm2FSMzzQXhsrBS5aVKBVXy8B/aThlpyT0JSlFd8OnlhtRYokAgUnCnBPQGi2h4yLJ7IlPVDpr+dvXqVYQQeJ43njb2efqtjzux7Eniia/h7YeDCM8Yw6effjqe2rVfCnuYt9ukdfpRIrK96PV6fPTRR2OL7aN+A0op2dzc5MGDBzN3b+zVCxaWUAeZDxwWFQohUGg+u/KreK5Lo9mkdu5NyuUytn0fEWpcV5L6pzDaRTmMUtj2TUR/HUwOVmD9KiZOUIMHBL4i7bYxyT1anQyRhGR+Bd1M6A82edAWXGisIb0Ksr9JHpdJnBpLus8LQciDvMKSv0LfGGyY4TgeXsWj6wVkUZfrkeVG6yrdUoOoVKPiwMuqTGeYcZOEtgP9ksDPM7AO2kgym9IGjBkFW3kGCIgspBhyR2BkBWwE1gExegxUBIHOkEbRsZIMiITEjLrhUBqEhEgIlg0MXci0wiLx3BLnGoqXYoOb5vTbfR50dhB1j6WFGsuLC5T8AL/u4uAQS4ErLGtCQW7wPAfPffy+MsagXIXwnszgob3ONbdv36bf7x9r+ttePGt9tPAFOh4XrWV7MZnCvvnmm/sSWlFt3W+9IU1TLl269Jh1+lFRGHV+9atf5e7du0eW0hQSgjzPjzVysTAc3dnZObTPeNp7VRiFnjt3jtXVVUzcR54KiI2iu73OreufEccRCyrD8VepBQ7Szb8vX9HZ6J9bgjSCrA+VJYxfIe8OMbmDTFuYeJc8svgM6HcTdF5BxIpQrxBJn5oSNCoJBJZte5/MKga5z4V0g9N1H9XvE2lBN8zopjE7uofOO2i3jiqVUbVFmjqnPQjZCTcJtcsDf5HIDZB4SCxN6dCyBkyGloLMjlJSpABrsQgMDkY4YJIRewHYh1VdARpNX+ZkRmCliwRcobCATUYSFQws+yNZS6A1GsGmFCws+PQSQdsYqqpOqyJZLit22l0G9+6RDyNKp6pUVxap1poEGbhKARZZc6dGb0/DsHUShT38SUx/e9asoeALJLxpa3hFCjuLhVKhp5v2phfdCnuNQ4+Cos3MGMN7772H4zhHtpaK45gPP/zw2CMXtdZ88MEH+L4/Ju1MZ6MqrHQfW6vbW+goDFSLJYEsGxUUMqMxJmZxZYlTCxcYpDE7Dz4hbG/S3o3IKz2W0lWWmjVKEsgsKuyj9AAx3EWnKXF7gHB8dBJREVvkvkPi1OmZJug2WTbAzx4gB0s4wiNdPE0/C2n17oKtsGYd4tywMkyo9LaQpoJjPaz18LyUpbDLQPv0ZIDqhMRyB2kFuVshWnyJrTCjrTPieEjDZKzpiGrVoqWgr3MkEpULhPTQxiKAXIElB/vwvrMwqtO6WGHR1hBJMAgUAikUrgVVGhFi4+G/hdQhqPsMBwmJdVESXhbQ1aCVoCdg2xqi3FByPJb9Bmf8JdwVQ6tmaQ873Lu7DpFhsVFjJahTqywivcfvj6fdSzs5s3nSO29yePis09+eO8KbNaUtUtgkSWauwk5LiSf7WL/+9a8fe/2giDIn7dPhaF56hXnAG2+8wcbGxpGNOZMkYXd3l7feemss58l0xoPwwdha6Uz5zCOkN2msevfuXdbX18epfCEszqxlx1UPK4OKus74eNgmryyB1+Brb57Fd31a25usf/obdHYSKq6iWnJZPVWnIjsY4UOWELgJWdpFLK/hmgF+4zzDgWEpyjHDASoR+PGQVIXc8ZdYTxxCXUWJJdbiHdzbD5CeoZ9EKD/A93PKeUBv+RUqUoOp4GnDqlEIJdlOHQIh2AQ6xqWhMxbyhHPZkCDP6TkBTR2C1fT8OmUt6CpIESTSjpzcizevuDUtIJKHqa1BCwHWRVlNzcKpUQMEuQYvBTfVBK7FIFhxIFUQW9AuCBd2hhkf9ZJRGlwXVCsuVU8hPcmdJGfbKkxzBeorrAwzwu0+Oxst7t69h11wWTw1IpBKpTL+AnuahHdQ1rR3eHiaprRaranT30ql0rFkKYVxgNaa69ev/8UpxgFvAP8N8A7wl6y1f/Mox//CIryCPGZNYfdiL+EVBpm+78/Ux7pfqlAYH04rlOwd5LPfce/cufOIecDW1tZMhFe0dbXau3x27Sq1Wu0R7WJmRj5yZbdMmIWP6e2KqPDy5ctYa3n33XdH79NDy6gsywmzEB04VEp1oiyil0Xk2lCKS3SymJ1uxounKqytLLKsLtJdEcTd2wx3drl1Y0B9eBWvvIHIPfL6yzhpjsr7SGk5I7dRlQaNwRZ+/xOytE/W8WjpNTayIS0XnDQDu0szvM9S7MHSGne0h6ZOKYcX7DbLXo+LvmDolImcGpVc4ccZIk/YwSdyAGmp+CnSGvJEMRQ5uZCY6gLLaYSbppQiyKsKKV1wQEsYKT8t2AnGyzWOGLWrGQEIjZPnVDQ0jOSBVEgDiQNuZtFKYHPNbSCujMjwlIJzRjBAkz5UP+cI4tySZZpBbEnCDM8IktAQVyRu3cOYKs3FBhe0JPM0XRFy69atsa1SHMfkef7UDASOIkvxPO+R2RmT1lE/8zM/QxRFvP322+zu7h7obzn52oVxwLlz5/B9/49MMQ5oAf8p8AeOc31fGOEJIciyjO9+97szuwBPYpLwiqLCSy+9xOnTp2d67b2EZ63ls88+o9frHaiNO6iyXBQHHMd5RDIyS9eEzg39VsSdO3fpdrq8/c7b3Lh5/ZFtXOmOJl9l4cjpd4/eLk1Ter0eq6urnD1zDpMDdkSiGIHrKZzcwRqIsmiUsrgl0B06SQfH8yhJB51qpNHkSZ/dbgs72EDGd7joDfHNLdI+JIOQ/uZ3cRyFChT+wjKBvstZJNYFt1IiTzOkMjiui5tEhHaFaq7x85xGd5uSFnTEgAcLX6dWUXTjAavpBo31AQulJfylc7yGQ0m7xI4kiXLawuB5EQPrECNpiIxQaGJVYmACcqfEiiwjdUSvJEY3uLBkQpCNPoxRVRY7+n8Nwghyx4J4GP2Rk0lLWwhykxGnOU3j4cWaWp7h5ND3HarAAiAULKKwWnMLzSCGRMBCYGg6kkhKNoBkkLPZt0hh8bTL3ZrFsQKVaCplhR8ErNWrnF5dw0oYhEMuXbrEp59+itZ6LB85idmz++G4EeVe66h/+A//IT/90z/N1tYWf+AP/AGEEPybf/NvDjzvSeOAh5hmHLAFbAkhZhq8vRdfSEpbpLBZlvGtb33r2AYCeZ6PiwpH8a4ryHJsuZ2mfPDBBzSbzQM9+Q6K8A6SjBxoSfUwqkvilE8+/pRyrcTbX3obz318H1e5nCmfmbqG1+l0+Oijj/C9gNXlM/R3I6RQIC2lmjfqp80MjnQ5XVvDSD06hrV8XWp2jKGU5gSehW5I3r9PEsa4ok3Z6ZOpbWwWI3SOX17CEQ7eVous/BrZ7l0GvU18Uko6RbkC8hClNVoPULbLUq4YVpbws4xa1qac3yegjRcv4aZrJPk5Uh2Q6fOk3RaNnQ2a0b/iohSkpVc5k0o+lQvs+KdpWYOfQWI0vVSwgyLxqmTSEoQhfgKBluTW4uUCEYCbg3FBWwEWVD4iOCUgkxY/fxjdWXClxc0FUqT4mWDoW3o2wpEKR43GPb4IVBnNBZQGEqHZHWr6GfhyJGmpSoWIDYEEM8zYGKQYDbiSi0YSKUVzMSAdZORK4kswgwwhRhKbWqOK53l8/etfH6+fFe1jjuOwtLT0SPp7Ejgp4bHv+5RKJX7qp36KH//xHycMw0OPewzjgCPjqUd4RQp7+vRpSqXSsbsmhBDcuHGDIAgOnP41DQXhua47LnAcNl8WRsSVZY/bHR02r2JfucjDVHM4CPn0kyucPXeWtdVVkBblTN/HVY8XKwrS//LbX+F73/6IQTsh7mfUTylsPnKbri4EDKMQk+YY7eLLYDShKx0SOLByqoJMUtyywHR6RJsPMNYjiSN09AAv2kBkKVmaIOM7pKnBREPCpqZdXsa1O8Q4VMJ7JFmJvs2RxsV1JCUsi9GAcn4Dk4f4eQ/faaHUgLqNeaH3G0TJEJkrtC5zpWww+Di9XV6VO3jGMkgvMhA9JFUyVSFxPNLcx/dcAjPyrOt7UM4gzzKC2BJVHHpKEgrQUqAtoyjOWgJG/bRWQOoKDCCsxRuNycARYIWk7wsyNA0jaCD5UqypOBaQLOWGag5nBvC6hFJJkmLYsnA+hhdcaCRQ1Tl9bfGQrBhLe6hRZUspNaORtgsBgVKIXGP7GusBqR71zzK61/f6FcZxTKvVGqe/hV3/fqMXZ8VkIPB5MVm0mGU9/SjGAcfFUyW8vVXY+/fvH6vsPhgMWF9fZ3V1lbfeeuvI51FUeG/fvs36+vpM82WL/SajLmst165do9vtHujaUqS0eyusRlt2t1vcXb/Na2+8yvLqAsqRI0mIsOPX2jujooAxhsufXGaYDPn6u19H5ApjRj2xUT8l6WdoabBphhc4bA436e8kSClYq52i0XQx+ZDN4QbaKqwRrFZWoLWOabWQDoi2RJsmcfQCHdWnXKmSao0n11G9CLavY0sNyn5CbDRdGbDVvEie5NhccoYS2XBIZzhyG67IHZrBBkrGWMCzOS/FDxhkq1gDt/0vsZG5hI6iEu9gcg83zdiQfa7V6wx1Tj/P0DJlYZBSzQyB9tioShCCUFoWLJxLMnbKIJWkZOS4XzUXkoa2VNMRs2lryYWDsgItofFw+qNjMjBglINyBIGFUOX0Fdiyi4+l7UBTgK5C2XFQjsNL3ZiahDMWnFzxUt2hlGvSYc4g02yZ0VpiBUNVKKyr8EseTq7R3QwzyLCRRpQcUHJURJ6CIAg4c+YMZ86cecSu/zjykUl8Hj+8vThqlXZW44DPg6eS0haDt/dWYQ/T0k1DYcW0trZ25HW/yfP69NNPx9HhrN9ok4RX6Pxqtdqh1vRSSqIk5k77HlIKpCs4XTrNrVu32Vpv89Ybb+N5j/avGmPGA3kKA0+kpboQkOmMQTjgow8/horg3Itn2Iw3OeWuYjVEgxSpIMkyOrSQ2wJTTol3DCpyyVVO7IXU2wNSG5HGYMUyqZJ0dvoEvR3ipE/cTtCiAosBUdokzhIumBiR58S5RS6uIJMBlCU9E2BsxCCosJn4OMZFihLLVkBu8USKlB0c2aGk1pFiNDPWWoXUDpW8x9C1ZE6X1GZsmUWWowWcZI2yluApQhvQIiAGNJbEdWmmKRhBSVsqw5TcsXjW0FcC3xj8XBF7YKxEWjsyDRAW6ylWQuiVoGJGaS4aqpkm8hWx64AAP4NybsiMJUgMqc1xM8iMYiMGUkvZCGppztqCw8tDeLDoIgKF5yrKgYQIypUSX7GCG1rju4qu51OrulRKLliLbqfY0ICSCN+iFvzRLJgZRAF77fr3ykccxxlHf4e5p5xkL20hXJ4Vk8YBDz0lHzMO+Lx44hFe4SR85syZx6qwxTrcLG9wse6Xpinvv/8+Dx48ONZ82eKb8Pz584cOAt+LgvCKIsnFixdnGtuY6ZxOa0iQBtT8KmkQ8xtXfoPl+jI/9KPfwBoeid50boiThCiLSR4+0E4gyWNDGMZcuXOda1dvUGGZRVUneSAQZw25M3L+DTsp1mpScmRd4BqfYZQQZiFK+8hU4ZQNudIMhyWinkMqMiqrVeQgw4TrVNwtlKvInRL9vIIwGWku2RqWaJpNanIXJQzCdilFIamRpHlK2wp8TxNJgZtaEuORW8XQhNRkH19kGOGzKz0yGyAzD2ktuTXEAroqJDU5d70FUinZLQe8EPZJnCFBfx03H7KYl/BTHyU8SiFkSOoYrDsyAz096NJ2ylgr0TbHWIeaMURKUcs1gbU4ShKWJRZLYEcEagT0/dHwI4XCIrCOxUtBWYFjDFkCG0lMpZdBzSPLQGtJKVQ0zvkYIXjVc0hdUIHDbklB3YVhTr1WZWmQUam6RALM4sj5xD70BBS+xMYaIR8SsIR9OgUPxF75SJIktFqtx6bVTUt/v8gh3JPGAQ+f7X+21zhACLEG/AYji0IjhPjPgLestb2ZXuOoF3EUGGP4+OOPeeutt6ZGY7P008Kj634FaSqlpq6nHYQiOpy8GY4CKSW9Xo/d3d2pDjDTkOmMnXiXKMmIbUjUi3jw2W3effMbvHDmBXRusBPLFDo3tHeGbAw22Gn1uN97wAIrEDtkNufu3Vt8dvU6L628Sm83GQ1CSiOczEdZhzzX5GrUS5WFGb3NGOXGdEybvGeROuXFxReoNKuEmz3SbkKFOq7XoKFLuMNbDHoxJtY4fsDqKY1NUpJmCL0+Mh6CXkf4CWQJGQHEOSYugVMlxeCJCN/mNKIBMjhL1enguAG+qOIqxe1gibtencx6LJDxWn6ZB74k8gSx3KEeZ6yFO6wMUlJlKCUDml5AOdvGx2dHKxJp8VVCzbrsSIcXh4osG6JrHk5mEK5AComRilwoQjnqkoiQuEaMBMVWk7sSaTTCShIHjBQkYkR0Qkjc1ODmAAKtFNYBzxpqxuXiwBLksJaF5D2H7evreG4A9QAR5WRrZXSmqSwFRAJkSeHUHVIpcZTEcxQ21VgswhWkVpK54NUDXNcZuyB/Xvi+/5h7yn7dEye5hjcYDI48sWzCOADgr8FjxgEbHMHSfS+eKOEVDr77YRbC208Xp5SayTIaHjfqvHHjxpGjQ2MMd+7cIQxDvvWtb83cNZGZDOGMtHNpJ2Vre4sf+aEfYXlpaWq6mqQpvahH3M9QSUDcyXHOWhwJ3/n4O7R3BzQWVogGOY7jUBN1yq7LucYi1lraYY/7G1v04wErKw28kqS2GHDz02sMWiEEGcM7XbrDVcqqhMoDhHJxhaQaZGR+SrtSQzkRIh9Q3x6yplNa3ZRUWwLdwbEZApc4ddDCJ0kMLl1KNid3zuFajZAJWUUS5hGJ59KwGVXTx/c75OoCfm6xGjq6xg3zGi3PoaZ2iV2PuuhTUbvs1BdQmWJoHErdHgt+zA+lFSLVZdP36HqSvrNMNVUEJiDNNCXpIa2glo9MOh1PYbGkwuIYgZUWJ9U4D2dRaANlFFZoElehEOQOSOE+jPEE1rV4uSVyJEGcUzMOpzKLZwU+8IoIWFp2uelmtPKQVq/Hi5FD4OW4KNKtEOUovIrP6mqAbTq4SuL0M4wBJGRlxaYDVkmEEpx15KFDuo+DyfT3/Pnzj6W/YRhy9+7dYw/enkQcxzOtjT9NPPGU9iAjS8dx9rWIKmbVDgaDqQWBWaPDaUadx2kR++CDD2g0GjiOc6QWMVe6OEqxHj1AWsVv/+0/Sq0yWsjNEk0eG9zyqJo6iIfcj+4xDHNaYY9UxQglEFbwvY9/E6EEX3/za/SzPqITUK9U8ByP6qJP4Pv0Bn1kYGieKTG8P0BYQdLR7KzfY9BNqFYapKmhubSMS43ubkS33cH1Y9ZqEbGqs3N3wO72Ao5u4pW79Lwcx2yyLXKEdEidBFcrjJZom5GgSVAMbROlIZHnWdLbtNwVrLNB1Q654Z8ho0Vsqrxk+vh5zMA26ekyi2GLNBuiKzW0ESzKNsu6g0x22FWwNDS0S5phVsZJljkT95BBl5pscEeW0KrBmUGHyG0SOpZBmBBkOVUFjdxBRwZlBUPlUEaiMsvZWJNLwUJiWEk0keOwU1L0zaiaawUEWU6gwdcWa+F0JkiEpWJd3gwtS9qyHChKQnJKSbpVhx1vtO6XlRxK90LyVkgpzrEKgkYJKyo4HYmjFEiNlSADF5tosoeRfll93/7d3ac+abV5zAr+uNib/v76r/86nufNlP7OgmdlxGSBL5Tw9iOtogd1aWmJd955Z+q3zCyEt59R56xkCd/vR33zzTdxXZebN2/OtN8YBqKtiKbX4Ife/SE8Z0TcaZzT24mI+hlqIPHqkt1onW7WwWl6LIkGSWtAlSoffudDzr/6Irqiidojv7vGcg3lCWxmsTl0tyOS1GBiQevBkHAHdtohrd1d1k4tc/H0RXq6SykL8KQhD2NWVhsElCipHfKwz73PbrGx6dDPq0hVoqxcFu09uhg2jE/FW8bGLtrtkPg1chWRDFu4IkFjwTy8NjdDmTJJusxApWBilswtYhwQPS6EuwTRJh2zhDPISdUKqgeqXEbrLlcqb7HZPEMkHYaBppQMaOTrGE9S0osILEYIdpyIXULC8iKJhFBJhq5hQE4tEQRxD2e1wflOn3Z5kWaqGUoPbQQrcU5VC+qZ4UZTUssEAw+G1lJPBRGGepKxmFq08hFmNH/iVGZYrwjIHj48JYeuL7njCLYDwQBLmufcWw141avzghXIQTpyId5t09vexFvyaDTr1BcWCITEWoMKwWrNIMyRZQdlIMsNKpdjg1AYkZ3ppqPq7T4OyJ8HUsqZ09+D1t4L44tnDV9YpwVMJ57CBukwN+GDRMCTDiPTbKFmifAmXYWLYwyHwyNFhoX/3drKyAS1IDudG3o7MclQj7offIlTsXjWxdM+URzTXCwR3HO4+tlnvPLqy5SdBiXPYVgOyRLDznAXGQuSXs6LZ86SpZpYxzjCw2w4yC2H7eGA0+fXOLN2ChyDaxXd3ZD2xiZ+XMHYFp6ziLEZSe7jZR7NoI5IqnRSRXuQc9mr43gh1g7pZwMCVaZUA6l8KqUcnaTkeYKykqZsQ7KIyhSuFqQqxcgQK3Ni6eGIFNdE5LZBzWh0OCBNGpTKJcrtFEKXO4svcE28gqGJ0hFRCfw05XZ9gZVkHUdskqgyXVEi0D6rgwGSgLY0OCWHUDmkniHXETslH1cPaAKvDVMqiSUSGT2vRiAMmeOyPADb0xjfReiUuyWJshAAC0bgm4zAClalRzMUxEoQudBzFSGKRWH5TRdqsSGXgtzAOQn1QLJc8ymXXOypKoGFurGstCPyLGOwNeRqdJ3cRjSriyyWllhbapDHBqeXI6KUrBcRtCW6FaMWgxGxaQuGIzkgHwWTJHVY+jtL9fdpOr3Mgi+U8CYtoiZtzw+zQYJHp49NonAULpfL+9pCHRbhFfNlPc97pEXsKJFhUSD5yle+QhRFtNvt8d8KTZ0bSLLY4FcE5ZJPN1Q4nRJB7jG8nxKHKWdePM1Ah7R3e/gPSkStHAskZKydb9AbdNl40KYdtunkHVpbXXJTJexqqqUyOhJYDMurNVRPkJZuk+UOnlcmTrro7AHb7Q5JqClVFLXFJuXQonOHxvJZXBMxGFh6eZk8jFFJSOQYyrRxnS5SlohVgM0dUhnjixs4GHZKZ0lxSc0KZ5PbNP11KiLEsYpMOPjlnAoG0Ag/Q2QWR1gCMtw4YliqEYkaZ6KE5UGFzWqA0X3aVYOwgluVZVq6iSzVWQsFi1ZQSg2JmxIr6JSqhL7HEgNix+flQc6rXej5cEfkLOYwkLCUQeTCnQrUDSxHFissQsLSMGYljPFdl2rDw9YCdByy7LhEOWjPpWJgc9GhH+YITyJSw5oQ+GmO0Jp4oYyXC7zAId8ZrTkHzTJOSdF8QUFlgV6nT3tjl/69u7g4NKp16kF1ND7SFSNSK4htzzjHozogH4TDIrJZqr+Fuajv+8ciuxnMAwTwXwA/AYTAH7fW/uasx38qKe1+KGQpaZry4YcfUq/Xeffdd480QWwShVzkMEfh/Tom4PsymhdffPGx+bKzRoZXr15lOByO/e+SJPm+xbvOiE2CERav5OB4ltpSgOc7LCenSJNddu/fxxMldM+ndy8DNJ20gzE5IvOo1wMcx6HX76MCQa5ztlptkmFMb5hi7Q7N+jKqamAhJl8ZYN2AW3cesLPTIc36sLRJxTVUrWDoKLRJSEyVaqBRlQzdgV5vk+V6iYuqzdBP8AhxvRiVtSDt0xYVbssVnHJEVC3T1V0qrkYnGUaDSi2RgdSASDQigIEK6KbLKAyJ77IT+IDEWp/VdIMgLnM2bZFoSX2oWLQOkfAJ8phG7rGl6nxaucAdcYYABdJjqEJit4TKclYGIdIvEzUkniPJ04BybDk1zHBtlWpqWZYai0vNahqJReaWqjYYmXHLhb6nHjqnWJZdh7oyLEWGhTzlEyy1koNOc3JhuYmhF0INOK0F52JY7PUoCdhWOX7FQTqKs9riOALhiIdFqhwUuG6N5qJiZWUZRYk4Senc3mJzfZ2sG5M4hp1Wi4XGKXzUvuMcTwLW2iOtue2t/g6HQ1qtFj//8z/P3/t7fw+Af/kv/yU/+qM/OpOiYUbzgB8HXn3475vAf8kR2s++8JS23R5Ns5+ltWvvvpOEN2nUedibu1+Fd2tri2vXrvH2229Tr9cf+/thhFcQd7PZfGSEZLHfpL2T8S2r/hq+930r9SzJuHn1DmunT1H2arR2upSXHAb9BENGSfgkIQw7CcunSyw3FxluZ7Q6fbK2YWO4TUu10OUcJQSqJqmtLdDOdtn5jYTtux1SCUZ69OjgeBXarU2cTgqhj3RzOtkini5R9QYYk1OrNfD0Ct3WDmLQJrUKx6TEuoINKmzbiwRZh3uBR7k0ZMFfpZZtMzR1HNmhrrbx8pi2rfHAPYfvlemLgFPRLo5rCbyMNIrZ9NcI3TLLOuKN5Aq16APS/D20WiaydYZpykDUuSsy2sbH131iZxHHcahGPsYaXmn3SZUk9RWGBN2NaQeCi5HgpVgQaDDWspQaEpkzzPpUhi6lskCmmi3XJ60G4CTErmI1HBIAbw9zmirHr1lUr4tXc2kOEm45DhvCskqZ1lIJERlkbjjlONAMyKQh6EQkC2UyY/CXPYQrINM4Th0TtNE6HBXSHB8hFSWvRPDmC6xePMOgF3Lj1g0iL+PBJ5cxxjxiIHDSBYHPI0kRQlCtVqlWq/zpP/2n+T2/5/fwJ//kn+SXf/mX+St/5a/wl//yX+b3/b7fd+AxZjEPePjz37ejcPTXhBBNIcRpa+36LOf5hRGetZbd3V12d3f55je/eeTydUF4hVGn1nps1HkYprWIffbZZ/T7fd599919W8QOIrxer8elS5emmo4WvbSP2DsRgmvGZLezs8Mnl69w/vyLlPwS1mocqShndXBjlGsZhDFpahAKetsh98s5xkLmZHQGLaqVEl0LpWVLUtnF9zzkSpVox4E4I9YpJsnpl0IIIFBNukkbIQ2iVmZ5QaGHLTZ22iijMKnB2W6y3gWT1VD5cNR/mwyw1sMdprAoSUt1KoHLKafBTiJJ+4pGNERIS4MecS6wnkucK0oiJTFV+vYF3DggNS63SotsBgFVbbBZjwtiHSVc1oMmbhxgkyHLYZe+TPF6HtuV01jXwwiXpaRLp5qic0UtHxKEhmGccqO6Sqvk0RcZEZrL1TqvdxXtsofSGg/LqVygdcIwdRkQ0g8EXTkqsg1cxVm3TNX3qMoUL7XkjkOQW4JuQmJBxJZTnqWbGJbCjIvawVQ88kGMHaQkjkC44KYGxxdIpZArzjg6s6KCtRlCuCMLq4cQSiJKEqkVXs3npVde5iW+P3+2kGoFQTBeQzuJ2REn2VZmreXs2bP8jb/xN8Y/H4YZzQPOAnf3bHMWeDYIb1p6WqyzCSFYW1s7llanEB5/+9vffsyoc5Z9i+iwiMoajca+FeHJa5n2wT148IDbt2/zta99bWp0WfTSTrN3stZy69Yttre3+fo7Xyfpaqx4OJ6vogkWFLbnYrMqJlOoUo7TEESbht07Q4RniPMeL796Fl3J2bh5a7QwFQC5Qz/uI/MyOhaQQpZD4Pm4Ow5DR+A4p6j7CTrS5G1QjsDxXMpUGYY9et0UHZfxCEi1wTEh1jap2gW0tdQ7ZUIGlOMUd2WJunTxfY+6bJEMY1ylcbwBwikxQLHr1LGZRQhDkkRU0pyqSAkzg1Y12ibglPbo1F+i49UpKQ83jzHSYNC0RBU3HFI1PZTw8AT4Xg0PRc0zaAWuSfBsikxTPOuTxYZcDNh2QGqFMmBlBp5giE+/4tCVkGhNplOaRjAIBKFUDF2DmxnKajTgqO8kCMciZE6uBeU4xlPgbIdYv4Ksl2hVSihXYT1JICSNboRjwK74SKVAjaqtIpcIFSD2iar2dj3snT876T8Xx/GB4xdnwUm2le3tspjl2ZzRPGDagWYuBz/1CK/b7Y7bsoIg4P79+8c6zs7ODmEY8v777x95olkRqRXnMqsV/N4Pba+g2QpLmIWPWTdJKUl1SmYyVoIVdG4gk6RDzfc+voTreHzl7a9hDfQ7IQJJkqV00i7r3U02HrTIOwLH9TFd0D2LHQiysIvF8Ob7L+LXHdzMZ710kzOll8iihETHOK0ynbsJKvaReJigTymtITZ9XFFC+E3SWhdHJtiKy5L1SHpdol4fo3OiPCfPYoxMsZ7PYsXC8Bw2kmTSEpmMcHdILsCPB6xWDCIboocRWSbIbRXLAr6IWI1ybH6HnWHAFfUmsrRCWecEMQwbLngSYUropI40Ab6OGGgPX+wSmU1uVs7jiUUqusFqNyELMnxjOOu2iBzFVmO0/4brsFOuEZKQuB53anC+o/GdAdVOTFe5NBDUTAlch/pQc79qMRZOZQrHk7w+NLyUpgQmpCUcUh1RCg263ScLDZ2Sh6pXEIslXuomuGGLrFTBuEt0m02cZok7u0OyQZ9EStQwxhmWR14A2j5iA7WftOSwNq9yuUy5XObcuXPj+7kYvyilPNB+fRq+yLYymNk84B7wwiHb7IunRniF7fj9+/fHkVC/3z9yx8OkQ0m5XD7W+EalFP1+n48//vjYVvBRGPPB9z5keWWJN772BrnJudu/S6pTPOWxVhkVTVzpotHsJDtsR9sYbamEC7S32nx69Splt8H584tsXOvhBoos1iA0cZKStgXbaUi/HyJil5ryR5W5WJBECUp7rJ1eJB9a2p022S4kO2WMKwjzLnl5QOf+FnGvQqkUoz1LFitKqcCIHOWl5K4ic3Jct0TZ88nyEuXFNym7MWGrBzlE0qFcyZA1hXJzUlml7KYMMw83b+PmCVAiTyWxdFj2W2R5jBQJmia5CYiSiIqTEGdVQlNhvbRCXUv6JuLlbJ0Xu4qBX8Y4y7Q8CGyf1bhNJ68hwzt8KmvsuE0WUqgLi+/WqeaQCo3NcrzcxzMW3ySIxhLLxiO1Hkthgpc7vBZp+qKF39ule/o0cTej5wgq2qKs4OwgZaOUsaBzRLXCWu6yJmvkjksYDdjVgi0ZU/MNtiqpupoXShDaFAYhbu5ASRGXchKruT1MaducElDyJFk3J9+NEO0M4TnYRKOWfMjtvtKSoxCQlPKR8Yt77dcrlcrYP28/BcQXPYR7RvOAfw78J0KIf8Io3e3Oun4HTymlzfOcy5cvo5R6xLvuKDIPeDT9/MY3vsGv/uqvHvl8jDHcuHGDKIr4kR/5kWN9wO3dNt/99se8dP4Ci80FjLaEOmQz2iRQAVvxFnEeU/EqICAwAdpqym6Z3ajNzsYdrlz/jFKlTq+bsNHboW4beJ5DnhjSJEdj0daShRmEPjbWDJKUSuAxzHpIWcLFZ2cj4kGnQzLMcZQgsYYwTBBW4noN2oNNTG6Ie5LYDsDN6cQptcjQ1Q4iFOSLhnSxg+usUM0XOeMs4Pox215EZnZJbQcnzylbj9VaicRZw7cpaR9E0kV3DHkYkZoyXTzypIOvcrqpS0UKXOuTaxdhUkLbI8IjtwukKIzvo/MdEDmZJwn0EJ31qHQzSlkHf3iPLQy1UpOhUCBLnBsO8USOJ6DvlqmkgiDvEbkeLRK8ocuXOtepVJbIMw8jNJGGxdRQsi7b2mADj13P4VQnxQcCqfHynLqvyWzGklC4ZkDW8LjrO6hc0HckyAxPKNZNRrKzgSorGi2DKTnshgqiMtESNANJVXhEyQCNxXUlwnUgFQhPYqMU008QgbOvtOTzRFx77deHw+Fj9lFLS0uPuCc/KS+8WTGLeQDwLxhJUq4xkqX8iSO9xpHO6BgYDAZ85zvf4fz584/JPPYb1TgNRzHq3A9Fi9ji4uKxF2jTNOXy5U946803qTUr5PFoOE6uNSa34ECeaTI0gV8ithFCCqyxdOIOn977hOH9jMpiA3dYItUQ9zLKZU3geHiOxHEV/W5KEFXJtUuQuKOeyxyGeURJ1tEKUJDaHolOMUIQpRqReQzva5yywo0UXuoh3Iy+2KWVtqnrBmbYwokNjm0gfTAti+cpZGZxHYtwcnJl8ColpKoR5T20aNLphPjWx69YdnWOrvh4Ime5ErM9tOjcMrSWXJZY9ToE+RCtGigbj8hbJySqio2qrDiCTCUIoWhVzxCFKdtyjUbSIUaxEq2TDrukcUiulqmKAasbl7GySTOvMSyvUnarRAYc7VNNciqJpWRGLXpGGM6GW2hqVK3EM7CQpeTCZ9dkCJGSqzK56RN5AdrkdCWo/iYeq5Q9MKnljp/TCgSO55EoiyqXWKxVOHeuxqqRVGNN1t6h41h2kpjkzl1MnOL5HivlgKqrODvM8UuL2NgipMGah126QjG5JLW3ZcwYcyLC3ckK6qSAuHBPdl2XxcVFpJRfKOHBTOYBFvgzxz2vJ054ruvy5S9/eaprwrRRjXtRDMVZX1+fmn7OaiBatJm99dZblMtlLl26dKTrKOyp8jznh3/bt4j7OXls0MYQ9ROsdaili0jX0MxWkLnLbtKjtOBQLVdZUAts3trE1x7v/LZ3uHb/NmE7o1RxcZWk2nTxHUUajQSl0TBkmIT4mQcOuC4M+30ap2tYwGXUehS2MmITo4RCqwyFQlRrVN1T6CjCj12inSE5PRabK+QiJBAaR2ps2iGLfGLPMshyqmUHRZV0xyWxfVKZEtsBUTQgjgBh2dIOtTglxOKu1rjfrbGQ1oiFRmkPU14gzRPyjge6xIKQ2KBBpSyJU43QijKChTihLzISxyVVdTwSSllKOQ7p4PDAOUNd1PHFECdNaEqNVJaabSHjNmE6pFdewzglXFGD3GKsxiGntrNOrqCctjDVi8S1OkZ6bCtDNUpYCC2xMDimS6YEkacoZ9BMExrCUN59QFpdQMsSKnVYzQVD17BkHWweUq1WKfU1dStRmcatKlwpSHyXLSdEO4b8zk16gWDtzDlcz0WfCUiVg1d38AygLbLijFNay+MtYydZNZ3EXgFx4Z784MED4jgmTdPx+t9xHcmPs4b3NPDECc/3/X2/NQ7TtU2mwu+9995jH36x/2E9fUUVtGgRy/P8SKl0kiR88MEHrKysUCqV8AMPx3EwemTQmQxyvJLLGc4g3NEaj+srhsOYCgF5rAn7IRcuXuCFpRfQuWZRLVHVEm0hMzGtfhejwYnKuI5g/V4bIksS5yBScHJwHLrd9simvJqDspSXAJXRC/s4cYBNDXE7Jpcppp+TJkOUrFBzDY6WDBKLjfsYSkgnwy2v4Ff7OJ5P1GrzQW5wcw/o4Hg10kGH3LHILMVxAhqNZUzNEna7mM4OutUminNKygWjIDdYxyeijpHgZBrXLuAIRa4jHAzChJjwPoEqYz1J2wtwTAkRGsRQkCxW2VQVttM+dSelGfdQYsgiksxWEKVFVjNL2o8QjiFwDBmW7ZKDFRKosRKmmMxDBzWGQrEgAzo02Kl0yPrrtHyXlSSh7zZwsoShANIQt7ODqyQqKOO6Do6GJorFVHFKKwZJTtVYGtbiLQTQNejFOu6wx7nzK5S2t5Gej3/+Ig1fEUYxD7rb3Op3CGoNamuLvO7VKCUCHRtk/WFKO6Vl7GkN4S7ck4vBWo1GY7z+B4y1f/V6feYIcDgcju3onyV8oZ0WB/1tMBiMOx72DsUpUKwB7kd4RYvY5CBrONp82SKVfv3111leXmZjY2NEso5EOaDJiXWECh1cx6FU8YhMioktZiho93tcu34Dt+rzykuvkOmMMIpRTsLQzRiKkCjOsZEkDnNsf0iaZKQhZJEk1yGlwOBVmiRpm14coiKfUiwYqgRlHYxwqOsm1tUkiWIYaYTSmDTFygTPGHSeESUxxhn5smnhIITElkA0fKprVfJ+RpynRA+2iDpDZH4bmxtcF4TRxDrFHbSoBx5O2cdkA+qNMsJqbKzAM6AEibNAbBWukeSOQ1atIByIPAFJG4ylkrVGHpeRpBSG1IWi4gywFoxs8MBt4lUUfjgkS3OuNdbwhEunvkZ9t4UyiuVOSM1ZwMs1Pc9hIHNc6aBKJXJtIczxkxSvUqUlIZEuJtTYNMFJegRRG73iUUlSPFmiFuc4qgQyw8Y90A6nENhAUDI5wlXEJkG1dnCNj441qlnCW2qQ78RUA0FwZhWzvIDnBriBQ9AeIJpVMlfhmYRWt8OVzZu4jqLZXKZZXqIhAsSUlrEvYgi34zg0Gg0ajQYvvfQSWZbRbrfZ2Njg6tWrlEqlcfR3kJwsDMPnM8I7DjY2Nrhx48a+HQ8FDip6FIR54cKFR2a7wsEOLpMoujcmU+mCLKWUZDpjM93ABBZjLGdrZ/B8B+VIBv2I7fsbbD7Y5ZUXX+H2gxukcY4QEqElg15Ev5/QN30iEZP2Y4LcBy0xWqMl5LmlpBxcvw7SkliLTDxsqBjGFkfVwMnwRIxWEXloscYjF6N+0yxIMaaOTgxKKmwGVloiC0pHOK5DVWjyWJDtKJKexmqFyVyUdHGzHtIaTJJgRArWw8abREEVV5eJOgNUKClJF7/i4JRO0cstmY4xXo8cgdVgzJDI5lidU9Z9YhIqSiFdhyx1CQiJh/cpOYrcPUNoXmTolkh8wUJvSF1VUKKKyA0ZHtKpkyqFth5W+WRkaBJuL5UxWoMI+GYSUgpKGD3AZGVwPAK/gZQemVOl379KJAeocItqBCWvhmsFxsRIT+EPO2Q1n4pJ0LqB41SxevT+h4N1XKdMVUdw6jyyUcZ1FpHNBYJSCbkQoFsJeTsl6eRsGUlbKowVLHe7vOh5SG2IfZetzg6f3R0No1pqLrLQWKBULY3X8I6jpzsupr2e67qcOnWKU6dOPTJ79urVqyRJ8oh7yuS+x6nSFmi1WvyhP/SH+KVf+qXPgFvAv2+tbe/dTgjxXwO/D9iy1r49y7GfKcIzxnD16lXCMOS99947dP1gPwOBgjD3Wzs8LE0wxvDJJ5+Q5/ljE9Emo8PJzolBPCQzGQE+SZbw7U//HYNWyumVF1C+g82htxOjyXnQ2cC6kC8kBLGk7iwxLIXYbY8+fXrDDBP18RzAE2Q6oyRdAqPIU0CMZqmiQVtNIBxcT2LSEB07GEeCK0lUiEo0Ju2S5xoXwF3EN00wIblOGUQdhCrhbmrSQQvh5Zh+iywbUjcKIUtY4RELhVdrIh1Jd6tPvHsDawQysZxqniP3LSIwZHFCoCVa1XCExa8HVFZ9SDtot4sNodyL8NwKqRAomSGVR+qUcFUJWW6yOMiJKhHVXof6sIPJMvDKdF2XYdIm8iRW5eRYpHHxZEBXSCrCoz5oM/B9zLCFivokgwHKC2j6i2SupJqBJxqsJVW2KxWElbSrLm6mcZA4+BhpGIoUKwYY6+PikaY5w2jAhk1xU0VqAnwtcUtlZL2CXFtB+gEWS5LkRLsRqpuTxCkSzbnE0jUpywLKpxvku10WV2usrq5grSWKInZ3d/ns1rUxkeR5PpaZPA0cJkvZO3vWGDN2T7l16xZKKRYXF2m325+L8P76X//r/NiP/Rj/+l//61eFEH8R+IvAX5iy6c8DPwf8/VmP/YWmtAWstY+sk73++usz7ec4ziOp6V7T0KMsuBaTwdIs4cMPP2Rl+RSvv3Z+33VDGGnskixlt9VHWEklyjHVAd/+ze8iSy5vXHyZTmtIlmVkCehME6cJ+dDgS59S1RC6MUalLNZKyEWXG598RJRKStUqKma0LmYNeQaUKkjRJhcSIoNAoHVMpBLidJPEWrTngiqTugZHh8gsxMlShFYIBaa7gw0qSBHixBFKR+hckGZDSLqo+hqZAByPRCqcdIiDQOQpxoC7uETJKZNQAq+CjXbYHPQRiUuiFaKmWFOCko1J2y160kfvVjCuYUCVllti0asivAChG+ROgiCiakBoB9c6vJgOEYkkHHQpCWigMd110voplodbxKUyfm7RpSZ3ywLPpOSiBFoT6yE6SXGcMrZeGRVY0j6h6+HpKrVU4yYxVnqU/QDlV4hMhmxtYMMudvFldpoVjHDInIwVEyLzHbZsnbAM29bjFaExiSGtCErDEL0jsEEIa6ukseXuzoB0u48MJCtSIGWArnvUgyrVrQjd7SJdULXK+BkpRMQvvPACWmu63S43btyg2+2yubnJ4uIiS0tLlMvlJ7aud9QUelLcDKO17t3dXX72Z3+W73znO2xtbfGH//Af5nf9rt810+yXAr/wC7/Ar/zKrxQ//rfArzCF8Ky1/5sQ4sLMB+YZiPCUUuzs7HD16lXeeOONI82amIzwimHaCwsLh7aI7UVhtd7pdLn22Q1efvkCi41Fhp2E6sLIS6+wdHps/S8Xo1QxFex2u9z/zXu8ePEcYtGS9BJKTcVCo4y8PpomFocGgyHyhzglkMaSDXNEqmj37hHJPouLLxN1MwgtKA3GkvU0uJJy2aFrBpg8xmYSoXukFU0Ut8lEDT83WCtRIkJpjdICYTTCjB5yMgdlQ6wUCM9F+hanl4AYQj4gz7pkKkLkGbk2eLnFuj5G5khSBkkfORSEdPHDIYgYJXzcqsOw7JD0ulyPQhrtdUqZJs0j4sSQNkrgLGADS+qf4oEccsYu4Q426Q0S6qJOSoWFYYcgScHmhDYkcQP6KmMYOPTpI2VCq7xC1YJxfZaSDm5/QKm8yGq3BXGIHbSoJhYpQIgaZ7Qg699G1gypHa3PuTZhw6tjPAfju5zquaAcdJaRpwlidRnpGgRNcuFgO9tUuj2MB/1yhYbn4IQJeas7amyqWlAhsZAk9x9Qut0llIK8UmNlbYlEedQaAdXTX8YmEbJcRu4jeJ+MlJrNJpVKZexBF0UR9Xp9TDQnmfJ+XuGx7/ucOXOGf/yP/zF/5I/8Ef7Un/pTfPzxx/zNv/k3xz21s2Bzc5PTp08DYK1dF0Ic3gY1I54K4e23ZlZEdtevX59q1HkYijW8okXsuBo9nRse3F9np7XFG6+9gR/44ylhWaJJwmw8dwIrHklpXdeh7tW5++ABm/0NvvTWW5RLAUmWodwIXwQMtjIGu5r7qk0S5+ggwWtYhqaDm/sEZY871++z1FzkwumX6XYMKlDooQEnJhwIBJaSUORkWKFBaoxrMGlKnGjIJY4aoI0h8yyYEJUrBBn4JUwCOC5JIPFVlQBNbvu4JgahwU9AO2Az8C1xFpOkPTQubigwwpJSQ2WGkoFqtYZIBVJbch0x6PdJTEJDW7KSg+4KEqMhikhyF8fkiFqNDoolUafnrVFSClV5EVe1cNM1dJgxtCGh7ZOWy6gUhPTolUp0hCQ0GVU05WTIsgbSHCeTKAkyi6hEEW6/Q2+wiZF1RGmRwGuiGU1zu6V2MXmELwasxpq1sIqrqqTGoGQdGe0QqwHrwSI28BBVnyUlKXd2aEmF9jxOGcXpSLJQVYiwBZmHCRWqHiA8B2eni31wn8GwP7pnnAGt5RVkRZH6glKlhFebbTG/iLgmZ9AaY+j3++zu7nLnzp1jtZDt+xycoPA4DEO++c1v8pM/+ZNT//47f+fvZGNj47Hf/7W/9tdO5PX3wxcW4RUGAgBvv/32kckORoS3ublJr9c7VouYtaNh159e+YR4oPnSm19GCLBWkMdmRHDwyJhEwaMprXQEdzs36CUx77/zHoEXEEcJ2+EOUggGJkSGAYPhkJ3NLlmaU1l2cJWPtT79Vs6t3h3WVl6kulJHGYd2sIP2c3ppj4QEN2ogtcKmlhKSQHnkKkfLBJspXCXQoopxXEy6i5vFSJVjypI8MUhj0GaRTEV4aZnM1RDkSJuCkqS+g0OCrJTIZAa5RcYJ5DlhluMoj5JyQMTE5KTDHKnKSAnkEUJbApMQxoKh8DC5y0KWIrNRv6hxA7Sr8DB4UcJQ5MjSAp52CEUDpaFtwPMdpFsjiXt4eUgiA0xphfXGImk8IM27NHc2EUmMcpcpG8tKb5us36ac5ThBHbIQi8C0b+OZQuamSR0fE7bxoyFdT9LAovsbWK+ElB5lx8coF/IOjZbAmB2SUxfIak1IJWtGkBlDrdOlXvLQmwki6aEWJCKoIptLyMDDrQpWs4i+jrBhj7Qq0d0WtXqV2B0NBPem9r9Pvz/3EpCUclxFffnllx9rIatWq+MWsv1cf/bDSer+DptY9ku/9Ev7/m11dZX19XVOnz6NEOI0sHUiJ8UXRHiF9fnLL7/M1tbWsbzvtdZsb28jpXyssDALhBBEUTSemXvmrbPj+bDw/RQWIAmzMQEqZyKlNbBxdYOV6hLvvnWBLM/IkojeIEYgIFGYICdXCVlmcBxFnmuMtkTDlOEwpr8Vc/b0S6wsLLG7u4MI/v/s/WmI5Wma3o39nu2/nT1OrLlnVVZVV1d3V/Uy00y/r+YV8itafi0GSzADgrEGISRGIDFmZKQewyCBZKENYyH8xUgfBtkYjCxjsKwPGpkX6UXunpnuqX3NyszKLTIizjlx1v/+PI8/nDxRkZmRW1VWdTH4gqQrO+P/PycizrnO/dz3dV+XowzmECiKcU5a5phAYJymrnNKAYEOsCLCuRAZJihbggsJa0HpQOsmMj+AwiKQVCLH+Qxhp1g7RbiaKq8RQU0mM2R8mmCWI4opkdDEYYPCW4q8xFEtdWJxAKpCaIXNFuArVLOLFiHSpkRC4dMUn1ii3BIGLULd5NAXxI0W41bARqvB2mzBYjYmHWtmdUEaTtmoUnR0BkNFFSlumxglPVXU4WyZU5oYXzgqnTNutDk32GetJWnMR7jaERcLhHXUnXVwGSbpU1cp87Ak9DnaJBgVUCrYP7OBsyXdQrE5LZCpRYsapEYk29jFVZzNiG1MdzShfbhHUQrq8RRR5MRBF9/pYHSIaLeQrS1kt42IFbITIEWLoh2wbxXeBCQXtzBJxEJYNALzNK0Wax9bsd2/QrbKXX777beP/PP6/f4TaeieZYVXFMVnCv0B+JVf+RV+7/d+jx/96EcAvwH8P5/Jk+LncKS9desWn3zyCd/61rdoNpuMRqOnNhDIsow33njjSBP0WT6VnHP87Gc/45VXXjlxEqaO/WSavYiqWD7HVQ9vJXt57rnn6G/0uTG7wV62R506dNEiNJraO6R0uEYFzTlFK6PRjOj3exzcGWCKiAtr24QE+EwiG565GlOogmzksHMwLkBaCbrGeYf0MUWRI+uYINR4m+PKGukkpZMEdHB2eQQXpPgqxUsLDpRweFHh3BSJpxYVzBxlMMYUCm09wllSn2M8SKOpXYXzBXmoCSNDUmrAUyUNdBWgZUTTBuTMaWhL1AwhiigKARMN0lMnCZzaJi8H9K2ko1pUZcpCG6ooIq8UiZQUVU5VgFQQNDvIuEFUKvoo9kSDeH+fhvU0i4J2NaIuK6yqSYsFHa2xs328hlbSI9UKioLajLHGQrbHdh4RKkmrtFjhsGVKMHHIpA2qQdmIGZo+ZT7CzT07eUUUKmR/m7qc4VxOmu6TzsCbEBU5whueRhJQ3x4gggg398gLL5BsLFD5BLG+xqaThCjMwqJljQ/UPaE8D3MvfloH4uMZFBcuXDjyz9vb23siDd2zrPBW6YCfBT/60Y/4tV/7NX7nd37nI+A68Kt373kK+Ffe+//h7t//r8CfBtaFEDeBv+e9/9ePuveXVuEdN+pcWZ/Dk62XHccq5OfrX/86aZo+dRj3alUtyzJ+8Rd/8ZE6v+NY9fHKhWdwMGQwPDiSvaRVSmlLIhXhGh5fCXpBj8gEBG3BuJyw/kLChYtd1oI+V9+6gVGC3toO+aIGr6hnjmoumCtL6CMC7ynyKcKDrh1ZINBqQVofUoRzNkybuqrxVU2uHVQlgdMIHyGqBcIKvI+gzDFC46jx5EtRrfJgC6SXOAwKCYsM52qkDjHCILxFVPXyBWIhyS26VqAtVgqmjTVstfSRuyAMYaERvsIsapT2BLpJrSFOGhQ2ZzG5TpVoXNTgG50W48EBdVaznk0RVUgnKPCdBs1OEw72mFUlWXmT3bjBYppRhzFV3KRpHaJzhtwJtKypFrcQUZNRq0XRWke7DG8krSjABS2QME0CjFFIGdNazNAqwpuAqJ7j8l0kBQSCOgjANNl0Y7LBPj6uYHMTb1NcnSIbDQglay9sIzubZNKT7x0wWMwI37tBPCtoNdvIbI5HUrZatLp9miYhjELsIMc5jwiW1SDwyASyzys8Pu6fd7+GrizLBxLInpVbyudNLOv3+/yn//SfYGnjfvy+t1kaB6z+/pee9t5fCuGlacrrr7/Ozs4O586du6dMf1LHlONJZKuQn6IoKIriiZ+HtZZ33nnnqNG7It3KVssBxH0+dis468EJVCiZzWYscssv/tKnWblGGgIVcFgegofeWpOOSAiNQkWKuZyBscQtzdV3L9PubLB2eo2iqBkPUpy0TAYpw0nKMM9IbYbyDmsddeRQXtHM2lhbE5uEPFiGPKfllKI4pFApvo7IFUTeoEuLsCWCGk+NpcIrjQ9bGCQoD3VB4SuEUqgqRbgSS8m8zimVxhQ1Aodg+X6sihrtJa32FuOyQgVL7ZrymrQsaRiJlQopJZkSKCHYVzmxlsyG17DOQLHJ4PTzjPqnOYjXuXNzzNat67QDSTOrSQONq3K2Ogn9znNcuXkLuVeSFwN6VrIgJywd2it8nVHrgEVzjcrVuCDElDPC+SGFrdBlhVYwEQ5EiLZTpFasZXOCJMGnFm0a2HpKGFao0BLbFKKIiQ/Q4QYqbiDX2uT9mDroIhYZ2fAQuZEgEkc8nRLJmp5LoNElFYaDOwcsxnfQvRZRJ2RLOAIh8ZkFIZbuKCtLKHhkAtmz3LS4X0N3fwKZMYY8z8myjEaj8cxMC75q+FIIb3d3l6997Wsnetc9CeHVdc1bb71FHMcPrIg96XF4dQw+ffo0Z8+e5Y033nggZwIBp5JTD5CeVILa1bz7+od477j4/MV7GsJGGc62ztKP+7jaUR7CfDxkLj1hU9Hqt4h9ws23djnbf558LBjcXqBjRZ5MGc4mjLMFs2qGKyU+V8zDDCcXICxVYBFFTByGBDoEUZEzpMyH1PYAW+XUQpLYLiJ3GB/hncfbOU46rNQoL9CVQgSCwFXUzmN0gnclUmoiU5NXgkou+4BSCrSDTw8+IdJr8nROrA0T45jFCmUicpcRRRHDyR3CRUYhIfaGMtfUdYqyGoWmSgvqxYyrboqfTFH5DItjri2ZTamjGBWDcz0WuyPMeIRFUiVNVO1ReUlnfIiyTYRqo+qSUCuSWpAHATmOmRboIkMVJZGsEBKKrmYaSILQsxEGxMWcsqpBBkipIDvEVRVqY4vtSFLJABM0iURN7msWRUHYbCJbTapum+SXfoH6zh2sAlvVMJ+AFDQzRe/iGZB9FosJk+vX+PhgD6EDuuefp9dco1EbhBRLi/eqwhUZwmqEMQ/YRH2Rq2UnGQj89Kc/5erVq0fSl36//5nck7/slbinwZdCeJcuXXooMWmtH3mkXfXKLl68eKTNWeFJq8PjTimrft3q2ntyJqp0WendR3hFmfP+x29x9sI5alstl/fvg1GGjupQYZm4KToQWK+4M99jPr7BYDDmm8+/xv7BBCEFoUmwQcEgusVcpsxueIrUUedQypxSFRQNhzUpURVySoZEizmQ0hSahVzgixIv5HJDwVV4ZfHK46RHF+CkY9qM8UJgpaBXF2jToshmWL+UaghbEhkFtsIqyazZpVYBVS1ozfdInV269LoFTa9x3hGJTTqzObaAZieHCAYdzWye0xIel1syEy9jBj04qdAiQgrNRl7TSkfMpKGsE0j3cX7GIl7m0x62e6ReMyrnZNUaWZ1CvmCtLFG+SStUuKJCUWCmtzGtHt6VuBxaxQSnPB0pobdFaQJ0DWtFhdfrtMIIvRbA4RilWmitKaWH0y1kbTHrLarpEO1LdHMDW1QMN08zHVzDjUfseIdqJOA9frHAHh7iRimyuQGFx42n2LYheuUCnaJFq9PifLtNPpkwVYqbszssbi1odpqspR06ZYlC4XPQp7cf6OF9mcQRRdGRs5Fzjul0ynA4vMc9ud/vPzR/9jjSNH0mGRtfBL4SwuOHHUsftyL2OMI7KUx7hdXw4aScieNY9Qy/8c1v0Ol0uH79+iONB6QSSKOop568nnNrcBMZJBAJPsjfZzLIwQmSskuzk7Dvx0zElDJ0yHVDvg+y1ITZ8liRNVPWu12i3QhhLEKE+DiHIias5sg8xVUOvMEpj6k8iApbV5Ta44UlqBxl2KJ2C/x8F1EWgEB7QYCD0oJzaB2ADAixuLBFkYcE1uHUMuHNO0s2XYDco5CSoNEhDyN8w+C0QjQ0dV2gswqDpa4tQgSIqEOQRCjZoBMZgrzEFJ5unuKSgEoUHFKwP51hjEZKQxo6pPS0yylzLYEIWSpiZwmEpJIRSXODXl1S5IfoZIN0Y4t6tMdho8GGbCK0JhYeow1exuAVPmxQNSZARTHZR/Q76DjCtFrojXUoF9j+BkGvz+z6ADcc0h2nTJVHmwh9mFLtDZG9PkHUoJR3AAkzi1qPUaEB61GdDj5NcWm2nKTu7LBjlhkms9mM4a1b3Ll6FR+GdKOItWZEO9y8h0y+7ErpeMpet9s9OpGtpC+r/NlWq3U0/DhJ+jKfz7+SxgHwJU5pH4aTSOtJd2ofRXjWWt5++22MMfeEaR+/1jmHUYZTyakHenjHbaVWPUNbO+rCY22NrT9NHLvnvlrS2WgiDVx982eE3TbRZsy1P76NSlO0aWMtpPMChUOXAaFIqNSctm/hzAJXezI9R9QKIxos8hLpCrSq0LVEY3Fa4KI1ShGg8gonarRMsCyohEPj0NojZYs6doRSo62klp6wtngHlAsKIYkFCDy1swhvKcIA4yUhDmVzvAULTGOPdp7U18hao0WBvb7HJEyWVW+laYqKoPQEsiRpNUAGlI0mVRzgdEShK/oyohaeIvIUtSBIC7TXVEVBOp1juiGtuMWs18LEGf35jJ1c4GjCWpsDcuLZHaJ0QVAUaN1ctr86LepQIpzGzRwVNaL0iHmGaTSx3uMQlHFAM5YI3UBfPEMgBPS6eASq14fpBHv9OtoZ5HaXNJkhbg8hnxOUXezI4dwYQlCtFmr7FHYwXYqPtUFEBmEMemcHX1UIs/z76r3QbrdpxTF1o0FVVYwnE27t7fH+tWv36Oi+LHuox+Fx0pcV+a2kL5/V/HOFJzEPEEKcZblDu82yzfx/8t7/i8fd++de4d3velwUy13WtbW1e7JdT8LDCC9NU9544w3Onj37UGup4/0/o+4dVqyGG1rro56hrR2Tg4zJfkVdlky7Ke315ETSm8/n/OwP3uL06YvE3ZB5dMjpjU0O1ZRJPSCbWqKqh78TkMc1QVcSGk16OCNLRsh5CyMitJToUJG4FkYl+NpQK4tsWfJygS1qXBkgEo1IS1QhwVqkcAglkMLTzgpEoAjsmFwWCJvjkQg0CTXOCwq/bCk4VxMvxgR1BCpFaoOTAmUzRAW1V1RWoW1FIjyUM6wLQQd0IkNRBwTeE4SCOAxBSpQSCG/IkAStPsJ4hs6TRG1a5LSkIg3apFGEQdGbzkjWusRVxUiUCGXw85SokuTpIXunziCDhLB/CtMKkVfewylJo6qYuA0QGtPoIvMZbasQcoLQAaUtydIaXEmuU6IwIYoi5J0Bbn2Nej6nkho9nSJnc4RURJ0em7d3SXsS0+kS9NaY7FegLU4KTL+NVAa9uQHP7ywj4ZQAZx8guvuxIkRVVeycO8ep49XfcMhbb73FYrHg6tWrRzq6L5L8nnSy+jDpy8o+6o//+I+5ffv251p5e0LzgBr42977nwkhWsBPhRD/8b7Q7gfwcye846R1v/fck1x7//Hy6Aj6jeUR9GF4mCfe/cONFZz1OLu0QncWXAVFWYJ3R5WhrR37ewd88M5ldtbO0TAN8nGFdxpVBlzcPM/QzxjmM3zXozE4lSFDT1c0Ge3uYUYemONUQq0KfOHwTU3ZkfiJIWvMkLrEyhrWFQUW6yy9qokWAiHaQIpXBuUMvpI4b6nqApEs32heKWTu8bUk9+AlCAdSgqXES7Gs7GoJOKJK4KQkjztkDpSUxGW5DPuWAlPVpE7gygphF2it8d5jkcSyjem2qRqG+owgHUkKWZFXY9JuxPMbp9Azx0CFCGtBatYXS/fkoBWRFxW+06TohLQnE/JIsVYOSWvHLJH0Oh2chKnVVOMJUZohmx493cdU4KYHeCWQ6+eQYYDxOXW+IBsPybbOE3qPqiT7ucVnE7wOOFVVGOGwd/aRWtBsJPhEgbfgNdn+h8iWoXJ3SLZfQjbC5c+1qqhu3DgiO3P27EMJD3iAEFfVX7vd5uLFi/zBH/wBjUaDW7du8f7779NsNo96aU+7RfE4fNbj8/3Sl1arxb/6V/+Kn/zkJ3zve9/jv/vv/jv+zt/5O8/cPOBucM/u3f+eCSHeY5lP+/MnvMdVaVVVcf36dW7fvs13vvOdJ86pPU6WJ8lWnvTaFUajEe+99949w40VpBJIJXC1pyotVlr2igGyFiBg02xx5d3rHBwMObX2HHlasXtrgpYCMQpQ0wY5gtZGwuKgYl7kaCtpJF1ye8ginXGnuk20FlNPHYEQlEFKK9U0w4BCzFg0DSax2KpChp65mBP4NcgPCcqKOnFIW+PLHBfcNZOMQihSqGucFbTDBrnxxEJQLOYIC1YZvIYiqigFGF/hpMEbC5UkE4JSKnDQcRVl0keInMDXCGFpW0lpPd14jbjRJy1TKmkwjQ7etMkTAwoOBwuk0yRxl7AXkpIy9gW5hcQ6ep11kjghXEwp555qNCYrMtxoiotCgjyDcsxcCWQh0YcZRq0xKitSJRgHIUZEbHUSoipAFBal2hCH6IZECY+dZPRPn8bsHSBaXahTph4m12+gpxOqpEkcaDaMXH6qRU2U6KJ6AmREYcd4N0Q2X0QYAaaxVEoDLk2p9/eRYYQtDlH9PuoRH7qPgxCCra0ttra2HnqUfFbV37MQHQshePnll/nhD39Iu93mH/yDf8B//s//+alzp5/WPOCuY8q3gZ887t4/9woP4PDwECnliTbuj8LqWPowZ+PHXXu8wlsR7sNMDJa9uZi8biBGBfGaIqvEXS+8Oa+//wamjHn1tW8x2puRVnPKwqKNpCVaRLGhxmJLj48sQS+gU3RQGka3D1kUlhZtWGismlDaHJlmiMozsgXaJ1jfIcxqZGiRtWQhJYU+pBksYF4RFuAig4tiIKe2EqOBXBBLgStqhJbEyoP3OFHjAav90g6qH1LagmAhUbVHOInwnlhrQm+ZeChlCFmJF56q0SJepAhnWdOSxAZUtYc6oAg0eWmpRUA5nGPDGu9mmGaIcpYo2SC3NflkwfzmDeqozWCyx/b5b5LXgrqWpIuKurLUTjMvoWp3EEWJkYZ1HaFbGm8q8vmUwkFlPZUoqJo1RSNCpDXBMEL1twgvbmHPnGb+X/4LKgyw6+uwsY0QkuTGx8RaM9vcQNuacjxhN44JyxARVxg9onvmOdxigRhPqfOSat+TnHkRFTfu0c/5vMBmOTzj4+dJR8lVDsX777//RDGMj8KzDuFuNpvEccwPf/jDE7/mWZkHCCGawP8d+F9776eP+/qfK+Glacpbb72F1ppvfvObT339ivD+8A//kHPnzj2QivYorCpL5xzvvvsuzrnHEq7SkqhhUHOIghBqGE5HvPfuezy3/TzbrXOU2fKIGbUVcRDhwhKmHlvCyI44LA+Y7Tk6RR8noHQ5SWxINtc5cDnF3gyBJRjOwDqE84jKQFzh8oKFHaBlgipq8jBD2IC6siwaKWEQUhpPkoYEVqGVRcp6eVY1yXKjQEpspFB5BSYiCQJcCHlHUWw7/FyhnEPPIbnbNBc6RFUlnarA6RDJgqSzQaEqcl+RqJhIBHd9hwtK58AEMJki1hKgIM1yNDXSeVRc49OUoBhT7V+n2LtN0FhfagBDTRZ0SHo9/O1dgsLRDBSEmkaU4AmQQiCyAls7KicRooFtJpSdJlnf89y2J3CSsM6Iwy2C0y/gREV6sIs/d47MCcx6g0AH1PsjXDbGh4agligh6UUBURwxtzPyLc1BAwbVhDXrEUoTNs5ghUOlMRQO72rEXcazh6NlL898sW8trfU9TsSrGMZ33nnnqXdo4dlOhNM0fezQ4lmYBwghDEuy+7947//dkzy3nxvhHRwc8OGHH/Lyyy/z0UcffeZ7rGxoHtWvOwlSSsqy5A//8A/Z2tri/PnzT3QsOJKzKEOcJ7z9Rx+ydeoMXmqujK+wEW3Q6EaIaCm1cc6weWqN/OqISkYUsxZBSyGM5PBgwLnTW8SJZuQOaLs2PkioVQ8bL8iDBWJWoETAQi6rF+8rSAtKP6fyc4QIGEeSOBUob6grQR2EyBqkkAReYxsSWw1ABaQiJahC0iIDL8gCj+yGyNMha1hc3KY9qVC6woSC2taEjQauKBGuxNcZUihqm5MbxfrZbWQ6YTxZ2riLCQQYnM8wDvIyBSwN4yhUSDWbY+2C+WxK1CyYNgy6G1CnNY2qxE3H7AUZQih0t40eTwlEQUOGaKGxGBqRxvS3iBcZk8kEv58y7QYQhfTWmqizCW46QLsWunEO3emS3rmKm0zQeGyng801tQkRGlyjQ6A8rTInO3cOnc0wZUkYCNTZszTOrNFYb7DYL8n3Ku6MZphOSPNwhO7vIUSKuDuRVf11VLuFnc7wT7n2+DD4qnrkEOT+GMb7BwlxHB+JjL+sEO7PI0t5EvMAsXyz/mvgPe/9//5J7/2l9/C891y5coXRaHQkOXla84Dj90iS5KnJDpbK8ps3b/Lqq68+lenoalBy+/Ztrly+xosvvchMTTg4HDBYDJiujTjFGU5H2xSzGuEkSIEJJUZL5FySzhfkk5oXT11iY3MT28w5FfSZ2CETs8/+fJfy0ODd+tKmfauNjWYMRwNkuSCa1DRtQDdfx6uCWTwnqLdxIiCsY9a6jaWhQJ6h/ZyynC77UdpRNJdbCTqLMV4QCocwiqTUVFlF2OogGxapS1Sx9OATjS7tQJOVIw7HQ4I6wmnDdmsN+pqg1ae4lmHHB7jCUTuobUUlI+rpbWSrR8PEtJOIkSshFJTpjKR2hKkh6KxTOwiCkLyuqZynkEO6YYhdC9AtaCiNahlc2mcwK4hNmztbHabCoi20WQYqNQpBMG3S2hsQxIL66hu4jTXscEhVlkxGQ2TlaNk2ca+BDLrY04Z5qKmUxWiFmU8ACM+dJWw0CXQDY9ZJ1jcon9vibGDIZhPmBwPev3IVYzTdLKO3toYyGpflyMA81ODzSbCamPqqot7dPdoE0js7jxyEwIODhMViwWg0OgrhXlV/nU7nqKp7lhXeYrFge3v7M1//hOYB/w3wvwLeEkK8fvfS/633/v/9qHt/qRXeygOv0Wjw3e9+9zP9gI+vmX33u9/lxz/+8VPf49atW9y4fpN+b4Nu50GnlMft1o7HY+q65rXvvcq12ze5M9nnxuTG0jcuC8l1ji0sxcijI8jmGaPZFBUHtIIIHwi+8+qLbHTWqW3NLJ4zykfkesEsqQhY4zCZoCSE3SZpMOVWfYXbnZskvkHTtAkX24QWXN2kaUMCn2CbErFQ+HYAM0udTallDoGHJGEgJhSxxZORCHBC4NEkXtMQPQoJFotqh7iyDdMFZHP8YkgmAuTGBiUZNpUICWI6wmeKRa9B2Oqhq4LMphgdgS+pGyFpXkBQMpnnqDqEusLnNUbGVMYS6QqZtGlWhnYcMl9MIG4wC0KGcZMkremYmnJW0iClMusUQYJQEZNb17igBUVTk1dTslN9YqVoG0cg4mXLoFhgfAepFaFXoCRCKuaTAXFiMLJCR4bTG5sMWglKK8oLz2OHu6j1daSBwBvUyFAO7qCFJDp1EXXrJg0bshUo3FabqQ745OYuaVHRiUN6OzsEn9EeCT4lIF8tN4FkEuPS7KjSe1Icr/7OnTt3VP3t7+/z0Ucf3eM49KwqvCc50j4KT2Ie4L3/n+AJjQWP4UsjvOMeeJ+V/R+2ZvakAk3nHB988AFZmnN2+zlGgxHzw5xmLzrS0z1qt7aqKt577z0AXnvtNbI6o7EWcqF5hkOzT+BDnC3RiaBcQL6o0JWj8iWzg5pZOSF3GS+/+hzbso9RmtuHB8yLObuzO7Q7MQO5RyoKShEQEBLXAY1+wqW15ygHOVVmkTqkMndNOTUkLYWyIV2VEIUN+sE5ysaU2WJB6SGrR5RFQdmyBCZkombELQ3zGhEpRsWMw1GFCUK6ss167wwLa3ECqiJDhxqH58Z8RLSxTjQW5Is5RVkSCI9Y1ASnmzRNA9gniAyTekKFQ+qQaGFxeYWPYgodkGkQTjGbCUKRINUGG1sBzfUYOTog1RGmzLCxIa0Ui8lomUaWg9OGsHKMrgV0jCPohpSRxOzPcMU11pvniFs7+G6TMh/jOi1sXRPkBUiBbDTI/VIrt5jOiTe2kZs9zNo6UUMT2pxFUaNiTdzrkO3tUe+O8IMhldIIB7xwHt1oITZ62NGcoLdJomK2m9v4s56ZzxiOR3zyB39AICVrGxv0t7cfuW51/7F19ZoWZrkJ5NJsGd70GYOxVzjJQWU4HHLr1i3KsgQ4clD5rBXfVzWEG74kwrPW8u677x554H0W7O/vc/ny5QeiG1dee48jvFXmxdraGs9ffIG9W0OEXrqgOOuP/O8etlu7WCyOxMx7e3sIIZaOx0YQCsO51jnMIsFow4bdJjIG34P5LKcWNYfTIXUrp9fcZDYuGXYGtGWbfG2MrUoO0gHT0nCYTilVQdzpEQaC0Hm6W1182KQcQ65KqqYlMBFVoWCmqGqLMZIyVkQqRocWm0a4KIT0kLLtEdIQr3XxRpIQYOoan03JcdQEVCYkbnTItcevB7REg1AkTIYplYN0NEc/18MimMsU/BxflURBB+EdDaVQzYAdvc3hfEbsQiinyFlGSIIVIZUVBAqcBBMF1LViXggOxnP2RE0j7NAPNO04odXtkzZaVIsp5UDiYkdmIkzmsXbG9gsv0h1N8MUc5VK6vR2KnSYydrjmAvm1BmHaI+xtUE7GyMkMtVgQXbtOVVTEmz1c7ihiS5xEMJ8ix56iqVBrXXRaUxwe4m7cxJcl1cEB4uIFKMulaFGAT2tEGEKtcIsaEUpEqOj1uvS6beowpMhyxoMBHx0eUlh7FGq9smSCk4+t7q4X3sM2Np4FjjuohGFImqa0Wi0ODg64fPnyMjry7uT3aaQlnyex7IvGl0J4Wmu+//3vP/JrHkZa3ns+/vhjxuMx3/ve9x4QXK70dI/6NJpOp7z11lu88MILbG5uYmuHVoqyqEF+6mwMnLhbuxIzf/Ob3ySOY3Z3d5dfe2wtbd1skumKOAqZzzMG6SE2h8NyzM29aySNGG3ahDKm2YiIE4PwHoG4uz8Ko3qI9Z5JPcZF0AhCtro9opaiTGs2yzPgBVXlCWJJ2swIkpBYB0ySHOEs2XiP27cO6eot1s+eZTqv4LAin01x+5L41DmkFAStlEKOGYucqrRUsSXQElE7XFrirKVaOLw05PMDdDPhdKfP4XCEUzGyE1KKATIJSERAIhRlPidpxOSFptfYYH6oGU81qvZ4VxLMC0TbIGuwYrm7W7cNkWoTxBmV9Nze3cdU+7i4R7i+jp2luCBCZCWzoELqIWtbXVpklKGl6J/HlrdYJDFB6NFIxDTHheCmkqrRg3abPM1wShNeOI8azdGlRrY0UaeHz3PcYJdG7bGNkMZWB9XtUqULysUcGYTLdUIAE6B7Heiv4fMKpMIXy51lnztEKJdOKMXygzNe6xFGEac21vFhyHg8Zjgc8vHHHxNF0XItK46XhqvHjq1OyqPX9LMmupNgrcUYw/r6+pHof1X9ffDBB1RVRbfbpd/vP7b6+7yrZV8kvrQj7aPCr1eTz/t7CPf3/E4ixBXhPWzfdnd3l6tXr/Lqq68e/RKUljTXIvQ+9xxn4V4S00Jz68Yt9vf3j8TMeZWT1RmVrY5W0owyWOEgy8nynI9mHyANHMwOyQ4XfP3FV1jUEyKREG2EkEpwkkQlrJk+4/yQQAb4EmIV0wt6dMMOa3GTrd4a2bRClh6fLjBKklUlykIUJPjAUuHobsZYFgTt5XMsshEZnmy4gFFKZAxUmvO9C+SyopyNSOIYaTJ8aUnLnKhskKCJSkM2OiQraowwdPunSW3BbG9EUYwxWISOafW7tLY2cOMF02rA/v4hotuhnOd085qySKmwBEkLrZoEgaUOHUVhKZzAO4vJHBs9SbVYoIOYykIsAlgsaGcpk4NDClPjrOOwSNjZ6ZO3NgmTBJc0SIuKfv95dKNJlE8ITEY+vIMKWiRVF7U/Q718AXSAODhA1xadtJB1QHhqHUYz7GxOvShw0ylq5nGDDUSnA2m6fL00GsgLF3EXL4I5NoyIQrx1uLqECIQB0ZT4Ir/7or/3KCrvs2RaEcqHV6/i7tyh3W7T6/Xobm/h7vsQf9yk9vPipKHF/dGR4/GYwWDA5cuXCcPw6Hu5v/pbVYpfRXwlhMerfdrjhHfcQv1RPb+H7dN67/noo4+YzWYnGhAEoUEof+IurFEGJRTvvPMOQogjMXNlK/ayPUbViNvp7Xv6e0pLmr2IbD7HWEE5Kdm/vcvFcxepqhIhJe1Gg+e2L5KXBTaDclbTKnsM5iO21vvYWUk2K1F1l4utS3S7DbYam9xeDBCNgiCGfr1NEmbYqGbg9nBJSegivnbqaxyktznI3yeTBa6bcPHCtygjx8HPxhTeMc9zrt2+ylp3g4uXvokMCm7LAxJfUxeCxkzhB0Pm2YjDvdtYGdIKNEIJjLJYV6G1QcmE3KXoKmQ4PMAQUuQpIxPREJqpgIafo1AIBHkECEsUz1BhQDPUDOQGUgXUHuJOQpS1yKqa0kFGTerA5DXtbsJ4vmASBMh1R1p8xDzdw5oz7Jz+Btlwub2hFxW+yBmpNmI8RVlP81xjGZRd1CAErtuF4ZC428HdHuFv7+Osg0AgGgotA9T6GqbRQG9vUzWb+MkEES7X3ny3S20dC+swQhDIpS277ARgPd7V2P29o6OpulspPYykjhNKneccHhwwmE75+I03jtQLRVEQSPnUk9qnxaOKBnjQP++4e/IqOLzf7xNF0efq4a2MA65du8bly5f/IycbB0TAfwZClhz2b733f+9J7v+VqPBWNu+r4+rjbKHuv/Z+wquqijfffJN2u/3QjNpHmYcWRcHrr7/O9vb2PQ7NlVvqqiIRgYe8XAp5V8dxqQStpMHtD2+RpinPv3KeF7svkk8qyrKinoH0ilBEjA4XzOYZKTPmZYoODYEI6Tc3iRshO8Fpes0W261tFpOKw2JE3SyYM6QCcI5aFYQ9z4RddnWCjQvypkPLBsNBytY8p9Ncp3zuLNnhgkhBuN7D1Qs++uhnhGtt2IjZau0wFAVZMaGsSorbt5F5xpkzz1MWOcoJciexd3dyJR4jHPH6OlOToQtJWnkq7ynmKbUSVKpNkAgaQYVsN1BRRbO9zjiVlFKADQlkiBSAh53NM4zqmuG4ZJrNkBYOhaAZR0yiCLOeUEQlsdOUYcBiMeSD93+K9A2sCun0W8T1HmFRYjoeNTXk0xQbJJjDEY1AUWcZut1GNxpkt25SfHQZZITstlGdTVwygm6MjA2m1Vr+qWpsUVDbmtsHB2RrG5g0h7rilBJEYYgKQ1Dg0+KeiSrwxNIUHUVsnD3LBssP6/39fW7cuMG7776Lnc/pAb2dUzS0eupJ7ZPgaWUpK7I+c+bMUVTqtWvX+Gt/7a9RVRX/5t/8G37lV36FS5cuPdXzWBkH/OhHP0II8Z842TigAP6M935+V3z8Pwkh/oP3/rGSja9EhbcircdVZY+6doXPWxmuMm5PCgVf9fdyn+OsJ52XzGxBMXN0egmVK7l8810uNS+x8/IONrVoF2A1OGkp5jVVYTkczLkzGjKaHSICx9wtaIkGLdflYvAiSWjoNFq0GwkLN0e3HZ2qwSm1TjWBUT5l5uZM5YiqWmBVhdXnSG1GTkmPFgZLqEO81QQ7W+TNQ6qGoK+bbPQvIOsaGba5fP0mN6/vkjpIOpL5YJ+GrIj7bcK1Bi2zSSgNo9EAs5ghtUe32izyHCcUoYvRkaYTGmgYSlfTE5auGVEXFSIJEL2SoGFAZQhilG6jyw6F0yTSsnA1Y+eprSfpdqmqmsB4ZpOc2gl6pzbRoSZVE5CalnE0ghBfBzR7XezAMT0cMCwOEb5Byyial3aoMolpRVDUbPU6BM6Bc7jpFFtmpFWK8OCu7tHc2abuZphGhO0LvAIpDebcWZjN+OjHPybpdGgqTdNZxgcDUusRWqBObaOjCJR6JhNVIQRhGNJut3nxxRepsozRBx9wcOM6V2ZzonNn6W9tfeY1spPweYTHq+DwtbU1fvrTn/KDH/yAOI7523/7b/O9732P3/3d333iez2hccDSYWMJc/fPE4VofCUIT2tNURR8+OGHj6zKTsJx4lpNcp+kMjyp4tzd3eXatWsPzbhd9feuqCusmT63xsvjS5ZXyLzLH73zR5y+uMXauTX6SR+ZSBZpysHikOl0yqJMSctT3F7c5kZ+i2kxZzPs0u1oQhdi2lsEQhG3DO1uwgfT9ynqgnE55mtrX6PIMtKgwAWOol5gtKZpevSSHk3ZIopj0laGcTGbaz12mqdRscJ3NhE2YJbPYG9CNIWboyEydPhJzlarxbgq+WT/CmFzQlHmxM4iSs/G9g7OwnAxQbkmcR1QVzWJSiBKaCZ9OgZEWXMKz145RWpHtN5cDkCqCg9UY88US1ULvFLshNXSbcZWoAULKRgvUsqqpJaCvHL0ui021xqkrQgrod19kVa1QX5wi7qao31GYOao/ibNjS5B8xTpomB86yY33nodoZtsqucIpSafSAJboVptJCB6a/hrn6B9hbWCssqRssIvcnxdLTdaMFgheOu999jo91m/eJGb0znpIkfn0Eia6CxD1G5pryQlfn19uXERhnil7hGKPU0f7njFZeKYzVdeYaOqQC8zRI6vkT0LE4FnKTxWSvGbv/mb/I2/8Tee+tonNQ4QQijgp8Al4P/ovX+scQB8RQhvJVt56aWXnspGBj49mn788ceMRqMTJ7kn4f7tj48++oj5fM4v/MIvPNLLa9XfK3yOrWsS3WSwmHDz1iecPnOGs1s7lFVOXmaEwiC0p7EWIGSDnBk5GaXIqXVBGc7Y4wYvcJ7ONEDEGwip2Ig7zPMpWVrQSdqMshF3Fnfo0KZRe67JG8QmppPEbMsdAhci54ZAwEtJl57pMJcHy4hGNCqMCKOAYNHGV4ZsYTF2gawkurbkswXT0XW2L8a013ag0sQIWr0L3B448vmYbC7QcUQQGsLZIWZjk9rXhFGDrZ1tbJGBLpGLAq01ab5gfCBweY4SNZ4QrWKM6lFnAuqMsLbMq4wpgsVsgpWCdrNNPZ3T7ffoyRJkSrifUStNXwiU0oS+gS2GtANBeLCHjDQ6bBE0OsSiw/ZLZ1nc2mc8mpN9cp0D67kdVvSUp4Vic+MUptMh+PrX8EWFmhzig0NqZXGqJvCXEMIcSZnOnDvHunP4NOW0lthGAqMFxi5bGkEYIcMQ5xxOa9zd/67r+tO4wrrG7e09cR/uftXCcZJsBsE9a2THTQQ+axD3l5lY9iyMA7z3FnhNCNEF/h9CiG94799+3HVfag/vJOzu7jIYDHjuueeemuxgSXhXrlyh1+t9pu2Nuq558803aTabfPvb36Z2NWmVPnTLorIVo3rE1E6YxiPG4yHDesAvff8HDCdjJtM5ztYk4YjaBNSuhthSm5LJZMzscIppQFWlODXHUmCFQUlJqBRpKhnuzbh6eJ2BGDFUE2g6QmuY7d5BZoYNHdPubFEVM1q+wXa4hZdtnHTk0Qw3r3GyZmFLJvM9NsMzqG3DVneTw+keVVGCFlRVTVZmpPN9+msx82xO7hZEcUxj8xy9tR3aseDG+3eYzWcUkxSRRCQKhPIIKzBGURQZzte4PGc+X1AWnrqeIG0TL0LmhwHeWVILYZSihaTb7RM7QT21QI0OPLl17I92STodRACFnKNFjbIlcXwKkBRItGlSLCJyGzKTnrjymFlFsNhFjx11JRHOc/rUDsIEWAOZHJIOBoz3bjO8PKexsUX73Hm6nTYm8JQHlxEoXGgJoi3y8Yy33nuP5+96M66qM2MMSIWtBL6qEUYjouDotXh8Vct7vyRB57B5jq0qVJJAUTy2D/ekFdf9JgLHLaS890fVX6vVemT19yxDuOHRlnDPwjhgBe/9WAjxPwJ/DvjqEN798N7z4YcfslgsOHv27BP16+5HmqbcuHGDfr/P1772tae+3jnHH/zBHxxtbhzfsnDWsxVuEwbBPZPcylV4PO2gzZWDK4Q25H/+3/4Q5QzahmA81bxA+AITRfg8Y920WVtf51TjFJPRhPGNQ/YXd3BRSVNaMjFmWAVs6R1KCq7NrzHLF4TtgGbQpJc0CcYF5e4hWkSERtFqJ6hOyLo8RXqY44OSjBLtJXOXEgUR6WJBquakrqDrupgwoNnvU9kpG5tnyLOCrJ6ws7lBOp/QcBG+aJL5jGkmkIPbNMaS+cEePhD4zNHqNGgIhagdWVGxmO4xLw+oS1C2xFU5CE+VJ5QLQ209oZIk62uI2YR2KyGoBRZLbisqX6L0mKouCeWUZG0HHeRMZw5EQVUpAuGIlKWYLVjkOUkQMHOGdRFTljnt6ZRWXCEQNFrbBEIjFym6LJACXKdBJRJajXVMryKJdqC1Tjo+5OMb13HUtE9HdBoJjU6b7PaAjz68wqXnn2ft7p72/cdQtb6yhXowQBs4Io9V1WSB4nCMy5bDjBoQVbUUF58QWv1Zjpj3W0hVVcVoNOLmzZvMZjNardZR9Xf/++1ZhXB/3krxCY0DNoDqLtnFwH8P/JMnuf/PhfDKsuTNN9+k2+3y7W9/mxs3bjy1gcBKDHz69OnP5P46HA4fcFpZbVlEImZ4OGUapcTG3aPVM9IgvOCP3vgjWq0W3//m9wl0gK0dRmuwAhkKqiplOhvh8CRExCpZplV11tg6u8XGaIMPdz/k5vAqN2ZTtlSPVr1AxBLtFQ2dkNk5DdMgViGDwwFSBHRMB9EUnNnZIpaGfFFzIKcE3QazdEqkNE7nZK4Aawh8l9wX5DZDekmjmeDqmsF4Dysdv/TL/y3DbI+DvdvY/TtkU4esQkIdgisRZUVQF4Q1+DCgd2aHRthhq99n4YYM52MGswHDOzlZXpIEIUpJ6jpBC00QToh0QJnfwKgCX44phSErN1ikBWWVYqjw1rG+2SHPBGW6oK7AtwTCeHSwBrKJy2soClxZUkvFPIpwUpGOBWXuCasKz3WStXU2zj2P3uiCBhElCDsg66bMRh1Ss4OQIZtrGWdbTfLhkMPBHfbvHJAOPqIUmnOvvEKSJNgiRUiDEAZ5LOBJKHlPhuzjoMKQ+NzZZaWoFGi9rPzuvu6ttUfEt9Klft6Kyxhzj4Hoyj7+5s2bAEcyk2az+cyOtJ93rWxlHPCv//W/BviznGwcsAP83t0+ngT+b977/9eT3P9LP9Lev/UAnw4tngT3h+uMRiOyu5+aT3r99evXuXPnzgNOK6sp7DxfgIA4CqG6b/WsqAjLkBeff5Fzp849oMNzdmltvruwpFnKsDxkPauggO3G9lEqWjtu88rZV+iv91nTayxmC7LZhMl0woHcJ2kkdFoNXj59CW01tGpEVeIrR3u9z8bGKYw0TOIxLV9hgoiD+YDZbEAnbtFQTeyaQRcL0JY8y7id3WZ0e4/B/h5KKV78xjeJGhEN3UPVEbaQRHKBDBS+KHCLlLL0IBRlnVEXmmg6RSjL7XJO0PAkYYd4NiSOA/AadEydz7CuRuqCui4hbuOqCZ1WRVEpnJjjyhZlWZGmBWGYE0WG6XRGXfu7xyvBLNOgNIvcY5hjrAVrKYXAeUeeZkSiRukmMRqxXlNJz6RZ0tlqI5KSZdOsJIx2sKrGCEcjiknnGXWaYhYLdF2z3uwRVp4b6T7baw3KO3d4+9pl2AlZ29hYCoK7z99Dek+LB6pEpTDGHBGf9/7I0LaqKoQQz2yYcL99/P1JZOXdQcj6+vrnyqP4vNZQx4wDAP5nq/+4zzjgTZYOx0+NL7XCu337NteuXeO1116754fypPmyq3AdpdSRGPhJrwWOzD699/zCL/wCP/nJT+6dht2dwua6oLAeKnnP6tnKAr7T7HBu+9wDPT6lJUpDWeY4D1HcwNtDDqvDpWh5scfp5mmMMqzH6xDDdnObvcUeyXqC2TJ8K/kGo8mIW3u3WAwXfDj5gHa3Q6uzSa+7hXU1a6fWiIKlur3V7jCuJuT5nF47wZXQTHr4vEJVAuElgY0ZZVMWi4xrH37E9toOjVhysHsdW/apTEjtJGHQItSedDqhLjOSnTNQzJFRiyjssMiXgmmbeNJFTUVGIqFjJbrZZWRn2DhEdjK0MMxmE7K0YjjbA58iRU1dg5I1i3RMmlq8VxRFk7J0aL12t9pICWLBUDTxsqaqBesuxTtBSxpmeY4OFZU+pNVu40WGdxphD6lMC8eY8egWa65BEDQpyxlCG3TQQUhNeleIrLtNvC3wOYw/vsadyx/z/Pe+i7YW2etxOoKsYZkWnqtXr1BWd1jr7bCxsfG5luvh3omtNOae3l+WZdy5c4cXXngBa+0D1d+zwP1JZD/+8Y9ZLBbcvHkTKeVR9ddoNJ5q8vtVXiuDL5Hwdnd32dvb4xd/8Rcf+AR5EtJ6WLjOkxLeSkx83OzzpKODUQYTGxLjcHZJdkpLbty4wa1bt3jt1W/z9lvvUJX1A32QVZNa+OW9K1dhsXjrMcocraOtFEOJTkjrFCUUQgim1ZSO7bDR3qTf2MCKmhvT64xGI64NrtGwTbY2NmmVFT5ePlZdVASZZ5yNMQ6QEQkhlbPMsglVUHGYTykLQX6zorfWQXjLrd2btPprzNMJz118lV6nST3JmeuUTNUEScCsnrO+tY5cZFghaZoIrSOGgwOUiahTS3NeoxYFEzvkMJsjEkMnqDGyj1rMCIIIrxVV0WScT3BVRRhk1HVNFCmKogEIvJeUpWQ8nhOGUAuJdTm6KnHe47QAVZGJLtYrolZC7UscAUkoabU60FDUtSJxKeJwzGLvMnmjRZWOiduXULrJxs55nNQw3cfaA+oy53A2YyAdl771TYJGE18W6LU1ilufoF1NP4zYevFFTHSG8XjG3t4eH3zwAUmSHO2ePo0ezlcV9e3bR5NYferUUeVXFAVvvfUWL7/8Mt1u96j6W/3v6rX+sN7fZ4EQAqUUzz///NFzGI1GXLt2jcViQafTod/v0+v1Hlv9/f8J7y62t7dZX18/8dPi/qjG+3F4eMi77757YrjOScll92N1jL4/DW1Flif9ElfVmnOOd95+lyIvefWb36ZMLTYXzIY5YRge9faOT+QCFbDT3FmaCiTr7C328HgOi8NPd3DvHo0qV3Ftdo1pMUULjXaGdSuRFrJyhkokZ0+fpbveZT1ax84tt27dOoqRTJKQqsxQrZDEhNSxJ3UFLEak49v0og5OC0aTBc+/8DVMoBkN7qCcQMSOusxI8zlR3CGOm+xOry4HJkpDEFL2I3rnzzKfzQiAui4R2lArwXSeI+oKX5dMFjlWe3S5nPQKOYRqjLQSW4IK1nGuhw5qbD3GmAqpKpQqqW2EwFNVAXFc0mwmYEKkbZCpBFdMoc5IWjnGNJGpBRo4V1HHBT6OsFEPvKIoZ2Bjml2LLDyVPAQETuX4ucAcTjBKUtsMkojB/iGHw32+9vUfEBAiu13ceIxLUxSaoHseV6QYsYE2ERsb0T3GmoPBgLfeegvnHP1+n/X19cfq4XxV4b1HNhq4xeKo0lssFrz55pu88sorR45AJ01+VwQIn05XnxX5AYRhyM7ODjs7OzjnmE6nDIdDrl27htb6qPpLkuSB73M+nz/SBuvnjS+N8Fa/lJOwWi07CY8N13lMhbdaU7v/GL16To8iy6qq+OOfvU6sWjx35kUWhxVKCnS4vG7V2ztOdkKIpXWU+JTUEp1QuYqzrWVlKp2CSpKLYkmGzqOkYiPawFlPWWa4bMYsS1nMJ5QdgdQKowxn18+yublJURTcuH2DK7sfs/fxLSpVcap/hrUXNwilx0iHTQoOZ7vMgR/+0l8kDJuk2YzKH5ItGhyORgShJmPMTISoJsihxgQRta8prUeg6e/ssHP2LFII5sMB8rYgzWaUeogOGlQGqAJMV+DzHGs1RTXD2YraNaitJxKeNKspixlrazlalyidI4TFLgKCIEWbjDCYUecJcSwJdYSrJdqUtBgSRzOEMMT9LlmmCNUFus0G+WhKUS0ITETSWoP5mFBqjG7g3QIw2GyGrB2VHiIx1PWI2x/vsVhUXPr+LxE1zqDCZOlF1+ng0hQbhFB5tEpQ4b1v4uPGmquJ6HA45MaNG8xmM9rtNuvr6/T7/Qc+UIUxy/7cYnHkeTefz3nrrbf4xje+8VDR/P2T3+PV3/EByOrfnwUBSinpdrt0u12ef/55iqJgOBxy5coVsiy7p/pTSn2uCu/4Hu2FCxf4/d///d79e7Qr3B1Y/BFwy3v/55/0Mb4SwuOTSOtJw3UethPrvefy5ctMp9MTj9EPe9wVVv53585coBX20JHEpUti81bgvUMqcfSJe2TYeAKpG/kp+dnaMZ8U4EtymyECSTfqslgsKFxBEBl8VnNncoDREkdKU3TYaJ+7x4lZa03SSFjf2KATr2N1hZ3B9es3uTX/mN7hnMh74qrDyz/4Ae2oiTYhs/llqmpAowdFHbO1s00naaMCQRQ3CA8i/FxRpUAtyWcFUijiOKLTbqONwYSGvlSYqoEXijiMECIhxeLVgmZLUIw1tRsgKVBSEUZtioMchKSuDVpngCOKJgjhqasIWTcIjAU9pXIa/JymBWvmoPYJozsIsUuncxFok9LHV1DbknldEhSWXqtHsLmFb0/wW320TdByHbc3QtgWfjiD/hbXszkHrseFb3ydiWrRjFto9akdk+p0kElyj17OpelDtySMMff0xCaTCYPB4KgqWh19k2RJqvrUqaN7z7KMd955h29961tP1fC/v/q7n/i+qOrv1KlTnDp1Cucck8mE4XDI1atX+ff//t8zGAxYW1v7TPc+vkf7j//xP+b3f//3T9qjXeG3gPdYuvs/MX7uwmN4kHhO6rc96bXwqZi40Wg8ck3tYRXecDjk/fff55vf/CaNpMn8MKfOHUoL4lZMtK+J2xqpBEVe4qxHG4XUJz9O5SryskB5vazuPJhQUeUG5TW9sEdsYjajNVomIdMl4dwT1YeIOsMUgqpcYIXC1x4rlhKCVqfFlfQqlRZIJTl95hRNE7G+6HDl9beY5ofQdFy+dYWgsUN/o820GOMJyPwhoVRMxwMC69lsP89aa41zWxfIJguE0HgsXkoacROTKDKXE251aaRj6iyjpyVZnuOUwGctIh9T+VPIaowjpXRruCqkqGqKMkdrTVl5yrKBCTKw4qgqVrpC6TnirtpDBV0CLygFOFegZIFzIUIovCtoNBa0dMFwNIF6yHR6AG6DZKtHEg7xMqewnjh5nsB3oRUhNhLSww+4euu/YMMGZy+9QqvVJasElXUE9+npjtyHH9FzOwlCiKOq6NKlS+R5zmAw4KOPPiLLMnq9Hht3J7+T6ZT333+fV1999XMdBR8lel6R4MMGH0/qGH7SY/Z6vaM2U7fb5V/8i3/Bf/yP/5Fvf/vb/PIv/zL/9J/+0yfubx7fo/2N3/gNfud3fud/yQmEJ4Q4A/wvgP8d8NtP85y/EhXe8R7eo5b3T8L9hJemKW+88Qbnz5/n1KlTT3UtLI/Qu7u794R5r+QmqwGGCRUOR5GXLMYlAgGiptkLH7CbqlzFrclt8sMaL2A9WEcrTVWAkZpT7R2ctCjAVQf4usDLHNVTuIWgk5xmI0iwukVWCIq0oKCg0+mglGKnu43qaKb5jLE/YFpOePf6OzSbETvrr7IRr1EIw8FoxJVrH5KKW0QmRCpJf+M0cadFUzfZbuwQmYitjW3GswOqecYiq9E6oLA5g/mUeT3H1ZZymtGuK6SWONGmtjFBEBAELYaHGXlZEwQeFwkK3ULYPVptAyJnOtMUeQshPHG8IIoXVKVEBSV5XlPkCUovcIcL4mCKqreJwxRNjRA53gsm033y/IB2uyTQBt1a4EuJZEZdHRJoCMM+VTUjS+8gwwBbjbC545Mb10hOxeyc/zb785R5PkXpLuYE8fAKD+u5PSmiKOLMmTOcOXMG5xyHh4ccHBzw3nvvUVUVFy5ceKZbDo86+p40+PB33ZU/L1544QVeeeUVfvCDH/Abv/Eb/PjHP36qYc7xPdq7//uwAO7/A/B3gKc23ftKEN7qWHrr1i2uX7/+0OX9k3CctFaV2Te+8Y0nSjI7XuE553j//fep65rvfe979xyhVwMM4OjFcefOHbY3TyEwy2qtsPfo9VaoXIWrPaGOqGWJlZZWK0FpeUSgANamlHicCDjMdpdbGpVmI0hIdEypG1RViTGGtEiZFTOCIEAiKeqcUTlYWhhdu82ljRd5+exLS33VvKYVaTbidcRal5uHAcPDfe5kNfNqj/WqprPZBb/8HSjvSfyMVM5pmpKtzXPcGY5ZsKCqK+wiZbHIkXqKLyd4KcjzkLzM8HoCFLSbcww1eTGkXEwwRuNwKJWztlaQZm3S1FCWfaTMsTYjCGco3cJ7R20DnBUYWWPcIcJG5PULODcHHxPHFnxCVVsOxyXpYoy1LUJfoYoaMc0pZMY8n2J0D+8tlRbcuHWDztbzbJxS2OoWa1GA0FOaSR8tLNbmDwiM4eSe22fFSvIByzCoV155hclkwjvvvIO1lrW1NdbX1+l0Op/ZCOCkx3zU4KMsy2em+VssFpw5c4YoivjTf/pPP/Dvn3ePVgjx54F97/1PhRAPPsBj8JUgPO89eZ5zcHDw2OX9+yHv+tGtKrOHDTdOwoosq6ri9ddfp9/vc/HixYe+0FbHhLNnzzIYDPj4ymXmhyW9Tpdev0ejt/7ANUYapBbkdY4X3O2BqQcqwaWtl6ColuHpnWiLXLYg7ELYRqEgK5lmU+6kd0h8AikkNkFVCsZw48YN1nbW2GhtsNXYYT2yjIMxWmrSyRzpLOc3X2Czsc1rrQa1h/l4xq07Q65c36XdbhPHnqpO8T4DnTEYvEOWthGx5HA4wFU1CigryPIOgZ5hdIUxJbWqKXJFWRSk+RxrodXyeF8TBClKZSAWxCLFugRbG5zzRHGGlAVKhlgX4KxaGnZahdYTpAyQMgAf4FxEXjiUrhCs842vfY3hjXcYHtTUs4rxYEpQG5p6HRN30LpkPr/GzVtTTp86T7e7hpIOrRVB0Mf7CkVOUYxZbfaH4al7tyru67kd9fTc0lXlJJJ8FA4ODrh69Srf+c53CIKAXq93zyrYrVu3eO+992i1WkeDj8+yenkS7q/+6rrm/fffZ319/aj6896jlPpMvb/HDS2edI/2bozCSXu0/w3wK0KI/wGIgLYQ4v/svf/1J3l+P/ce3sqRQgjBq6+++tSfaiuh5irz4mnWY6SUpGnK5cuXuXTp0tHmx0k4PpwIw/DoiFLkJYODIcPRPjf/6AqdToeNjY2j6DsjDac7p8jjZQ8vNMEDZLd8LoYwPIXQKTNnyG0NOsBEHZAGBSSthDuHd5jLOcN0SOhDGraBzCQ3P77J6Z3TbKxvsBlvIr1cuuZmNTd3byCAusjpdTrU2YIwLRBSkrQ6vPDCGkpKdvd2ubp/mWz6EVpkdBtbxE1NWWS40pAfVjhnEb4maUMjkWhdYa1lXqSUeUhRxAgrCI3EmIi6FizmMb3eEIRHCoWgwpgJ+BZSery3WBtRVgqta5xb3tdjEcJRlZIoDsnSCMiJxIyyqFikFUX+MhuNb3Nup4O7fZ2qPaH0jnH6CaPdOVo3yPMJ29u/xMbGJbyviOOEqrpFWQ5RanWS8CiVYG16ZA11HPcPK5yryPJP8K5GSE0cnX8i0tvb2zs6xdxPYvevgk2nUwaDAdevX0dKeTT4eFox8MOwGgx2u10uXLhwYvVX1/VRhfgk5Jem6WfetDi+R/t7v/d7cMIerff+d4DfAbhb4f1vnpTs4Euu8O73oJvNZrz55pu88MILXL58+al/iWVZ8vrrryOl5Jvf/OZTX7+qKr/73e8+0j/vONnd/0sPo4DTZ3c4fXbnaGq18uVLkoSNjQ3W19dpRY8f1UtpiGSHU63k01zcY28iJxxhEBK6kHE6phk0OTw4JNvP+O4r38V5RyfoEOsYvCefz5kM9inTBUFgCOKEpNki9Y68smhpSQcDqEqi2OCoOHXmHJG8wK3bf0xdw+HogMWigdcG6SVRFFBmKUGjTaQr8nSA8zVGzajLdebjCuFiaDYoco8JPEFQIlUMfopjhAksUkaQTHFO4L3AWWg2cupa02hMKUpDGM4wukTpBVodoFQXKT1aCbTuEIU5Xo6ohKHp2ujtTVwk6DQ2iN0duusJN65P2djcJstKPvjgY/r9DtYOkHKA9yVRdO6ourY2BcTdvz8a1qaUxT5KhdiqIDAbSPnoNsru7i63bt3i29/+9mNPMUIIOp0OnU7nSA4yGAz4+OOPSdOUXq/H+vr6kRzkaeGc4+2336bdbnPhwgXg6Xt/JxHg55GlHN+jPXfuHMA/hgf2aD8Xfm5H2pU+bhWuc/ny5aeaFq3ExC+++CIfffTRU5Hd6gg8HA45f/78I8lu9Qt/2C+4ctU95LSaWq2EqQcHB0cV7CoP9Hh/Mq9y8irHiGBpea4ERt9LdCusLKt6YY+syJgP5hR5sZzw3dWJNZtNlJSkh0MW8wV1tiBJErI8Q2qHDAIcUFclQknGh2MMjsOsornWJtOeJNmiv/UKQTVGCkVZOq5cmzCdTaknOUk4Yz7VxBstVFDSVJscTizzrEAqQdLMELJAElJWTYScUddTwnBBGNZ3fwcO78H7EOcCjC7wHpw3ICqM8QRmQRRbBAFS1Rg/wXuFVDVaGaQKKMo5hbQIAV5u4OoUP34T7y2DwR5nz75Mo9mm0/4OUjbY37/KrVtvURS3CKMG3U7Bzs4OYXjqMx1PnxS3bt3izp07vPbaa59pVzUMQ06fPs3p06ePBh/HA3XW19fZ2Nh4onbOSWR3Eh4me3mU6PnzRDTet0cLMIJ792iPw3v/P7J0RH5ifOmE9zAb94cll52E+8XEH3300RM/vnOO9957D+fcURn/sOd5v5j4flSuYnfxabjKTmPniKiOC1MvXrxIURQcHBzwwQcfUBTF0qG212aQDrDWUc4sZ9cvEJnwxGkvLAlvp7HDJJtw49YNAhHwrZe+xVp3qXtapcdXeY4HkmaTQAfIICSJI7bOnKOqa+Jel/xwjJcSqSRoST6bs74e0mn3aAUdujJG0qCuFLDAWoWSNaPhPkFQUJUFh8MZQoLWGd7P6DYjaE2IwikejZQpk8kWUlQImaFUgbWgdYXWNZ7Vm8liXYzWNaEs8NZQOoUUHiVLEDValWijwHsQDbRa9ks9t1nkC+b5h8AmSdxGCMtsVnD6zKWlAYTXFMV1gmCHZjPn9JkGWZbgXZc0LXj9jdcJg94RacTxk0QLJATB5rIHqDrHjsYP4saNGwwGA1577bVn4khyfNcVlsfIwWDAu+++S1VV9ww+TrKdehKyO+kxj5MffFoMHNf+zefzr2xiGXzJhFfXNa+//jqtVusBfdxqgPCoF8RxMfFxsnzSCdOqX7i+vs6FCxe4c+fOiU4rT0J28KmdVGxisio7qvROwvG+X13XS7Hm9atcP7hOv7kO3rCzXoAPj6a9JzXF67rmx6//mDiI2Tmzg5DL5xYEAdbaZXq8ECyzcTwbOztE7S4mDLHOsShSDosxhSwo8ESJxmhJ5gWqqdESGiIidTllvcDaAinGhOGM/tqMqjwkCATeDwlNjyJVpIspJnRAilBjwmhMWbYJggVB0EOpGTqYEoYl3tdI6RBCYl0ApkGIRakKKdUy/7V0aG0JouXXIQQQIYQF4e/+TA5xtmaxuEWRe6yTVKVnfDjFeccLl3rU1XUWNiCUfXxVYJOMPL+FMT0C0yaKzrO5uUEcn6coagaDAe+99x5lWdLv99nY2HjotFRKQxyfx/vqbqVa4RwPVIfXrl1jMpnw6quvPlPpyXEkScK5c+c4d+7ckQPy7u7ukQPyqvenlPpMZHc/Vt/H/dXf9evX+elPf/pYx+OfJ75Uwnv77bc5ffr0ieE6Wut7ksvuR13XvPXWWyRJ8lCyfNQLahXu88ILL7CxsXF03f0V3pNsTqywspPKquwotPtJoLVma2uLzlqH5u0mk+mU0e0p777+Pu1mm9MXN1nf6OL9gOOTw7K0/OEf/yHbW9t0G13yMscqe/T9TyaTo8cIO21qWxJ3myAEs9kc4QXD8ZBZMaPb7OJERRSWKFFzuq3oNgV2PmLuK6bpHiiPFPvU1qK0o9FosdbPKYsG3mm0WGde7dKKK3QUkuV3UGqBFClRWCIIUEoTmABBk6qUBOEUEEgZonVOiEQKiXMK5zWhyfEmR3iDtQrrYqR0lJUD4QCJFAUVe9TWoWTMsu8WUVUzPEM2NtYoimsE4SZ2OkLICqdnhOE2QoBUhjA6Q5JcIAjW75KX4ezZs8vIxGO26Y+alkppcA7K8jb3T3i991y9epX5fM43v/nNL4zsTnptHXdAns1mDAYD/viP/5g0TWm32/T7/c8sNj4JUkr29vb4y3/5L/Pv/t2/4/z588/kvl8EvlTCe+211x4Z1fiwNa+VmPjcuXOcPn36odc+bHR/cHDARx999EC4z/1raU9DdvDpEfOkAcMK9/f4jiMyEZdOXSLfyDGXAowIyPOU4eGQN9/8GTBhff0U3W5CUYx4770rvPzSy2Qmo0gLhBI0G02UUuT5AudSjEnIq4rD7BAk7E53aYkWrnCc7p6mDmuKsmBYDemFCXEco6XB2pxIJszqlBvzK1TzIV6FrLUlEo13E+K4Qat1kSILGN6YcTi9zVo3oiSnqhLqWmBdgqw1UVwCAUm0i1RTjKkQosT7ZcCUteCdxOiltk0KRe0sghzpBULNkKqB9BbnYpzLsLVCyBqplkMGMCglUTpkMm7gvGJz8znCQCPkbbAZHkfYOI2sFNgKbfoo2SYw8RHZ3Y/7SeP4tFQpdc+0dDnRvXfC673m8uXLlGX5mYZpzworD7xms8l8PmdtbY0kSbh69eqRC8pxRcFnxd7eHr/2a7/GP//n/5xf/uVffobfwbPHl0p4j8qCfRjhrTzoXnnlFbrd7kOvPakX573nk08+YX9//8Rwn+PC48cNJx6GhxEdPLrHt0JkIiLzaaM5iNq0u23Onz/DbHaV0WjEBx/eJF2EnDp1jkAFxDrmxvQGRhqG+0OCHYW1e9T1jKoa4uTy51RVFXvjPWxoqcqKxCTEJual8y+xKBf0ww4+GyJVRZbWWJeTlhn55IDQ5CyqEVnWp9c+jxBdikJjbcx0csgi77O1uQFCURf/3+VesQsRMkUbjxCOMIyIozGelLp2CFFSVR4pl39A432BtQlSgbMCRIU2JUI4nBMo6XCuJgjmgAFvgAitIrQ2SBWBT2g2m2w1d8jzfapKoLUmalzEl2NMnWB0lzA5S2SW7QGlkiOye5Se7v5p6fE1sTzP6XabdDqWdtshpQI0H374Ic45vv71r//cyG6Fk3p2qz3Y8Xh8NPkNguBIURDH8RPffzAY8Ku/+qv8o3/0j/gzf+bPfEHfxbPDV0J4DCdbRD3OKWWFk4h0pTECjsxC78eKZI+T3bN8gT5Nj+9+SGlotS4ymQi0gh/84FWm0yk3btxgOBzinGNnZwerLUWxIAgknc4mVTUDlTA6uMPe4R7z4Zykk6BiRRBJqjInrzxJ2KDbWGdc1kzyKyAaoCW99bMMZ3cgaBHXJZ32JeI4pCz30fqQ4fA9JvMRazsXSKQj8yPaYYOsACkL0syCXx47pdhnGS5VsCwgLMbU2DpCUCKkQKoSBHgMdbWFkAUIuxQbiwghLUq2EbLAWolUimYrBi9BxHiXIWSb9Q1JUQxZBsI2gS3C+EV0lKNlF2VirFggnV4S6d0hg3MVRXEb58q7z28HpcyRNOV+Ijy+JmatZTQacXBwh2vXPiFJOlTVAUmS8PLLL38lyW4FKeVRliws/SZX625lWR4NPh5ldDoajfjVX/1V/v7f//v88Ic//KK/nWeCrwzhHa/wjq95Pcop5aRr4VN93ubm5iPNB6SUVFX1hZAdfPYeHyyr048/vkaWZXznO8ufQRzHbG1tkec5H374Ifv7+8xmM8bjddbWPO12myAwYDUd3UE0BKIS6EATBYY0u0GZFeR4Wu1XoAFTOeWgGiN0hCtG9FREp5lSuUPiZI21rsK6PawbcGfvDlk+o9ur8exSiBgpQ/BreHeLMBJ4FEruI0SJdRlh0KIoDUJUeOeWGxUhSGEJw4AsT/BOUNcaoQ4xOkcpkLKkque0G+sUwhOGMaAQogAsCEFdlWiT0uuFeFfi3AyjN5FyHaUaGN3GmA5B0KeuZ3cnqu2jo6dzUNcT6jrD+5S6npHnt4misyyJE6TQnLR9AcvX3UpqZK3lzTffBJb94j/6oz86mvo+K6Hw0+Bpp7FxHB8NPlZEfufOHT744AMajcbRMX51SppMJvzar/0aP/rRj/jzf/6J3Zl+7vhKEV5d1w9MUp/khXKc8Gaz2VFmxmo4cRK89wRBcJRc1u/32dzc/FxhxvfjSXp8J2FlZR/H8Yk9oCiK+NrXvkZZLndr8zzn2rWPuXr1GnXt2dycL5f5TUDP9AjDkCCuqN2UJGzjfInzFXmV44QkjhLwnqrKmGcVRoXgJhg9p7a7aNVj785lwsDRaDSo6wlKeWy9ixAxQobkuSEIDd4WIHOSwFJUNUqXJDoGNFUZUNU5oHG+AJGzPD3F1FUfaz21XSz3ep1FioIs94SBRKnn0bqgrAZAAzhEmxKtIM9vYEwL7wpqazCiJAhexNoBzkNVDTBmAyE0ZTnE1TUud9RMkDqkLO8AEqkCvPVIGSxFyF6ggk8J8v7tixWcc7zzzjt0Oh2ee+45gCPfuONC4ZVDyhc9wPis0pMVjhP5KvpxMBjwxhtv8B/+w39gMpnws5/9jN/+7d/mL/yFv/Dsv4EvEOIxI+RnOl9eBZSchOvXr1MUBfv7+7z44ouPJKv78dFHHx2ZBVy+fJlvfetbjxQ/3j+ccM4xHA45ODhgOp3S7XaPmrlf1nRthVWi29bW1j1W9o+7Zj6fA7C/v09pS3b3dsHBzuYOm5trBPGY3em7zBcpWq1zpvttmq0uN+c3GWZ7CFHS1S2iYkJVvI21GXG8RhBs88n1O7TbLdb7ZyjLnOnsI7ybMZtfA0IEAk8braC2e1B+SG0XOAxBrFl+rpZAAZR45xFSAwlLEtFAl+XLbcYyxLAEFOlik6qGwHSIkww4XNrrBxatNgmCGOcEQdCitglGn0NKxdraKyBmCAxCeLReQ+sYV2akex+Ar6jJaW69gtBNhLAIEVLXE4JgnZMqPHjwiOuc480336TX6z10OnncIeXw8PDIGn5jY+MzJe49Ciuya7VaXLx48ZneG+DKlSv81m/9FlmWMZ/P+f73v88/+Sf/5DN74D0Gz7ws/spUeIvFgjt37vALv/ALT63UllKyu7tLWZYnDieO47hI8vgqzWoqt2rmria7zWbzqJn7edKcngRpmvLmm2/y/PPPPxXhH2WfWosyCmcc3a0uQRRgKsPe3m2mi+vkgaPRiGg1tmi2uhRpQZcuoVGE4YJABEzSKdatI8QCax2fXN9lY/0iQdBgPC7xPgFqrB0DY0DhccABzi/bAj5YR5ODF0AOdIAYcIBHSICKZVqrvftvM6DJ0g0vZUl6nqQxQ+ttnOviXE2WKhA5BkNt59RZCiwHNMY4VFjiUSzSd9A6xOiYIDiN8xm1rVEuRniHjDv4zFEVE+Jw7e6K2V1ds+CBHh5AUdwrP/Fe8sYbb7CxsfHID6fjQuHj1vBvvvkmzrkj8ms2m5/rdPFFk12apvzWb/0Wv/7rv85f+St/hbqu+clPfvKVFhrfj597hee958qVK9y5c4f19XVeeumlp7qnc46f/OQn9ySZnYQnFRPff818Pmd/f5/BYIAxhs3NTTY2Np7K5+tJMB6Pj6bRqzyDp8Fq+JLWKXvTvSWfaGiqJv12l0V5neuDT8hnBXuTis34FL1mj+3tbaxdoNSCPHdMprsU+QjnLrNY3GZz8xKd7ovk2Rrz+Sd4HHX9Pt7n5PnHQLZ8IEYsqzIBJFBnFJXC+pQkMSyJTC6/3gOiZHk0lXevr1gSnmBJgpKl2HgN6OD9/t21Jc3yBh5YpqLFSQiEKLlNq/U1lOqiTU6zcenu/rbDuQznHUZ2sKNDBAqBJ9p+kaT1Ilo/2o7M2pSy3D+Sn0i5xttvf8RWv8+pzc2HOiE/Dqt4xIODAxaLBd1ul/X19aeWinzRZJfnOX/pL/0l/uJf/Iv89b/+17+snuQzf5AvlfCcc1RVdfT3uq55++23iaKIfr/PaDR6KsJbOSNHUUS3233okeKzkN1JyLKM/f19Dg4O8N4f9Tk+Tw4nLHVM165d49VXX31ia6uHoXIVN6c3SacpHs9mskm/16f2BbvzGwihEcKQ2ITbN24zHA5RyrO+ITF6qW8bDG8zmfx/2NrcpNmM0XqHw7FmMR+jdROp3iEMBVVZUNsRMAQmLCs4C3gWc42QHlAkSZ9l1WaoqwPAU1Y5SWLwjrsVn2R5rB3fvYcC2kjxHAjLYjEijntE0TZ1PcK5A5xbsKwEl3IQZztIdZYk3iRJJK3W84TRacQRQXqECAnVNsLetX2KOk+0O7ua5oK/a6k04PTWKTa8f2In5Mc/xqdSkdFoRBiGR6eLR70uvmiyK4qCX//1X+fP/bk/x9/8m3/zyxzA/MkhvCzLeP311zl79ixnzpzh8PCQ3d1dvv71rz/RvVZOKy+99BJlWVIUxYm/7GdFdvejLEsODg44ODggz/OjY8nTDD1WOsHRaMS3vvWtZ3ZkrtxyICG9JDLRUaVQ1CmlTQlUQqiTo6qwqioODna5fv0Ko9GUuh5w+sxt2q2IsppS1uu4ugvkGGPQahcpSxCKxaIGdoGPgQXLl4wmywzCe4TQhHGHJdk08P4mUJFlkjB0SFlTVzXawJLw8rv3cCwrwD5pmpMkDcKwTxhuIqVjPn8PW1s8NaBJ4jWC4AxSdUgXB+RFAazT7fz3rK3FGLMUOAfBJnF8spXT4/ztlqS34M033+XixUusN5vU+/tHTsh6cxP5DBO7VkffwWCAtfbEVLQvmuyqquI3fuM3+FN/6k/x27/921/2tPlPRg/vpNjF1WrZk2Bvb4+PP/74yDxgf3+fNE0f+Lqn3Zx4GgRBcOReYa29J7Gq2+2yubn5yImcc44PPvgA5xyvvfbaUw1HHvfGXJqO3qtNdK7CVQdoPHU1AbmJDlbW7AHGbFDXA6Cm3f4OaWq5eXsPq1LCKMOWC5SvSVhg9CZRnOBciZRdimJEWfaAiKosMWZZdUVJA4hQCpwt8H60eobEcZOSMXVZI6xEG8myUhN82t+rSNOKdnsLrTVh0EMqWCz2wSd4FkTRNkafotm6SBytIWWbRnwHqTRCJDjXZW9vwWJxSLO5xtZWQhQ9+LM+XsE9TIZS15433/yA555bKgB8VT0zJ+ST0Gg0aDQanD9//qGpaLu7u3Q6nS+E7Oq65q/+1b/K97///Z8H2X0h+NL98FaB1veLiZ8kUHvV7zs8PHzAaeX+a733RwT6RU9aTxp67O/v8+GHH9JsNtnc3Lwnrm+1F7x6oTrnKMvyyO3kUXiSN+b9e7XLCfZyBQofMB/dQKkK7yRJbwshaz6+8m/J0oxLL5yl171EVZ1hOP0jbt/+gNn8Nml2QGIKrBGk2ScguksSdc0lmbkYIRNMcAikxHHJcgLrwLfwWJbH3tWgwhPQIQgqPj2WGpZV3iF1JSmrkiRZoPUZwqCL0gFlOaOq9hFCAxGCPs3WL9BpnyHQ2xSH16gX46WLS6NPt9eg2zXAGrPZnMPDQ65cuU4URUfHxTAMT1wROy5DyfOcN954g0uXLh25lDzMCfmLwP2paIeHh7z33ntHovlVMtrTbEk8CtZafvM3f5NvfOMb/OhHP/oTQXbwJRPeypn4JDHx4wjPWsvbb79NEAR85zvfuYfEjl/7RR1hnxTHFeyr5e39/X2uXbt2ZOd9584dzp8/v9yUOIGcHu0Y8+g3JnD0szDGUJYpZTnDmAAQVOUMhwWmTGczcjfm5u4ugSm5dOk7VNUA58YoVRKHAe21mLCxSTWvwQ8BR5rOWL50CpYk1sC5bZSULKUm07t/KqDEuinLgQYsXbmLu9c7lhNaC4RI0cJ5AUwpK0eSRHf/3SOlxnuP0QlG96ntgijaodHYotHoonUDZ8fLHl20ifAJQnqqag9rZ8Txc7RaMd1uk+eeO0Oee4bD8VGI9tpaQqtV0GiUKBXeYwKaZRlvvPEGL7300gNB8F800Z0E7z03b97k1KlTXLx48Z4tiZU91KOcXh4Hay1/62/9LS5cuMDf+3t/708M2cGXTHgrIe1JeBThrYYTp06dOnH8v7r2501292O1vN1ut7l06dKRZ5kxhps3b1IUxdF+sDGGqqoea5H1JO68q+vLMiUrbuF0SORD4uAUStZUC09WjLDWcPXaNUzYYmNjk6oaYJ2lqhfY+hDhUjpxm9ooaGwznbyFdwqtBeCo65w0tdh6GXLUCl5Em48oyznL1vDqQ6lmGR/qWU517y4XI1j26QRGLz3l8tmEKg9JGqtemMW7Gda1l5ZSdooJNKIOMSZAG700FrAz8BJPTZHvI2SI0BWySnBuwmLxAUIapNAIoQmCTc6dO8+FCxfI8wX7++9y6/YeRX6ZTudFtraa9Ho98jznzTff5OWXXz7Sen7WLItngZN6dse3JFZOL8dzMVaa0ifJxXDO8du//dv0+33+4T/8hz/399Czxpd+pH0YHkZ4K2fjR8U2rq79KpHd/RgOh1y+fJnvfve7NBqNo6HHxx9/zHQ6pdPpHDWkH4VV9sWj3nBKKTqdDot8xNCmVLVgWk45bTaIwg7tDUM9yLn6/utEcUA76iDlWeI4xLoxafoeVbmPc45EnyUMNxAKQhNQlmMQl0jTK8znI5JkwDItr411GVVqkEqzJLhDlqTmgPDu/5fwKemVSNkjMC3C6CzzyQLyDNMOWU51Y4QIlpshbszy2BuhdQB+ipSOohgThnM8fYriOhVTCEqS5iXy8g7e54TR6eVpHnG3CuWuq8myOjZm6Ua9vX2eqpqTpoaDgwPef/99yrLk/PnzR0fFJ2kpfFF4kgHFSU4vBwcH9wSC3++6ffz+f/fv/l2iKOKf/bN/9qWL7r8MfGWExycR1P3Oxg+DlJLFYsFsNvvc4s0vArdv3+bWrVtHKVVw79Bj5Ya8+mR+3BrS8g326DeZUgppApRQGFGRuZzSVkQGrBN8fG3KmbMvY0yKUh5PgVJbeD9HqwYqOEd68AEL/xEIi2o2MdEG2mxi6w2kWK5+OQrgHEmyTRw3icImg+GMopiRphlh4FHasTzeJiyPr0utnRQSKS3aNJlMLNQzZFvhVv3Gu6QihEEQoHWINqfwvgSfE4ZraL2G0X2k0ETRKaRKWNRjyvoOUhqM2cDo7jIFzVXU1eHdn0/nqDo+XjVLqej3t4minNFoxNe//nUWi8VRdsraWoNOB1qt/mNXzp4lPss09rjTy/FA8JXr9sogoN1uI6Xkd3/3d7HW8i//5b/8E0l28HMgvPuDfE7CcnH+YyaTyT3DiYd9rVKKc+fOceXKFbIs+0L2Yj8LVkOW+XzOd77znYceVY+7IR9fQ/rwww+PjiQrx9qT8DDPvVAnKNMjK3YRQiHcmOlU8c47b/PCCxextmQ63UWIECnEXZutCOdKfJWjZQ8ZBFDmVK6gqjOyrMK5GiVP02wanF9KJpKkg3eaqloQhYYoPI/RNdYanK9I04zApGizquI1CIH3kvk8BXKSVo+yvIWzIVLEOF8RBE3i+AJKxniqpa2T6LC0WFEY3cSYHh6PcwXeO7RpYcwGUoYkyUW0To7IbdkK4B57qPur5tks5d133+XVV189+qB97rnn7n4w7fLJJ+9TlB/Q6XTZ3krodqMvlCCelfTkJKeXW7du8Wf/7J89mgj/23/7b//Ekh18yTo8WOrXHvaY//W//le+//3v89ZbbxFFES+99NIjCeskW6eVRGTlJNLr9djc3Hykzc0XgZU9lTGGF1988TMR7/EjyXA4JAiCo02PVaX4OM+9vJqQ5ruEps18OuDKx3u8+OIZkiSmrkuqqkQIs0w1C05RVTnWjnH1AnewT1rdoLS3qOJqmRomLlLXEd4nKD0gDArmi32cO4fRpxHiGlLeQKmYPN9H64TaluT5mDS9Bcxw1lEUgjjps9TdQaMR410HawuK8hZSlgihaDReIo4vEoXrCBli9Dr2/9feuUdHVd7r/9lzyySZJDOZZAZICAEhBAlJCFL8ISIFQVDIJKhRalsspbQHaeGoR7Fqq12lBZa21YMLjudQixVaSQJBCBfFihSLUg0QYkgImHsmc0kyuc197/f3x7C3kzBJZpI9k5Dsz1qspTju/W7IPPt9v5fny3RCIpGDdndBLFF5jDwpCdzubtB0F5xOPcTiCIjFUYiKyoBUOvBQdhaLxYKKigpkZGT0mfFkGBfcbjsslk6YzRa0t7dzLybvbDwfBLvOjhCC3//+9ygrK0NWVhY+/PBDZGVl4c033+T9XoPg9i48BvoXvHPnzkEikXBvoT4X5Wdygt0tGY1GWCwWREdHQ6PRDNnhdSBcLhdKS0sRHx/PjpvjBXYKmslkAjsFTaFSoBvdnOdeXEQcwqgwLvlBUQwcjia0tJhR39CItJnzIJHYQFEyuN0dkErjIJFEgGFE6Ohoh9PZDEJsANWIMHEM3M5uSMKiPDtFewPcrnBQVDTE4niEhVnAMG2gGSnsNgqRkeNB03Y4nWVwuSygaRukUhUkEhlkMg3aLOdhtRpBUW4wjAx2mwiUyAkQJ0BFQh7mQJg8FhQYUCKPy4lMGo+IyMmIib7X424sjkR3dwUAAqlUBUACkUgGmSwWTmcr3G4bGKYTbtoGuXwCFJGpfsfYWltbce3aNWRmZgbU8dL7xSSVSr0GAg2+TCQUYvfHP/4RV65cwf79+zmhdjqdgzY1OHnyJDZv3gyaprF+/Xps3bq1x38/cuQIXn75ZYhEIkgkEvzpT3/CggUL+rrc7S94rP9cb9rb2/HFF18gKysLcXFxfS9okJlYQgg3M7alpQWRkZHQaDS8mwLYbDaUlpZi8uTJ/Q72Hips3E9v1KPZ2gylSgl1rBqT1ZNh6/p2MFFMTAyamurQ3NyI9PQsSKVS2Oy1cDo8Q93ZzgO3m6CjwwhC2uFydYFhaqBQTIbT2QqpTA2ZVAmHoxmUKAoUJUNkxJSbYloHp9OB9vZaSKWxcLk6wTAidHXXw24jkMm6EB4ejhhlEhwOPey2OhBI0Nbmgko1A1EKGlZrLZxON+x2EZxOJUDVQy4nkEpdUMZkQirTIEoxCxTlMR+gaTsYxgWxWA6GuAFCQyQKA0074HTq4Yn7iRAVNQcymX+7O9b5NzMzc8h90jabDWazGSaTCS6XixsIFEiIJRRi99Zbb+H8+fM4ePCgXxncgaBpGikpKfjoo4+QmJiIuXPn4m9/+1uP7qmuri7OH7C0tBR5eXmoqKjo65Kjo9OiN3q9HjU1NVwJR18MpXOCoigolUoolcoepgC1tbU+j4qDob29nesgYUsYgoV33M/mtMFgNqDN1IavbnwFuVwOrVYLuVyO69evw263Iyvr/3G7Wpk0HoRxQSKJBiEuuFztYBj65j/bwTA2ADTs9loAEohFcojFCoSFjYNYHAG3uwOAAxJJDIAk0HQlIiMVABiIxRI4ndEQiYwQiy2QSKIhkcRCLFIgJmYeJOJY1NU3Qh2rRlxcMghxICxsPJc4sFqr0d3d5UlC2Z3o7KxEdJQDYpEGUVHJN+Ntngyjp6HfBFAS0LQdhNCe61AiiMVR8P7x6K+UhK2TnD17Ni92TeHh4T0GAnl3SPgzRyIUYve///u/OHv2LAoLC3kROwC4cOECpk6dynkCPv744zhy5EgPwfN2Quq+2aESSoZV8HqPXbxy5UqftXh8dk5QFIWoqChERUXhjjvugNVqhdFo5AZms+IXyHHEaDSiuroamZmZvFW7+0u4LBzJE5KRPCEZLpcL9fX10Ov1aGlpgUKhQEpKSo/Pe+JbESDEBTdtg9NWBxElBkW5IA+PhlgUB0IUICAIk2lAUWyhczdstloAAHVzLgRF4aYYErhcHZ4RkS43QMRgGCnc7i7QDAWKUoJ2u1FXb4dWmwhlTCzEEgUoKCAWywFQcDrNEIkoKBTjEBFhQ1jYeLjdDKzWGFy/fhEEVxATPQETJsxBTIwaIpEUIpHk5rO4wRDPszkcjQBxw+kycVbufZWSNDc3o76+HrNnz+bti+8NO6FOq9WCENJjjgRrDuDtvhMKsfvLX/6CkydPoqioiFfXn8bGxh51somJifjiiy9u+dzhw4fxwgsvwGg0ori4mLf7+8OwZGkB32MXWdfj3gRr5gRLREQEkpOTbxah2mEymVBeXg6aprkfyP48+urq6mAymZCVlRWUL00gSKVSJCYmorW1FdOmTUNcXBxMJhOqq6shl8u5YzyXlXS1w+1qhVSqAk03QyyiEB6ugcvVfnPH5BEJz+DpeNCMDWJRBEAYrx2TDEDUTSGNg1RihEzmglg8Ey53G8QiCYA4XKuqwLSpWZDLGW53KZGobo47dIFhnJDJ1HC5LADECAtLhkxmR0SkGPJwQCJWorPLkyW1WsWIiVEgRklDGcOAoqQQi6QgjBsikRxhYUkQUZKbZSOAr+6UpqYm6PV6zJ49O+heh4DnZ1+lUkGlUmHatGmcOQDb7aFWq2GxWKBSqYIidgDw3nvv4fDhw/jggw+G7MzTG1/hMV/f19zcXOTm5uLs2bN4+eWXcfr0aV7X0R/DssOz2+2cU4r32MXexcfD0Tkhl8u544jHRcTEHQt7l7sQQnDt2jW4XC7Mnj17RKTzWYv8CRMmcH+2SqWS+4KZTKYeO1mVyuNJ53K1gRJJIBLJYbdb4HLRCA8fD5lMxh0DGUYK2t0FGh0ARAgLS4BY/G1JByGA3V4Hm+0yXK5WiMQqgExCl41BXX0JZqWlQ6mcCJfLdFN0KK48hKJckEgiIZcnIyzMAZlsIsRiKdxuKxwOA8SiSMhkUYiJBpIm3gGZTHuzZ7kJdbWeATpxcTGIiADE4khPv61I49WJ0rM7paGhAUajEZmZmUFNYPWHtzkA203EMAwMBgOcTifvlvAHDx7E3/72NxQXF/ssPB4qiYmJqK+v5/6dbX/ri4ULF+LGjRswm839xu35JOSC19bWhitXrvRwSmHx1RPLDtgejno6qVSKCRMmYMKECaBpGmazuYcjSldXF1Qq1aDLTviG7fnsyzGZ/YIlJydzSY+qqlo4nE6o1ZHQxE+ERBKJpqY6UJQEHR1tSEhI8AwGgscJ2DPLVQaGcXIxMrYQmqatcLs7wDBOiMUxcNM2OByAvsmJ6SnzEB09DhJJBESib2veAE8sjqI8wimTffv7bHKFZmwAxKBpNyRSNSSSmFt6lru6umAw1KG+vh5isQxKVRi0GgV3dPWus2to0KO1tRUZGRnDJnbesM45Go2GM5PwrsWMiIjgajEHG2M8fPgw/vznP6O4uHjI/o19MXfuXFRVVaG6uhoJCQn4+9//jgMHDvT4zPXr13HHHXeAoiiUlJTA6XT22UEVDEIueDRNIysry2ecix3V6J2cGC6x641YLOZiMTabDRcvXoRMJoPJZILD4QhJuUt/dHZ2oqyszO+EiXfSw+123xw0bUBbWxsoioJWq4VEwvQoUfj2+OoZdEMI4HJ5jA888TwpKEoGmnZCJJLCZiVobjYjNXXeTQsqzxGKFcjeIxLDwpI452GatoIwbojFYRCLwyCRqBAmi4dUqr7FnZiNyUZGpiAhQQGHw47W1jZUVdXB7a7lduZRUVGoqalBR0cH0tPTR8SO3FfMzpclPLszBxCw8eyxY8fw1ltvobi4OKh27BKJBLt27cIDDzwAmqaxbt06zJw5E3v27AEA/OxnP0NhYSHeffddSKVShIeH4/333w/p9zvkZSn9DfKprq5GWFgY1wc4Entiu7u7ualo7A9kKMpd+oOtH5s1a9aQ3952ux0VFRVoa2uD1WpFYmIixo8fD7VafbMTw8UdXx2OJjidN8tbwjQIl08Cw7jQ1VUJs1kPk7kbGenLb1o5edrpvF8ING2F3d4Imu6E290JsTga4eFTIBZLPdd3Nt1SPjNQTV3vbCybJTUajWhtbYVEIkFKSgrUavWwC95gEhQOh4MrebHb7QPOjz116hR27NiB4uLikO6keOL2r8PrT/Bqa2vR0dGBKVOm8B5Q5YO2tjZUVlYiLS3NZxKj9wwMvspd+sNgMKC2thYZGRm8ZdycTic3ApK1HmppaUF4eDi3uxCJXLDZasEwnlYtkSgS4eFJEIsjUFt7A2azAenpWRCJpH3aXzGMCzbbDTgcBtB0NwAZRCIR5PKJEIk8pqRs0sG7FSxQCCGoqqqCy+XCuHHjOAv1yMhI7qgY6mQTH9nYbweBm9De3s4NnFKpPKM5//GPf+DVV1/F8ePHAxoKNYIYnYLHxuucTieampq4TgLW9WEkiJ9er0d9fX1AwsKWuwTredjscEZGBu+7SV91a93d3ZyYUxQDpdIJhcLtmXsbpoE8LAk1NfXo7u5GWloaABoORzesVgdksgi4XC4oFIoe4u92W9HdXXlz1q3nOBwRMRkAgUym4cpKBgshBJWVlQDQo1WRfTmZTCaYzeYes1iDXVYUjNIT1nvRZDLhN7/5DaqqqtDW1ob8/Hx85zvf4eUew8DtL3i9B/n0lYlly0OMRo9NUXx8PDQaTVCyS/1BCEF1dTXa29sxa9asQQuL9/Ow5S4ajWZQR1DWXMFqtSItLY33o5k/Fkh2ux1GYxOMxga43W7ExU2A1epxbb7zzjtBiBsORxNo2o3Ozi7IZONAUVJuh+ctqJ771YFhALe7jUuMDNV6iRCC8vJyyGQyTJ06td/wCPv3490dwcb9+AyrBLvODvD0pP/yl7/E6tWrcfbsWZjNZpw5cybk3x0eGF2C52/nBOsdZzAY4HK5OLEIdH7tYNZ69epViEQiTJ8+nTdhYctdjEYjNwDI3y8XuyY2FhWMGGfvkYQD7bQcDgdKLpfA5rBBKpZCo9YgLk6B8HAnJJJIOJ2dEIvVkMmiOLFjxZBhGISHT4RYLOZig+xc2P7EbuCBOwy+/vprREREYMqUKQH9OXnH/bq6uvyaUeIPoRC7CxcuYMuWLfjggw+4Pm6r1TposRuoN3b//v3YsWMHAE8Xxe7du5GRkTG0h/iW0SN4g20Tc7lcMJvNMBgMAYtFILhcLly5cgWxsbGYNGlS0JInbLkL++Vi7blVKtUt96RpuseU+2CtKRCTS5qmcbH0IlzhLiRMSAAhBDKHDC1GPTo7qxAREY24uHHQaGZwWVqatsJm08NqpcEwNkgkaqhU4/3OcA+0PoZhcOXKFURHRw9ZWLwHsw8l7hcKsSspKcFTTz2FoqIiXu7hT2/sv/71L8yYMQMqlQonTpzAK6+84rO7YpDc/oJHCIHNZuMMBIbyxmTfxAaDAd3d3dwxZLBe/izswJbk5GRotdpBXydQGIZBa2srjEYj2tvbOXcXtVoNt9uNy5cvc1nT4K9lYBtzdk3RcdEIU4Vxji2x8hiIaQto2oHOzna0t8tgsdgQHh5+s3wnBg5HI6xWKyQSKShKjagold+Jnf52oOxLQa1W8+pUAww+7sfuNhUKRdDErrS0FBs2bEBhYSGmTZvGyzXPnz+PV155BadOnQIA/P73vwcAvPDCCz4/39bWhrS0NDQ2NvJyf4wG84CjR4+itLQUOTk5Q/7L9+5TZDNWDQ0NuHr1KueD52un1B9sPduMGTO4eROhQiQSIS4uDnFxcVy5i8FgQFVVFRwOByZOnBiybNtArspOpxOXLl3CpEmTEBsfC323HjaXDaAACQUQEMhkSiiVnky1SBTuVU9WD4BGRIQMsbEqyOXSgOoX+5rrQdM0Ll++DI1G06+92GDx7sGeMmUKF/djh+ewllDep41QiF15eTk2bNiAgwcP8iZ2gP+9sSx79+7FihUreLt/MAi54M2fPx9NTU145pln0NbWhgcffBA6nW7I8Sjvty1bqd7c3IzKykrExMRwhcH97SjNZjOuX7+OjIyMYQ/wsu4uIpEILS0tSE1NhdVqxVdffRWScpf+YFsD2VpEwGM8yrouiwE43JYegkRRFBQKBffFt9vtNxv39SCEcBls1jqoP3zN9XC73dygp/7amfjEuw2RPW3U1tZycT92bmwwj7GVlZVYt24dDhw4gNTUVF6v7W9vLAB88skn2Lt3L86dO8frGvgm5Edab1paWnDkyBEUFhaiubkZy5cvR05ODmbMmMFbgoCd4Wk0GtHW1oaoqCjumOi9q2hoaEBzczPS09OHRUR80dLSgqqqKqSnp/cQYG8jUJFIxCVxQlG+wxZep6am9rsDDmSyFxuXNRqNsNlsiI2NDSg04XK5cOnSJSQlJYU0BNEXbGiisrISbrebm1HCd73fjRs38MQTT+Ddd99FZmYmb9dl8fdIW1paitzcXJw4ceIWZ54hcvvH8PrCYrHg6NGjKCwsRG1tLZYuXYqcnBxeW4BYZ1qDwdCjK6K9vR0OhwMzZ84cEb2VwLd1f5mZmf0KMJ/lLgPR0dGBr7/+GmlpaUFrUWJDE0ajkZvm1p9/HHu0Tk5ODqrhaiB4H2OTk5ODUu9XW1uLxx9/HHv37sVdd93F4+q/xe12IyUlBR9//DESEhIwd+5cHDhwADNnzuQ+U1dXh8WLF+Pdd9/F/Pnz+V7C6BU8bzo7O1FcXIzCwkJcu3YNixcvRk5ODubMmcOr+LGGnS6XCzExMdBqtcNSdd+b2tpatLS0ID09PaC6v6GUuwwE22XSe7cZTFj/ODZD6t1EL5VKOYeRqVOnjpi2qYFidr3r/XzF/QaioaEBeXl52LNnD+6++26+H6EHx48fx5YtW7je2BdffLFHb+z69etRWFiISZMmAfDE1b/88ku+bj82BM8bq9WKEydO4NChQygtLcV9992HnJwczJs3b0i7MafTidLSUowbNw6JiYk9WsK8Z3uG8njLtkA5nU7ceeedQxJ3X+Uu7DCjQMXPZDLhm2++QUZGxrB1vbBN9N92elCw2WxISUnBuHHjhmVNvQk0QcGaNphMJs55ZyBLKL1ej0ceeQRvvPEGFi5cyPcjjDTGnuB5Y7fb8dFHH6GgoABfffUV7rnnHuTm5mL+/PkB7YSsVitKS0sxdepUnz5c3i1hIpGISxAE88vOflnkcvmAXQGDubZ3uQt7TGQb6PuLt+n1ejQ0NCAzM3MY+k19r4t1q1Gr1ejq6uph1OpP0iM4ax1aNpat92Njzb7q/QwGAx5++GG89tprWLx4Md+PMBIZ24LnjdPpxCeffIKCggKcP38e8+bNg06nw8KFC/vdlVksFly9etXvOJSnhcoIo9HYI5vIZ7+l2+3masfYo0GwYI+JrHtIREQYYtUEypgYSCTSHkW89fX1MJlMAR+t+aCv4mI2aXLnnXdy80/Yo7zJZOLmEsfHxw+5HtP/tfJbetK73u/999+HQqHARx99hB07duCBBx7gYdW3BYLg+cLtduPs2bPIz8/HP//5T2RlZUGn02Hx4sU9Gv1ZZ5H09PRB7dacTicnfm63m5cEgcPhwOXLl5GUlBTyoxkhBJZ2AwyGSlja7JBKaWg0KdBoJqKxsRGdnZ2YNWvWsNgo+Souttk8HRT9vax8JT38KUkaLKGos7t48SKef/75m4XaEqxYsQLPPfdc0Iw8RxCC4A0ETdP47LPPUFBQgE8++QQzZ86ETqfDhQsXEB8fj02bNvGyW2F3FQaDAQ6Ho0d/r7+7Cna3kpKSgtjY2CGvaTB476SsNjvaLSI0NDSDEMJ1mgxH3K73Ds/hVOBq+TWkp6f7/UVnGIbzKmTbwtiSJD6O56EQO4vFgocffhjPPfcccnNz0d7ejg8//BCrV68edAx7oP7YiooK/OhHP0JJSQm2bduGZ599lo9HGQyC4AUCwzD47LPPsGnTJtjtdsycOROrV6/GsmXLeDUeYIPPRqMRVqv1ltkXvmAzxMEs8fCXb009xaisvA6pVIqJEydyzxTscpeB1tXZaUVFxfUhZYi9vQpbWlogkUi4ZxqMj2AoxK6jowOPPPIIfvGLXyAvL4+Xa/rTH8uOLy0qKoJKpRIE73bi0UcfRVZWFp577jlcunQJBQUFOHnyJJKSkqDT6bBixQpeZ8jSNM05bXR2dvrMjnp3dIR6pGNfeDfcJycn9xBqp9PJiV8wDRt8wZbD8P1nxRqbsrFZ9pn8EfRQiF1XVxfy8vLwk5/8BE888QRv1w2kP/aVV16BQqEYVYI3IgZxB5M//vGPXF/lnDlzMGfOHPzud79DWVkZ8vPzsWrVKmg0Guh0Ojz00ENDPlqKxWIuscFmR5uamlBRUQGlUgmxWAyLxYKsrKwR09HBmgBoNJoevZMsMpmMa9nq3UI1lHKXgWhpacH169eRmZnJ+7E6PDwcSUlJSEpK4gS9qqrK53Q6b0IhdlarFWvWrMHatWt5FTsg8P7Y0caoFzxfTeQURWHWrFmYNWsWXn31VVRUVKCgoAAPP/wwYmJikJ2djVWrViEuLm5IX2JvMwCaplFZWQmDwQCxWIyqqiounjScsxXYsY4TJ070K2nibdjACrper0dFRQWvCQJ2lu7s2bOD/mLwFnR2h97Q0ICOjo4eXngAgi52drsdTzzxBB577DE8+eSTvF8/kP7Y0cioF7yBoCgKM2bMwMsvv4yXXnoJN27cQEFBAdasWQO5XI5Vq1ZBp9NBq9UO+geDLSgmhOCee+4BRVGcE8r169ehUCi4wT+hbG1jbbDuuOOOQc0F7e3uwpa7VFVVcc+kVqsDThIZDAbU1dVh9uzZIa/9671DZzs9rl27BrfbjZiYGJ+7YD5wOBz4wQ9+gJUrV+InP/lJUIQo0Nmxo41RH8MbLIQQ1NbWorCwEEVFRQCAVatWIScnBwkJCX7/MLLGjxEREdw8zt736ezs5Pp7Wc+4+Pj4oNa+scXX06dPv2U+8FBhn4lNELCT6PyZq6rX69HY2IiMjIxhb/FjYf8OpVIpZDIZzGYzpFIp9/fEx/Akp9OJJ598Evfeey+efvrpoO26/OmPZRmNMTxB8PyAEIKmpiYUFhbi8OHDcDgcWLlyJXQ63S0Bfm9cLhdKS0v7jI35uk93dzcMBkNQp56xnn+hyhCzLWEmk4nbQfnqXGlsbERzc3NQhhINFjZmFxkZiSlTpnC/b7VauWJnQkjAs2K9cbvdWLduHebMmYOtW7cG/Yg5UH9sc3Mz7rrrLnR0dEAkEkGhUKC8vJwr9A4hguANN4QQGI1GHD58GIcOHYLFYsFDDz0EnU6HadOm9RhCVFpaikmTJg3assiXUAy2jILFYrGgoqKClxm2g4HtXDGZTGAYhsuOtra2wmw2Iz09fcQ41vQldr1hZ66ws2JZQ4D+ypJY3G43fvrTnyI1NRW/+tWvxlQ8zQ8EwRtptLS0oKioCIWFhTAajVi+fDnS0tKwZ88evPfee7wVFNtsNk4oAHC7pEBKNcxmM27cuDGsJgDesNnRmpoaOBwOJCYmQqvVhqTcZSAIISgrKxtQ7HrDmjaYTCZ0dnb2awhA0zQ2bdqEhIQEbNu2bdifeQQiCN5IxmKx4A9/+APeeustTJ8+HQsWLEBubi7v7VkOh4NrcWOLgrVabb9FuR53Yc9c3ZFSDgMA33zzDbq6upCamsoZtQa73GUgBit2veltCMAmciIjIxEREYEtW7ZAqVRi586dw5qpH8EIgjeS+fzzz/Hzn/8chw4dQkxMDOfpV1VVhSVLliAnJwdZWVm8/nCzxymj0Qin04m4uDhotdoeriENDQ0wGAwjKjbGztZ1OBy48847e4iaL3eXYPbD9l4XH2Ln67qdnZ1obGzE97//fYhEImi1Wuzfvz8kQ5luUwTBG8lYrVbY7fZbjrFWqxXHjx9HYWEhysrKsGjRIuh0uiF7+vWmt1V6XFwcXC4X7Hb7iIqNEUJw7do1MAyD1NTUfndwvd1dhlLu4s+6giF23jAMg5deeglmsxmZmZk4evQopkyZgnfeeWfQ1xyoN5YQgs2bN+P48eOIiIjAX/7yF2RlZQ31UUKBIHi3O6ynX35+PkpKSrBgwQLk5OQE7Ok3EG63G2VlZejs7IREIkFsbCy0Wm3ILJP6ghCCiooKiESigAc3eZe7mM1myOVyLpY51BKWUIgdIQSvvvoqTCYT/u///o97ATkcjkEnovzpjT1+/Dj++7//G8ePH8cXX3yBzZs33y7dFYLgjSacTif+8Y9/ID8/H1988QXmzZuHnJwcLFy4cEhfYEIIrl69CrFYjJSUlB5HxI6ODm6EJTsVLVQQQjiTU181iYHib7mLP+sKhdht374d1dXV2LdvH2+7bX96Y3/6059i0aJFWLNmDQBg+vTpOHPmzO1wlBZ6aUcTMpkMy5cvx/Lly+F2u/Hpp58iPz8fL7zwArKyspCTk4Pvfve7Ab392SJZtv2JoiifIywNBkNAIyyHive6+BKVyMhITJ48mRv7aDQa8fXXX4NhGL/r4kIldn/4wx9w7do17N+/n9fQgj+9sb4+09jYeDsIHu8IgjdCkEgkWLJkCZYsWQKapnHu3DkUFBTg17/+NdLS0pCTk4P777+/3zIUdhB1XFwckpKSfH5GJBJBrVZDrVbf0g7W1wjLocIwDEpLS6FSqYLm6CyXy3uYAbDtYE6nE2q1Glqt9havwlCJ3a5du1BSUoKDBw8GJe7YG1/dPAN9ZqwgCN4IRCwW47777sN9990HhmHw+eefo6CgANu2bUNKSgpyc3OxdOnSHp5+7GzWxMREv9/cFEVBpVJBpVJxIyyNRiNu3LjBmWXGxcUN6UtK0zRKS0sRFxcXtB7U3shkMiQkJCAhIYHzKqyurkZ3d3ePmbf+FBUPBUII3n77be7lFYxWOX96Y8d6/6w3t3UMr7W1FY899hhqamqQnJyMgwcP3tIXWl9fjx/+8Idobm6GSCTChg0bsHnz5mFa8dBgGAYlJSWcp19ycjKys7Mxa9Ys7NixA6+99hovs1m9zTLNZjPXCxtocoC1nRo3bhwSEhKGvK6hwjAM51VoNBq5/uZgHOcJIXjnnXdw7NgxFBUVBa3Q25/e2OLiYuzatYtLWvziF7/AhQsXgrIenhGSFt4899xziI2NxdatW7F9+3a0tbVhx44dPT6j1+uh1+uRlZWFzs5OzJkzB0VFRT2yWLcjbEzs7bffxv79+zF//nxkZ2dj5cqVvJsBeCcH/B1h6XK5cPnyZSQkJIyoWBF7jI2IiEBsbOwt5S58Odb89a9/RX5+Pj744IOgz/EdqDeWEIJNmzbh5MmTiIiIwDvvvBO04d08IwieN97ZJr1ej0WLFqGysrLf/0en02HTpk1YunRpiFYZPK5evYo1a9Zg7969CA8PR0FBAYqLixETEwOdToeVK1ciPj6e13vabDYYDAaYTCZQFMWJn/cOhj1eJyUlDbqPOBj0FbPju9zl/fffx759+1BcXDwWBu0EE0HwvFEqlbBYLNy/q1QqtLW19fn5mpoaLFy4EGVlZcPh/MA7BoMBFosF06dP536PEILr16+joKAAR48ehVwuR3Z2NrKzs4fk6eeL3kYAbM9oZWUlJk+ezLvYDoVAEhRDKXc5dOgQ3n77bRQXFw/7rJJRwNgTvPvvvx/Nzc23/P62bduwdu1avwWvq6sL9913H1588UWsXr06WMsdUXh7+h0+fBgikYjz9JswYQKv4ud0OtHU1ITq6mrIZDKMHz+ea3EbboaSjWVnX3iLukaj8XlMPXr0KN58800UFxdDqVTytPoxzdgTvP7w90jrcrmwcuVKPPDAA3j66aeHYaXDj7en36FDh+B0Ojk350mTJg1Z/Gw2Gy5fvozp06dDoVBw/b2sXZKvspBQwGfpia++5bCwMEyYMAEffvghdu7cieLiYqjVap5WP+YRBM+b//qv/4JareaSFq2trdi5c2ePzxBCsHbtWsTGxuJPf/rT8Cx0hEEIgcFg4Dz9Ojo68OCDDyInJwdTp04NWJRY9+QZM2bcMgHOe4Rld3c3VxPnj1fcUAlmnR37XDt37sSpU6fgdruxe/duPPjgg4LzCX8IgudNS0sL8vLyUFdXh6SkJOTn5yM2NhZNTU1Yv349jh8/jnPnzuHee+/tYdH0u9/9Dg8++OAwr37kYDabOU8/s9mM5cuXQ6fTYcaMGQOKUnd3N0pLS/1yT+49wlKlUkGr1QbFAioURcUAcPbsWbz00kvYvHkzzpw5g3//+984fPgw7rjjjiFf25+yKwBYt24djh07Bo1Gg7KysiHfdwQhCJ5AcGlra8MHH3yAQ4cOoa6uDsuWLUNubi7S0tJu2bmwVvGzZs0KeLC5LwsorVbr0ygzUEIldp999hmef/55HD16lKszdLvdEIlEvOzy/Cm7Ajyiq1Ao8MMf/lAQvIEuKAief4zFt21HRweOHTuGQ4cOoaqqCvfffz90Oh2ysrLwr3/9Cy0tLbj//vuHnJgghHDmn21tbYiKioJWq0VsbGzANXGhErsLFy5gy5YtOHr0aNA6SAIpu6qpqcHKlStv+5+5XgiCN1yM9bdtV1cXTpw4gcLCQnz55ZdwOp347W9/i9zcXF77bgkhaG9v5yaeRUZGQqvV+lUQHCqxKykpwVNPPYWioqKgzacFAiu7EgTPP4ReWj85cuQIzpw5AwBYu3YtFi1a5FPwFi5ciJqamtAuLgQoFAo8+uijiI+PxzPPPIONGzfio48+ws6dO3n19KMoCkqlEkqlskdBcHV1NTfCMi4u7paCYNZ6KthiV1paio0bN6KwsJAXseuv7EqAfwTB8xODwcC1SI0fPx5Go3GYVzQ8nDp1CsXFxRg3bhx+/OMfw+l04uOPP8bBgwfx7LPP4u6770ZOTg7uvffeITfLUxSF6OhoREdHY+rUqVx/b0lJCaRSKbRaLdcN8fXXXyMiIiKoYldeXo4NGzbg4MGDmDZtGi/XPH36dJ//TavVQq/Xc0daPvqkxzqC4HkhvG0HhjWYZJHJZFixYgVWrFgBl8uFTz/9FAUFBdi6dSvmzJmDnJwcLFq0iJdh1QqFgvPTs1qtMBqNuHTpEmw2G6Kjo4NqUFBZWYl169bhwIEDSE1NDdp9vMnOzsa+ffuwdetW7Nu3DzqdLiT3Hc0IMTw/EQLIgUHTNP75z3+isLAQZ86cwaxZs5CTk4MlS5YENFqyP9hjrEwmg1wuh9FoBCGE6+/l6z43btzAE088gXfffReZmZm8XNMf/Cm7AoA1a9bgzJkzMJvN0Gq1ePXVV/HjH/84ZOsMIkLSYrjwp8iZRRC8ntA0jc8//xyFhYU4ffo0pk+fjpycHCxbtmzQGV5W7HofYx0OB9cN4Xa7uVawwd6ntrYWjz/+OPbu3Xu7OIyMJgTBGy6Ety0/MAyDr776CgUFBTh16hQmT56M7OxsrFixwm9Dh77ErjculwsmkwkGg4FrBdNoNH63uDU0NCAvLw979uzB3Xff7fczCvCGIHgCoweGYXDlyhXk5+fjxIkTGDduHHQ6HR566KE+Pf38FbveuN1ubudns9mgVquh0Wj6bHHT6/V45JFH8MYbb2DhwoWDfkaBISEI3mhnFM8Y7RdCCMrLyzlPP5VKxXn6xcXFAfAcjcvLy4dcesK2uBkMBnR1dXG272yLm8FgwMMPP4zXXnsNixcv5usRBQJHELzRzCifMeo3hBBUVVVxnn7h4eFYuXIlTp48iby8PHzve9/j7V7etu+7du1CR0cHrl+/jh07dgj91sOPIHijmVE+Y3RQEEJw48YNPPbYY3C5XFAqlUHz9DMYDHjyySehUChQX1+PuXPn4uWXX0ZycjIv1x9rM1h4gHfBE3xsRhB9zQ8N9DOjCYqi8Prrr0On0+Hy5cs4cOAAZDIZ1q9fj2XLluGNN95AbW2tz1GEgWCxWPC9730P//mf/4ni4mJcunQJ69atC9gUoT+2b9+OJUuWoKqqCkuWLMH27dtv+YxEIsHrr7+Oq1ev4vPPP8dbb72F8vJy3tYw1hEEbwQhzBj1zS9/+Uv86le/AkVRSExM5KyYCgsLERMTg5///OdYvHgxXnvtNVRVVQUsfh0dHcjLy8PTTz+NnJwcAJ75vffccw8XP+SDI0eOYO3atQA87YlFRUW3fGb8+PFcTDYqKgozZswY1S+0UCMI3ghCmDHqG19uJBRFYdy4cdi4cSNOnz6NY8eOYfz48Xj++eexaNEibN++HeXl5QOKX1dXFx577DH8x3/8Bx599NFgPQKAwNsTa2pqcPHiRcybNy+o6xpTEEL6+yUQQlwuF5k8eTL55ptviMPhIOnp6aSsrKzHZ44dO0aWL19OGIYh58+fJ3Pnzh2m1Y5cWlpayDvvvENWrVpFMjMzyfPPP0/Onz9POjs7SXd3N/fLZDKR7373u+TPf/4zb/desmQJmTlz5i2/ioqKSExMTI/PKpXKPq/T2dlJsrKySGFhIW9ruw0ZSJ8C/iUI3gijuLiYTJs2jUyZMoX89re/JYQQsnv3brJ7925CCCEMw5CNGzeSKVOmkLS0NPLvf/97OJc74rFYLOS9994jq1evJunp6eTpp58mZ8+eJSaTiSxdupT8z//8D2EYJiRrSUlJIU1NTYQQQpqamkhKSorPzzmdTrJs2TLy+uuvh2RdIxjeBU/I0gqMGVhPv4KCApw+fRrPPPMMXnjhhZDFQIUZLAEjlKUICPBBS0sLYmNjQ5rwEWawBIwgeAL8MlBnR0VFBX70ox+hpKQE27Ztw7PPPjtMKxUYgwiOIy5jBQAAAydJREFUxwL8QdM0nnrqqR6dHdnZ2T06O2JjY/Hmm2/6LKEQELjdEMpSxjAXLlzA1KlTMWXKFMhkMjz++OM4cuRIj89oNBrMnTt3yO7FAgIjAUHwxjBjrWtDQEAQvDGMr/jtaO/aEBjbCII3hhmLXRsCYxtB8MYwc+fORVVVFaqrq+F0OvH3v/8d2dnZw72s24LW1lYsXboU06ZNw9KlS33Oi7Xb7fjOd76DjIwMzJw5E7/+9a+HYaUC3ghlKWOc48ePY8uWLaBpGuvWrcOLL76IPXv2AAB+9rOfobm5GXfddRc6OjogEomgUChQXl7utx37aMWfweyEEHR3d0OhUMDlcmHBggV44403BLt4/xHq8AQERgKBTLEDAKvVigULFmD37t2CGYD/CH54Arc/J0+exPTp0zF16lSfnnD79+9Heno60tPTMX/+fFy+fHkYVtk//jqf0DSNzMxMaDQaLF26VBC7YUYoPBYIKf4UO0+ePBmffvopVCoVTpw4gQ0bNgyLjT0fg9nFYjEuXboEi8WC3NxclJWVIS0tjc9lCgSAIHgCIcW72BkAV+zsLXjz58/n/vnuu+9GQ0NDyNcJAKdPn+7zv2m1Wuj1eu5Iq9Fo+r2WUqnEokWLcPLkSUHwhhHhSCsQUgItdt67dy9WrFgRiqUFRHZ2Nvbt2wcA2LdvH3Q63S2fMZlMsFgsAACbzYbTp08jNTU1lMsU6IWwwxMIKYEUO3/yySfYu3cvzp07F+xlBczWrVuRl5eHvXv3cs4nAHo4n+j1eqxduxY0TYNhGOTl5WHlypXDvPKxjSB4AiHF32Ln0tJSrF+/HidOnIBarQ7lEv1CrVbj448/vuX3J0yYgOPHjwMA0tPTcfHixVAvTaAfhCOtQEjxp9i5rq4Oq1evxl//+lekpKQM00oFRiPCDk8gpEgkEuzatQsPPPAAV+w8c+bMHsXOv/nNb9DS0oKNGzdy/8+XX345nMsWGCUMVHgsICAgMGoQjrQCAgJjBkHwBAQExgyC4AkICIwZBMETEBAYMwiCJyAgMGYQBE9AQGDMIAiegIDAmEEQPAEBgTHD/wfptcsfHNzEUgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "fig = plt.figure()\n", + "ax = Axes3D(fig, auto_add_to_figure=False)\n", + "fig.add_axes(ax)\n", + "for c in range(len(np.unique(y_test))):\n", + " ax.plot(\n", + " embedded_features[y_test==c, 0], \n", + " embedded_features[y_test==c, 1], \n", + " embedded_features[y_test==c, 2], \n", + " '.', \n", + " alpha=0.1,\n", + " )\n", + "plt.title('ArcFace')\n", + "plt.show()" + ] + } + ], + "metadata": { + "environment": { + "kernel": "python3", + "name": "tf2-gpu.2-8.m91", + "type": "gcloud", + "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-8:m91" + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 8093f1f7587e4557303c6cb2c57818e17c38eaa2 Mon Sep 17 00:00:00 2001 From: Owen Vallis Date: Wed, 12 Oct 2022 15:41:42 +0000 Subject: [PATCH 25/25] Fix formatting errors. --- benchmark/supervised/components/__init__.py | 4 ++-- benchmark/supervised/train.py | 2 +- tensorflow_similarity/retrieval_metrics/map_at_k.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/supervised/components/__init__.py b/benchmark/supervised/components/__init__.py index 0eda9b56..e1093a0b 100644 --- a/benchmark/supervised/components/__init__.py +++ b/benchmark/supervised/components/__init__.py @@ -1,6 +1,6 @@ +from . import metrics # noqa +from . import utils # noqa from .architectures import make_architecture # noqa from .augmentations import make_augmentations # noqa from .losses import make_loss # noqa -from . import metrics # noqa from .optimizers import make_optimizer # noqa -from . import utils # noqa diff --git a/benchmark/supervised/train.py b/benchmark/supervised/train.py index 474506dc..5db82487 100644 --- a/benchmark/supervised/train.py +++ b/benchmark/supervised/train.py @@ -106,7 +106,7 @@ def run(config): version, dataset_name, architecture_name, - aconf['embedding'], + aconf["embedding"], loss_name, opt_name, fold, diff --git a/tensorflow_similarity/retrieval_metrics/map_at_k.py b/tensorflow_similarity/retrieval_metrics/map_at_k.py index 0fed2e2e..8c074365 100644 --- a/tensorflow_similarity/retrieval_metrics/map_at_k.py +++ b/tensorflow_similarity/retrieval_metrics/map_at_k.py @@ -157,7 +157,7 @@ def compute( avg_p_at_k = tf.map_fn( lambda x: tf.math.reduce_sum(x[0][: x[1][0]]) / tf.cast(x[1], dtype="float"), elems, - fn_output_signature="float" + fn_output_signature="float", ) else: avg_p_at_k = tf.math.divide(