diff --git a/advanced_source/sharding.rst b/advanced_source/sharding.rst index 7dfeeb88bf..8044463b0c 100644 --- a/advanced_source/sharding.rst +++ b/advanced_source/sharding.rst @@ -14,9 +14,10 @@ Requirements: - python >= 3.7 We highly recommend CUDA when using torchRec. If using CUDA: - cuda >= 11.0 +.. Should these be updated? .. code:: python - # install conda to make installying pytorch with cudatoolkit 11.3 easier. + # install conda to make installying pytorch with cudatoolkit 11.3 easier. !sudo rm Miniconda3-py37_4.9.2-Linux-x86_64.sh Miniconda3-py37_4.9.2-Linux-x86_64.sh.* !sudo wget https://repo.anaconda.com/miniconda/Miniconda3-py37_4.9.2-Linux-x86_64.sh !sudo chmod +x Miniconda3-py37_4.9.2-Linux-x86_64.sh @@ -213,7 +214,7 @@ embedding table placement using planner and generate sharded model using ) sharders = [cast(ModuleSharder[torch.nn.Module], EmbeddingBagCollectionSharder())] plan: ShardingPlan = planner.collective_plan(module, sharders, pg) - + sharded_model = DistributedModelParallel( module, env=ShardingEnv.from_process_group(pg), @@ -234,7 +235,7 @@ ranks. .. code:: python import multiprocess - + def spmd_sharing_simulation( sharding_type: ShardingType = ShardingType.TABLE_WISE, world_size = 2, @@ -254,7 +255,7 @@ ranks. ) p.start() processes.append(p) - + for p in processes: p.join() assert 0 == p.exitcode @@ -333,4 +334,3 @@ With data parallel, we will repeat the tables for all devices. rank:0,sharding plan: {'': {'large_table_0': ParameterSharding(sharding_type='data_parallel', compute_kernel='batched_dense', ranks=[0, 1], sharding_spec=None), 'large_table_1': ParameterSharding(sharding_type='data_parallel', compute_kernel='batched_dense', ranks=[0, 1], sharding_spec=None), 'small_table_0': ParameterSharding(sharding_type='data_parallel', compute_kernel='batched_dense', ranks=[0, 1], sharding_spec=None), 'small_table_1': ParameterSharding(sharding_type='data_parallel', compute_kernel='batched_dense', ranks=[0, 1], sharding_spec=None)}} rank:1,sharding plan: {'': {'large_table_0': ParameterSharding(sharding_type='data_parallel', compute_kernel='batched_dense', ranks=[0, 1], sharding_spec=None), 'large_table_1': ParameterSharding(sharding_type='data_parallel', compute_kernel='batched_dense', ranks=[0, 1], sharding_spec=None), 'small_table_0': ParameterSharding(sharding_type='data_parallel', compute_kernel='batched_dense', ranks=[0, 1], sharding_spec=None), 'small_table_1': ParameterSharding(sharding_type='data_parallel', compute_kernel='batched_dense', ranks=[0, 1], sharding_spec=None)}} - diff --git a/advanced_source/torch_script_custom_ops.rst b/advanced_source/torch_script_custom_ops.rst index 0a0e6e2bd7..f59aa6e883 100644 --- a/advanced_source/torch_script_custom_ops.rst +++ b/advanced_source/torch_script_custom_ops.rst @@ -190,6 +190,8 @@ Environment setup We need an installation of PyTorch and OpenCV. The easiest and most platform independent way to get both is to via Conda:: +.. these need to be updated + conda install -c pytorch pytorch conda install opencv diff --git a/beginner_source/introyt/captumyt.py b/beginner_source/introyt/captumyt.py index abf2391d25..25998ddd90 100644 --- a/beginner_source/introyt/captumyt.py +++ b/beginner_source/introyt/captumyt.py @@ -106,14 +106,7 @@ - Matplotlib version 3.3.4, since Captum currently uses a Matplotlib function whose arguments have been renamed in later versions -To install Captum in an Anaconda or pip virtual environment, use the -appropriate command for your environment below: - -With ``conda``: - -.. code-block:: sh - - conda install pytorch torchvision captum flask-compress matplotlib=3.3.4 -c pytorch +To install Captum, use the appropriate command for your environment below: With ``pip``: @@ -127,51 +120,56 @@ A First Example --------------- - + To start, let’s take a simple, visual example. We’ll start with a ResNet model pretrained on the ImageNet dataset. We’ll get a test input, and use different **Feature Attribution** algorithms to examine how the input images affect the output, and see a helpful visualization of this input attribution map for some test images. - -First, some imports: -""" +First, some imports: -import torch -import torch.nn.functional as F -import torchvision.transforms as transforms -import torchvision.models as models +""" -import captum -from captum.attr import IntegratedGradients, Occlusion, LayerGradCam, LayerAttribution -from captum.attr import visualization as viz +import json import os, sys -import json -import numpy as np -from PIL import Image +import captum import matplotlib.pyplot as plt + +import numpy as np +import torch +import torch.nn.functional as F +import torchvision.models as models +import torchvision.transforms as transforms +from captum.attr import ( + IntegratedGradients, + LayerAttribution, + LayerGradCam, + Occlusion, + visualization as viz, +) from matplotlib.colors import LinearSegmentedColormap +from PIL import Image ######################################################################### # Now we’ll use the TorchVision model library to download a pretrained # ResNet. Since we’re not training, we’ll place it in evaluation mode for # now. -# +# -model = models.resnet18(weights='IMAGENET1K_V1') +model = models.resnet18(weights="IMAGENET1K_V1") model = model.eval() ####################################################################### # The place where you got this interactive notebook should also have an # ``img`` folder with a file ``cat.jpg`` in it. -# +# -test_img = Image.open('img/cat.jpg') +test_img = Image.open("img/cat.jpg") test_img_data = np.asarray(test_img) plt.imshow(test_img_data) plt.show() @@ -183,26 +181,23 @@ # range of values. We’ll also pull in the list of human-readable labels # for the categories our model recognizes - that should be in the ``img`` # folder as well. -# +# # model expects 224x224 3-color image -transform = transforms.Compose([ - transforms.Resize(224), - transforms.CenterCrop(224), - transforms.ToTensor() -]) +transform = transforms.Compose( + [transforms.Resize(224), transforms.CenterCrop(224), transforms.ToTensor()] +) # standard ImageNet normalization transform_normalize = transforms.Normalize( - mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225] - ) + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] +) transformed_img = transform(test_img) input_img = transform_normalize(transformed_img) -input_img = input_img.unsqueeze(0) # the model requires a dummy batch dimension +input_img = input_img.unsqueeze(0) # the model requires a dummy batch dimension -labels_path = 'img/imagenet_class_index.json' +labels_path = "img/imagenet_class_index.json" with open(labels_path) as json_data: idx_to_labels = json.load(json_data) @@ -210,89 +205,96 @@ ###################################################################### # Now, we can ask the question: What does our model think this image # represents? -# +# output = model(input_img) output = F.softmax(output, dim=1) prediction_score, pred_label_idx = torch.topk(output, 1) pred_label_idx.squeeze_() predicted_label = idx_to_labels[str(pred_label_idx.item())][1] -print('Predicted:', predicted_label, '(', prediction_score.squeeze().item(), ')') +print("Predicted:", predicted_label, "(", prediction_score.squeeze().item(), ")") ###################################################################### # We’ve confirmed that ResNet thinks our image of a cat is, in fact, a # cat. But *why* does the model think this is an image of a cat? -# +# # For the answer to that, we turn to Captum. -# +# ########################################################################## # Feature Attribution with Integrated Gradients # --------------------------------------------- -# +# # **Feature attribution** attributes a particular output to features of # the input. It uses a specific input - here, our test image - to generate # a map of the relative importance of each input feature to a particular # output feature. -# +# # `Integrated # Gradients `__ is one of # the feature attribution algorithms available in Captum. Integrated # Gradients assigns an importance score to each input feature by # approximating the integral of the gradients of the model’s output with # respect to the inputs. -# +# # In our case, we’re going to be taking a specific element of the output # vector - that is, the one indicating the model’s confidence in its # chosen category - and use Integrated Gradients to understand what parts # of the input image contributed to this output. -# +# # Once we have the importance map from Integrated Gradients, we’ll use the # visualization tools in Captum to give a helpful representation of the # importance map. Captum’s ``visualize_image_attr()`` function provides a # variety of options for customizing display of your attribution data. # Here, we pass in a custom Matplotlib color map. -# +# # Running the cell with the ``integrated_gradients.attribute()`` call will # usually take a minute or two. -# +# # Initialize the attribution algorithm with the model integrated_gradients = IntegratedGradients(model) -# Ask the algorithm to attribute our output target to -attributions_ig = integrated_gradients.attribute(input_img, target=pred_label_idx, n_steps=200) +# Ask the algorithm to attribute our output target to +attributions_ig = integrated_gradients.attribute( + input_img, target=pred_label_idx, n_steps=200 +) # Show the original image for comparison -_ = viz.visualize_image_attr(None, np.transpose(transformed_img.squeeze().cpu().detach().numpy(), (1,2,0)), - method="original_image", title="Original Image") +_ = viz.visualize_image_attr( + None, + np.transpose(transformed_img.squeeze().cpu().detach().numpy(), (1, 2, 0)), + method="original_image", + title="Original Image", +) -default_cmap = LinearSegmentedColormap.from_list('custom blue', - [(0, '#ffffff'), - (0.25, '#0000ff'), - (1, '#0000ff')], N=256) +default_cmap = LinearSegmentedColormap.from_list( + "custom blue", [(0, "#ffffff"), (0.25, "#0000ff"), (1, "#0000ff")], N=256 +) -_ = viz.visualize_image_attr(np.transpose(attributions_ig.squeeze().cpu().detach().numpy(), (1,2,0)), - np.transpose(transformed_img.squeeze().cpu().detach().numpy(), (1,2,0)), - method='heat_map', - cmap=default_cmap, - show_colorbar=True, - sign='positive', - title='Integrated Gradients') +_ = viz.visualize_image_attr( + np.transpose(attributions_ig.squeeze().cpu().detach().numpy(), (1, 2, 0)), + np.transpose(transformed_img.squeeze().cpu().detach().numpy(), (1, 2, 0)), + method="heat_map", + cmap=default_cmap, + show_colorbar=True, + sign="positive", + title="Integrated Gradients", +) ####################################################################### # In the image above, you should see that Integrated Gradients gives us # the strongest signal around the cat’s location in the image. -# +# ########################################################################## # Feature Attribution with Occlusion # ---------------------------------- -# +# # Gradient-based attribution methods help to understand the model in terms # of directly computing out the output changes with respect to the input. # *Perturbation-based attribution* methods approach this more directly, by @@ -300,7 +302,7 @@ # `Occlusion `__ is one such method. # It involves replacing sections of the input image, and examining the # effect on the output signal. -# +# # Below, we set up Occlusion attribution. Similarly to configuring a # convolutional neural network, you can specify the size of the target # region, and a stride length to determine the spacing of individual @@ -310,42 +312,45 @@ # image with the positive attribution regions. The masking gives a very # instructive view of what regions of our cat photo the model found to be # most “cat-like”. -# +# occlusion = Occlusion(model) -attributions_occ = occlusion.attribute(input_img, - target=pred_label_idx, - strides=(3, 8, 8), - sliding_window_shapes=(3,15, 15), - baselines=0) +attributions_occ = occlusion.attribute( + input_img, + target=pred_label_idx, + strides=(3, 8, 8), + sliding_window_shapes=(3, 15, 15), + baselines=0, +) -_ = viz.visualize_image_attr_multiple(np.transpose(attributions_occ.squeeze().cpu().detach().numpy(), (1,2,0)), - np.transpose(transformed_img.squeeze().cpu().detach().numpy(), (1,2,0)), - ["original_image", "heat_map", "heat_map", "masked_image"], - ["all", "positive", "negative", "positive"], - show_colorbar=True, - titles=["Original", "Positive Attribution", "Negative Attribution", "Masked"], - fig_size=(18, 6) - ) +_ = viz.visualize_image_attr_multiple( + np.transpose(attributions_occ.squeeze().cpu().detach().numpy(), (1, 2, 0)), + np.transpose(transformed_img.squeeze().cpu().detach().numpy(), (1, 2, 0)), + ["original_image", "heat_map", "heat_map", "masked_image"], + ["all", "positive", "negative", "positive"], + show_colorbar=True, + titles=["Original", "Positive Attribution", "Negative Attribution", "Masked"], + fig_size=(18, 6), +) ###################################################################### # Again, we see greater significance placed on the region of the image # that contains the cat. -# +# ######################################################################### # Layer Attribution with Layer GradCAM # ------------------------------------ -# +# # **Layer Attribution** allows you to attribute the activity of hidden # layers within your model to features of your input. Below, we’ll use a # layer attribution algorithm to examine the activity of one of the # convolutional layers within our model. -# +# # GradCAM computes the gradients of the target output with respect to the # given layer, averages for each output channel (dimension 2 of output), # and multiplies the average gradient for each channel by the layer @@ -353,19 +358,21 @@ # designed for convnets; since the activity of convolutional layers often # maps spatially to the input, GradCAM attributions are often upsampled # and used to mask the input. -# +# # Layer attribution is set up similarly to input attribution, except that # in addition to the model, you must specify a hidden layer within the # model that you wish to examine. As above, when we call ``attribute()``, # we specify the target class of interest. -# +# layer_gradcam = LayerGradCam(model, model.layer3[1].conv2) attributions_lgc = layer_gradcam.attribute(input_img, target=pred_label_idx) -_ = viz.visualize_image_attr(attributions_lgc[0].cpu().permute(1,2,0).detach().numpy(), - sign="all", - title="Layer 3 Block 1 Conv 2") +_ = viz.visualize_image_attr( + attributions_lgc[0].cpu().permute(1, 2, 0).detach().numpy(), + sign="all", + title="Layer 3 Block 1 Conv 2", +) ########################################################################## @@ -373,7 +380,7 @@ # `LayerAttribution `__ # base class to upsample this attribution data for comparison to the input # image. -# +# upsamp_attr_lgc = LayerAttribution.interpolate(attributions_lgc, input_img.shape[2:]) @@ -381,53 +388,63 @@ print(upsamp_attr_lgc.shape) print(input_img.shape) -_ = viz.visualize_image_attr_multiple(upsamp_attr_lgc[0].cpu().permute(1,2,0).detach().numpy(), - transformed_img.permute(1,2,0).numpy(), - ["original_image","blended_heat_map","masked_image"], - ["all","positive","positive"], - show_colorbar=True, - titles=["Original", "Positive Attribution", "Masked"], - fig_size=(18, 6)) +_ = viz.visualize_image_attr_multiple( + upsamp_attr_lgc[0].cpu().permute(1, 2, 0).detach().numpy(), + transformed_img.permute(1, 2, 0).numpy(), + ["original_image", "blended_heat_map", "masked_image"], + ["all", "positive", "positive"], + show_colorbar=True, + titles=["Original", "Positive Attribution", "Masked"], + fig_size=(18, 6), +) ####################################################################### # Visualizations such as this can give you novel insights into how your # hidden layers respond to your input. -# +# ########################################################################## # Visualization with Captum Insights # ---------------------------------- -# +# # Captum Insights is an interpretability visualization widget built on top # of Captum to facilitate model understanding. Captum Insights works # across images, text, and other features to help users understand feature # attribution. It allows you to visualize attribution for multiple # input/output pairs, and provides visualization tools for image, text, # and arbitrary data. -# +# # In this section of the notebook, we’ll visualize multiple image # classification inferences with Captum Insights. -# +# # First, let’s gather some image and see what the model thinks of them. # For variety, we’ll take our cat, a teapot, and a trilobite fossil: -# +# -imgs = ['img/cat.jpg', 'img/teapot.jpg', 'img/trilobite.jpg'] +imgs = ["img/cat.jpg", "img/teapot.jpg", "img/trilobite.jpg"] for img in imgs: img = Image.open(img) transformed_img = transform(img) input_img = transform_normalize(transformed_img) - input_img = input_img.unsqueeze(0) # the model requires a dummy batch dimension + input_img = input_img.unsqueeze(0) # the model requires a dummy batch dimension output = model(input_img) output = F.softmax(output, dim=1) prediction_score, pred_label_idx = torch.topk(output, 1) pred_label_idx.squeeze_() predicted_label = idx_to_labels[str(pred_label_idx.item())][1] - print('Predicted:', predicted_label, '/', pred_label_idx.item(), ' (', prediction_score.squeeze().item(), ')') + print( + "Predicted:", + predicted_label, + "/", + pred_label_idx.item(), + " (", + prediction_score.squeeze().item(), + ")", + ) ########################################################################## @@ -437,9 +454,9 @@ # imported below. The ``AttributionVisualizer`` expects batches of data, # so we’ll bring in Captum’s ``Batch`` helper class. And we’ll be looking # at images specifically, so well also import ``ImageFeature``. -# +# # We configure the ``AttributionVisualizer`` with the following arguments: -# +# # - An array of models to be examined (in our case, just the one) # - A scoring function, which allows Captum Insights to pull out the # top-k predictions from a model @@ -447,15 +464,17 @@ # - A list of features to look for - in our case, an ``ImageFeature`` # - A dataset, which is an iterable object returning batches of inputs # and labels - just like you’d use for training -# +# from captum.insights import AttributionVisualizer, Batch from captum.insights.attr_vis.features import ImageFeature + # Baseline is all-zeros input - this may differ depending on your data def baseline_func(input): return input * 0 + # merging our image transforms from above def full_img_transform(input): i = Image.open(input) @@ -478,7 +497,7 @@ def full_img_transform(input): input_transforms=[], ) ], - dataset=[Batch(input_imgs, labels=[282,849,69])] + dataset=[Batch(input_imgs, labels=[282, 849, 69])], ) @@ -488,12 +507,12 @@ def full_img_transform(input): # configure different attribution algorithms in a visual widget, after # which it will compute and display the attributions. *That* process will # take a few minutes. -# +# # Running the cell below will render the Captum Insights widget. You can # then choose attributions methods and their arguments, filter model # responses based on predicted class or prediction correctness, see the # model’s predictions with associated probabilities, and view heatmaps of # the attribution compared with the original image. -# +# visualizer.render() diff --git a/beginner_source/introyt/tensorboardyt_tutorial.py b/beginner_source/introyt/tensorboardyt_tutorial.py index 49d321bd6d..b932ed6eb2 100644 --- a/beginner_source/introyt/tensorboardyt_tutorial.py +++ b/beginner_source/introyt/tensorboardyt_tutorial.py @@ -24,13 +24,6 @@ To run this tutorial, you’ll need to install PyTorch, TorchVision, Matplotlib, and TensorBoard. -With ``conda``: - -.. code-block:: sh - - conda install pytorch torchvision -c pytorch - conda install matplotlib tensorboard - With ``pip``: .. code-block:: sh @@ -43,14 +36,18 @@ Introduction ------------ - + In this notebook, we’ll be training a variant of LeNet-5 against the Fashion-MNIST dataset. Fashion-MNIST is a set of image tiles depicting various garments, with ten class labels indicating the type of garment -depicted. +depicted. """ +# Image display +import matplotlib.pyplot as plt +import numpy as np + # PyTorch model and training necessities import torch import torch.nn as nn @@ -61,10 +58,6 @@ import torchvision import torchvision.transforms as transforms -# Image display -import matplotlib.pyplot as plt -import numpy as np - # PyTorch TensorBoard support from torch.utils.tensorboard import SummaryWriter @@ -79,51 +72,59 @@ ###################################################################### # Showing Images in TensorBoard # ----------------------------- -# +# # Let’s start by adding sample images from our dataset to TensorBoard: -# +# # Gather datasets and prepare them for consumption transform = transforms.Compose( - [transforms.ToTensor(), - transforms.Normalize((0.5,), (0.5,))]) + [transforms.ToTensor(), transforms.Normalize((0.5,), (0.5,))] +) # Store separate training and validations splits in ./data -training_set = torchvision.datasets.FashionMNIST('./data', - download=True, - train=True, - transform=transform) -validation_set = torchvision.datasets.FashionMNIST('./data', - download=True, - train=False, - transform=transform) - -training_loader = torch.utils.data.DataLoader(training_set, - batch_size=4, - shuffle=True, - num_workers=2) - - -validation_loader = torch.utils.data.DataLoader(validation_set, - batch_size=4, - shuffle=False, - num_workers=2) +training_set = torchvision.datasets.FashionMNIST( + "./data", download=True, train=True, transform=transform +) +validation_set = torchvision.datasets.FashionMNIST( + "./data", download=True, train=False, transform=transform +) + +training_loader = torch.utils.data.DataLoader( + training_set, batch_size=4, shuffle=True, num_workers=2 +) + + +validation_loader = torch.utils.data.DataLoader( + validation_set, batch_size=4, shuffle=False, num_workers=2 +) # Class labels -classes = ('T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', - 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle Boot') +classes = ( + "T-shirt/top", + "Trouser", + "Pullover", + "Dress", + "Coat", + "Sandal", + "Shirt", + "Sneaker", + "Bag", + "Ankle Boot", +) + # Helper function for inline image display def matplotlib_imshow(img, one_channel=False): if one_channel: img = img.mean(dim=0) - img = img / 2 + 0.5 # unnormalize + img = img / 2 + 0.5 # unnormalize npimg = img.numpy() if one_channel: plt.imshow(npimg, cmap="Greys") else: plt.imshow(np.transpose(npimg, (1, 2, 0))) + # Extract a batch of 4 images dataiter = iter(training_loader) images, labels = next(dataiter) @@ -138,14 +139,14 @@ def matplotlib_imshow(img, one_channel=False): # minibatch of our input data. Below, we use the ``add_image()`` call on # ``SummaryWriter`` to log the image for consumption by TensorBoard, and # we also call ``flush()`` to make sure it’s written to disk right away. -# +# # Default log_dir argument is "runs" - but it's good to be specific # torch.utils.tensorboard.SummaryWriter is imported above -writer = SummaryWriter('runs/fashion_mnist_experiment_1') +writer = SummaryWriter("runs/fashion_mnist_experiment_1") # Write image data to TensorBoard log dir -writer.add_image('Four Fashion-MNIST Images', img_grid) +writer.add_image("Four Fashion-MNIST Images", img_grid) writer.flush() # To view, start TensorBoard on the command line with: @@ -157,17 +158,18 @@ def matplotlib_imshow(img, one_channel=False): # If you start TensorBoard at the command line and open it in a new # browser tab (usually at `localhost:6006 `__), you should # see the image grid under the IMAGES tab. -# +# # Graphing Scalars to Visualize Training # -------------------------------------- -# +# # TensorBoard is useful for tracking the progress and efficacy of your # training. Below, we’ll run a training loop, track some metrics, and save # the data for TensorBoard’s consumption. -# +# # Let’s define a model to categorize our image tiles, and an optimizer and # loss function for training: -# +# + class Net(nn.Module): def __init__(self): @@ -187,7 +189,7 @@ def forward(self, x): x = F.relu(self.fc2(x)) x = self.fc3(x) return x - + net = Net() criterion = nn.CrossEntropyLoss() @@ -197,7 +199,7 @@ def forward(self, x): ########################################################################## # Now let’s train a single epoch, and evaluate the training vs. validation # set losses every 1000 batches: -# +# print(len(validation_loader)) for epoch in range(1): # loop over the dataset multiple times @@ -213,44 +215,50 @@ def forward(self, x): optimizer.step() running_loss += loss.item() - if i % 1000 == 999: # Every 1000 mini-batches... - print('Batch {}'.format(i + 1)) + if i % 1000 == 999: # Every 1000 mini-batches... + print("Batch {}".format(i + 1)) # Check against the validation set running_vloss = 0.0 - + # In evaluation mode some model specific operations can be omitted eg. dropout layer - net.train(False) # Switching to evaluation mode, eg. turning off regularisation + net.train( + False + ) # Switching to evaluation mode, eg. turning off regularisation for j, vdata in enumerate(validation_loader, 0): vinputs, vlabels = vdata voutputs = net(vinputs) vloss = criterion(voutputs, vlabels) running_vloss += vloss.item() - net.train(True) # Switching back to training mode, eg. turning on regularisation - + net.train( + True + ) # Switching back to training mode, eg. turning on regularisation + avg_loss = running_loss / 1000 avg_vloss = running_vloss / len(validation_loader) - + # Log the running loss averaged per batch - writer.add_scalars('Training vs. Validation Loss', - { 'Training' : avg_loss, 'Validation' : avg_vloss }, - epoch * len(training_loader) + i) + writer.add_scalars( + "Training vs. Validation Loss", + {"Training": avg_loss, "Validation": avg_vloss}, + epoch * len(training_loader) + i, + ) running_loss = 0.0 -print('Finished Training') +print("Finished Training") writer.flush() ######################################################################### # Switch to your open TensorBoard and have a look at the SCALARS tab. -# +# # Visualizing Your Model # ---------------------- -# +# # TensorBoard can also be used to examine the data flow within your model. # To do this, call the ``add_graph()`` method with a model and sample # input: -# +# # Again, grab a single mini-batch of images dataiter = iter(training_loader) @@ -266,10 +274,10 @@ def forward(self, x): # When you switch over to TensorBoard, you should see a GRAPHS tab. # Double-click the “NET” node to see the layers and data flow within your # model. -# +# # Visualizing Your Dataset with Embeddings # ---------------------------------------- -# +# # The 28-by-28 image tiles we’re using can be modeled as 784-dimensional # vectors (28 \* 28 = 784). It can be instructive to project this to a # lower-dimensional representation. The ``add_embedding()`` method will @@ -277,9 +285,10 @@ def forward(self, x): # and display them as an interactive 3D chart. The ``add_embedding()`` # method does this automatically by projecting to the three dimensions # with highest variance. -# +# # Below, we’ll take a sample of our data, and generate such an embedding: -# +# + # Select a random subset of data and corresponding labels def select_n_random(data, labels, n=100): @@ -288,6 +297,7 @@ def select_n_random(data, labels, n=100): perm = torch.randperm(len(data)) return data[perm][:n], labels[perm][:n] + # Extract a random subset of data images, labels = select_n_random(training_set.data, training_set.targets) @@ -296,9 +306,7 @@ def select_n_random(data, labels, n=100): # log embeddings features = images.view(-1, 28 * 28) -writer.add_embedding(features, - metadata=class_labels, - label_img=images.unsqueeze(1)) +writer.add_embedding(features, metadata=class_labels, label_img=images.unsqueeze(1)) writer.flush() writer.close() @@ -309,19 +317,19 @@ def select_n_random(data, labels, n=100): # zoom the model. Examine it at large and small scales, and see whether # you can spot patterns in the projected data and the clustering of # labels. -# +# # For better visibility, it’s recommended to: -# +# # - Select “label” from the “Color by” drop-down on the left. # - Toggle the Night Mode icon along the top to place the # light-colored images on a dark background. -# +# # Other Resources # --------------- -# +# # For more information, have a look at: -# +# # - PyTorch documentation on `torch.utils.tensorboard.SummaryWriter `__ -# - Tensorboard tutorial content in the `PyTorch.org Tutorials `__ +# - Tensorboard tutorial content in the `PyTorch.org Tutorials `__ # - For more information about TensorBoard, see the `TensorBoard # documentation `__ diff --git a/intermediate_source/dist_tuto.rst b/intermediate_source/dist_tuto.rst index 1b622aa277..080bffc57f 100644 --- a/intermediate_source/dist_tuto.rst +++ b/intermediate_source/dist_tuto.rst @@ -523,6 +523,7 @@ for an available MPI implementation. The following steps install the MPI backend, by installing PyTorch `from source `__. +.. needs an update 1. Create and activate your Anaconda environment, install all the pre-requisites following `the guide `__, but do diff --git a/recipes_source/intel_neural_compressor_for_pytorch.rst b/recipes_source/intel_neural_compressor_for_pytorch.rst index 02ce3d7b37..3c108afd9f 100755 --- a/recipes_source/intel_neural_compressor_for_pytorch.rst +++ b/recipes_source/intel_neural_compressor_for_pytorch.rst @@ -50,9 +50,6 @@ Installation # install nightly version from pip pip install -i https://test.pypi.org/simple/ neural-compressor - # install stable version from from conda - conda install neural-compressor -c conda-forge -c intel - *Supported python versions are 3.6 or 3.7 or 3.8 or 3.9* Usages diff --git a/recipes_source/recipes/tensorboard_with_pytorch.py b/recipes_source/recipes/tensorboard_with_pytorch.py index 4bceda81ea..d32450ea32 100644 --- a/recipes_source/recipes/tensorboard_with_pytorch.py +++ b/recipes_source/recipes/tensorboard_with_pytorch.py @@ -1,24 +1,17 @@ """ How to use TensorBoard with PyTorch =================================== -TensorBoard is a visualization toolkit for machine learning experimentation. -TensorBoard allows tracking and visualizing metrics such as loss and accuracy, -visualizing the model graph, viewing histograms, displaying images and much more. -In this tutorial we are going to cover TensorBoard installation, +TensorBoard is a visualization toolkit for machine learning experimentation. +TensorBoard allows tracking and visualizing metrics such as loss and accuracy, +visualizing the model graph, viewing histograms, displaying images and much more. +In this tutorial we are going to cover TensorBoard installation, basic usage with PyTorch, and how to visualize data you logged in TensorBoard UI. Installation ---------------------- -PyTorch should be installed to log models and metrics into TensorBoard log -directory. The following command will install PyTorch 1.4+ via -Anaconda (recommended): - -.. code-block:: sh - - $ conda install pytorch torchvision -c pytorch - - -or pip +PyTorch should be installed to log models and metrics into TensorBoard log +directory. The following command will install PyTorch 1.4+ via +pip: .. code-block:: sh @@ -29,31 +22,32 @@ ###################################################################### # Using TensorBoard in PyTorch # ----------------------------- -# -# Let’s now try using TensorBoard with PyTorch! Before logging anything, +# +# Let’s now try using TensorBoard with PyTorch! Before logging anything, # we need to create a ``SummaryWriter`` instance. -# +# import torch from torch.utils.tensorboard import SummaryWriter + writer = SummaryWriter() ###################################################################### # Writer will output to ``./runs/`` directory by default. -# +# ###################################################################### # Log scalars # ----------- -# -# In machine learning, it’s important to understand key metrics such as -# loss and how they change during training. Scalar helps to save -# the loss value of each training step, or the accuracy after each epoch. -# -# To log a scalar value, use -# ``add_scalar(tag, scalar_value, global_step=None, walltime=None)``. -# For example, lets create a simple linear regression training, and +# +# In machine learning, it’s important to understand key metrics such as +# loss and how they change during training. Scalar helps to save +# the loss value of each training step, or the accuracy after each epoch. +# +# To log a scalar value, use +# ``add_scalar(tag, scalar_value, global_step=None, walltime=None)``. +# For example, lets create a simple linear regression training, and # log loss value using ``add_scalar`` # @@ -62,7 +56,8 @@ model = torch.nn.Linear(1, 1) criterion = torch.nn.MSELoss() -optimizer = torch.optim.SGD(model.parameters(), lr = 0.1) +optimizer = torch.optim.SGD(model.parameters(), lr=0.1) + def train_model(iter): for epoch in range(iter): @@ -72,18 +67,19 @@ def train_model(iter): optimizer.zero_grad() loss.backward() optimizer.step() - + + train_model(10) writer.flush() -###################################################################### -# Call ``flush()`` method to make sure that all pending events +###################################################################### +# Call ``flush()`` method to make sure that all pending events # have been written to disk. -# -# See `torch.utils.tensorboard tutorials `_ +# +# See `torch.utils.tensorboard tutorials `_ # to find more TensorBoard visualization types you can log. -# +# # If you do not need the summary writer anymore, call ``close()`` method. # @@ -92,7 +88,7 @@ def train_model(iter): ###################################################################### # Run TensorBoard # ---------------- -# +# # Install TensorBoard through the command line to visualize data you logged # # .. code-block:: sh @@ -100,9 +96,9 @@ def train_model(iter): # pip install tensorboard # # -# Now, start TensorBoard, specifying the root log directory you used above. -# Argument ``logdir`` points to directory where TensorBoard will look to find -# event files that it can display. TensorBoard will recursively walk +# Now, start TensorBoard, specifying the root log directory you used above. +# Argument ``logdir`` points to directory where TensorBoard will look to find +# event files that it can display. TensorBoard will recursively walk # the directory structure rooted at ``logdir``, looking for ``.*tfevents.*`` files. # # .. code-block:: sh @@ -114,9 +110,9 @@ def train_model(iter): # .. image:: ../../_static/img/thumbnails/tensorboard_scalars.png # :scale: 40 % # -# This dashboard shows how the loss and accuracy change with every epoch. -# You can use it to also track training speed, learning rate, and other -# scalar values. It’s helpful to compare these metrics across different +# This dashboard shows how the loss and accuracy change with every epoch. +# You can use it to also track training speed, learning rate, and other +# scalar values. It’s helpful to compare these metrics across different # training runs to improve your model. # @@ -124,7 +120,7 @@ def train_model(iter): ######################################################################## # Learn More # ---------------------------- -# +# # - `torch.utils.tensorboard `_ docs # - `Visualizing models, data, and training with TensorBoard `_ tutorial #