feat(logging): add positive/negative cosine-similarity and embedding … #52

akberovic · 2025-04-17T15:20:04Z

…stddev metrics

import torch.nn.functional as F # For cosine similarity

   # Compute and log cosine similarity between positive and negative pair representations.

```
   with torch.no_grad():
```
```
       # Positive cosine similarities.
```

       cos_sim = F.cosine_similarity(z_i, z_j, dim=-1)

       avg_cos_sim = cos_sim.mean().item()

       writer.add_scalar("Cosine Similarity/avg_positive", avg_cos_sim, args.global_step)

       writer.add_histogram("Cosine Similarity/hist_positive", cos_sim, args.global_step)

       # Compute negative similarities.

       norm_z_i = F.normalize(z_i, dim=1)

       norm_z_j = F.normalize(z_j, dim=1)

       cosine_matrix = torch.mm(norm_z_i, norm_z_j.t())

       mask = torch.eye(cosine_matrix.size(0), dtype=torch.bool, device=cosine_matrix.device)

       negative_sim = cosine_matrix[~mask].view(cosine_matrix.size(0), -1)

       avg_cos_sim_neg = negative_sim.mean().item()

       writer.add_scalar("Cosine Similarity/avg_negative", avg_cos_sim_neg, args.global_step)

       writer.add_histogram("Cosine Similarity/hist_negative", negative_sim, args.global_step)

       # Log embedding standard deviation as a collapse indicator.

       std_z_i = norm_z_i.std(dim=0).mean().item()

       writer.add_scalar("Embeddings/std_dev", std_z_i, args.global_step)

…stddev metrics + import torch.nn.functional as F # For cosine similarity + # Compute and log cosine similarity between positive and negative pair representations. + with torch.no_grad(): + # Positive cosine similarities. + cos_sim = F.cosine_similarity(z_i, z_j, dim=-1) + avg_cos_sim = cos_sim.mean().item() + writer.add_scalar("Cosine Similarity/avg_positive", avg_cos_sim, args.global_step) + writer.add_histogram("Cosine Similarity/hist_positive", cos_sim, args.global_step) + + # Compute negative similarities. + norm_z_i = F.normalize(z_i, dim=1) + norm_z_j = F.normalize(z_j, dim=1) + cosine_matrix = torch.mm(norm_z_i, norm_z_j.t()) + mask = torch.eye(cosine_matrix.size(0), dtype=torch.bool, device=cosine_matrix.device) + negative_sim = cosine_matrix[~mask].view(cosine_matrix.size(0), -1) + avg_cos_sim_neg = negative_sim.mean().item() + writer.add_scalar("Cosine Similarity/avg_negative", avg_cos_sim_neg, args.global_step) + writer.add_histogram("Cosine Similarity/hist_negative", negative_sim, args.global_step) + + # Log embedding standard deviation as a collapse indicator. + std_z_i = norm_z_i.std(dim=0).mean().item() + writer.add_scalar("Embeddings/std_dev", std_z_i, args.global_step)

…dogs-vs-cats + # Compute and log cosine similarity between positive and negative pair representations. + with torch.no_grad(): + # Positive cosine similarities. + cos_sim = F.cosine_similarity(z_i, z_j, dim=-1) + avg_cos_sim = cos_sim.mean().item() + writer.add_scalar("Cosine Similarity/avg_positive", avg_cos_sim, args.global_step) + writer.add_histogram("Cosine Similarity/hist_positive", cos_sim, args.global_step) + + # Compute negative similarities. + norm_z_i = F.normalize(z_i, dim=1) + norm_z_j = F.normalize(z_j, dim=1) + cosine_matrix = torch.mm(norm_z_i, norm_z_j.t()) + mask = torch.eye(cosine_matrix.size(0), dtype=torch.bool, device=cosine_matrix.device) + negative_sim = cosine_matrix[~mask].view(cosine_matrix.size(0), -1) + avg_cos_sim_neg = negative_sim.mean().item() + writer.add_scalar("Cosine Similarity/avg_negative", avg_cos_sim_neg, args.global_step) + writer.add_histogram("Cosine Similarity/hist_negative", negative_sim, args.global_step) + + # Log embedding standard deviation as a collapse indicator. + std_z_i = norm_z_i.std(dim=0).mean().item() + writer.add_scalar("Embeddings/std_dev", std_z_i, args.global_step)

…epoch if args.nr == 0: + # Log embeddings for a small subset of images. + model.eval() + try: + sample_batch = next(iter(train_loader)) + sample_images, sample_labels = sample_batch[0][0], sample_batch[1] + sample_images = sample_images.cuda(non_blocking=True) + + with torch.no_grad(): + embeddings = model.encoder(sample_images) + cifar10_classes = ['airplane','automobile','bird','cat','deer','dog','frog','horse','ship','truck'] + metadata = [cifar10_classes[label] for label in sample_labels] + writer.add_embedding(embeddings, metadata=metadata, + label_img=sample_images, global_step=epoch, tag="embeddings") + except Exception as e: + print(f"Error during embedding logging: {e}") + model.train() + + # Log gradient histograms for all parameters. + for name, param in model.named_parameters(): + if param.grad is not None: + writer.add_histogram(f'{name}.grad', param.grad, epoch) + # End training: save final model checkpoint. save_model(args, model, optimizer) + + if writer: + writer.close()

akberovic added 3 commits April 17, 2025 11:18

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat(logging): add positive/negative cosine-similarity and embedding … #52

feat(logging): add positive/negative cosine-similarity and embedding … #52

akberovic commented Apr 17, 2025

feat(logging): add positive/negative cosine-similarity and embedding … #52

Are you sure you want to change the base?

feat(logging): add positive/negative cosine-similarity and embedding … #52

Conversation

akberovic commented Apr 17, 2025