rearrange logits before calculating kl

Author: lucidrains

dalle_pytorch/dalle_pytorch.py

@@ -166,10 +166,11 @@ def forward(
 
         # kl divergence
 
+        logits = rearrange(logits, 'b n h w -> b (h w) n')
         qy = F.softmax(logits, dim = -1)
         log_qy = torch.log(qy + 1e-20)
         g = torch.log(torch.Tensor([1. / num_tokens]))
-        kl_div = (qy * (log_qy - g)).sum(dim = -1).mean()
+        kl_div = (qy * (log_qy - g)).sum(dim = (1, 2)).mean()
 
         return recon_loss + (kl_div * kl_div_loss_weight)
 

setup.py

@@ -3,7 +3,7 @@
 setup(
   name = 'dalle-pytorch',
   packages = find_packages(),
-  version = '0.1.2',
+  version = '0.1.4',
   license='MIT',
   description = 'DALL-E - Pytorch',
   author = 'Phil Wang',