|
| 1 | +import sys |
| 2 | +import numpy as np |
| 3 | +import unittest |
| 4 | + |
| 5 | +import torch |
| 6 | +import torch_xla |
| 7 | +from torch_xla import runtime as xr |
| 8 | +import torch_xla.core.xla_model as xm |
| 9 | +from torch_xla.distributed.spmd import Mesh |
| 10 | +import torch_xla.distributed.spmd as xs |
| 11 | +import torch_xla.distributed.parallel_loader as pl |
| 12 | + |
| 13 | +xr.use_spmd() |
| 14 | + |
| 15 | + |
| 16 | +class MpInputShardingTest(unittest.TestCase): |
| 17 | + |
| 18 | + class fake_dataloader: |
| 19 | + |
| 20 | + def __init__(self, batch, size=1): |
| 21 | + self.batch = batch |
| 22 | + self.batch_size = size |
| 23 | + self.counter = 0 |
| 24 | + |
| 25 | + def __iter__(self): |
| 26 | + return self |
| 27 | + |
| 28 | + def __next__(self): |
| 29 | + if self.counter < self.batch_size: |
| 30 | + self.counter += 1 |
| 31 | + return self.batch |
| 32 | + raise StopIteration |
| 33 | + |
| 34 | + @unittest.skipUnless(xr.global_runtime_device_count() > 1, |
| 35 | + "Multiple devices required for tupled partition spec") |
| 36 | + def test_multiple_inputs(self): |
| 37 | + device = xm.xla_device() |
| 38 | + batch = {'x': torch.randn((16, 128)), 'y': torch.randn((16, 128, 128))} |
| 39 | + train_loader = self.fake_dataloader(batch) |
| 40 | + num_devices = xr.global_runtime_device_count() |
| 41 | + mesh = xs.get_1d_mesh('x') |
| 42 | + |
| 43 | + train_loader = pl.MpDeviceLoader( |
| 44 | + train_loader, |
| 45 | + device, |
| 46 | + input_sharding={ |
| 47 | + 'x': xs.ShardingSpec(mesh, ('x', None)), |
| 48 | + 'y': xs.ShardingSpec(mesh, ('x', None, None)) |
| 49 | + }) |
| 50 | + train_loader = iter(train_loader) |
| 51 | + data = next(train_loader) |
| 52 | + annotation_x = '{devices=[%d,1]%s}' % (num_devices, ','.join( |
| 53 | + [str(i) for i in range(num_devices)])) |
| 54 | + annotation_y = '{devices=[%d,1,1]%s}' % (num_devices, ','.join( |
| 55 | + [str(i) for i in range(num_devices)])) |
| 56 | + self.assertEqual(annotation_x, |
| 57 | + torch_xla._XLAC._get_xla_sharding_spec(data['x'])) |
| 58 | + self.assertEqual(annotation_y, |
| 59 | + torch_xla._XLAC._get_xla_sharding_spec(data['y'])) |
| 60 | + |
| 61 | + @unittest.skipUnless(xr.global_runtime_device_count() > 1, |
| 62 | + "Multiple devices required for tupled partition spec") |
| 63 | + def test_single_tensor(self): |
| 64 | + device = xm.xla_device() |
| 65 | + batch = torch.randn((16, 128)) |
| 66 | + train_loader = self.fake_dataloader(batch) |
| 67 | + num_devices = xr.global_runtime_device_count() |
| 68 | + mesh = xs.get_1d_mesh('x') |
| 69 | + |
| 70 | + train_loader = pl.MpDeviceLoader( |
| 71 | + train_loader, device, input_sharding=xs.ShardingSpec(mesh, ('x', None))) |
| 72 | + train_loader = iter(train_loader) |
| 73 | + data = next(train_loader) |
| 74 | + annotation = '{devices=[%d,1]%s}' % (num_devices, ','.join( |
| 75 | + [str(i) for i in range(num_devices)])) |
| 76 | + self.assertEqual(annotation, torch_xla._XLAC._get_xla_sharding_spec(data)) |
| 77 | + |
| 78 | + @unittest.skipUnless(xr.global_runtime_device_count() > 1, |
| 79 | + "Multiple devices required for tupled partition spec") |
| 80 | + def test_error_single_tensor_with_input_sharding_dict(self): |
| 81 | + device = xm.xla_device() |
| 82 | + batch = torch.randn((16, 128)) |
| 83 | + train_loader = self.fake_dataloader(batch) |
| 84 | + num_devices = xr.global_runtime_device_count() |
| 85 | + mesh = xs.get_1d_mesh('x') |
| 86 | + |
| 87 | + train_loader = pl.MpDeviceLoader( |
| 88 | + train_loader, device, input_sharding={'x': xs.ShardingSpec(mesh, ('x', None))}) |
| 89 | + train_loader = iter(train_loader) |
| 90 | + with self.assertRaises(ValueError): |
| 91 | + data = next(train_loader) |
| 92 | + |
| 93 | + @unittest.skipUnless(xr.global_runtime_device_count() > 1, |
| 94 | + "Multiple devices required for tupled partition spec") |
| 95 | + def test_input_sharding_none(self): |
| 96 | + device = xm.xla_device() |
| 97 | + batch = {'x': torch.randn((16, 128)), 'y': torch.randn((16, 128, 128))} |
| 98 | + train_loader = self.fake_dataloader(batch) |
| 99 | + num_devices = xr.global_runtime_device_count() |
| 100 | + |
| 101 | + train_loader = pl.MpDeviceLoader(train_loader, device, input_sharding=None) |
| 102 | + train_loader = iter(train_loader) |
| 103 | + data = next(train_loader) |
| 104 | + annotation = '{replicated}' |
| 105 | + self.assertEqual(annotation, |
| 106 | + torch_xla._XLAC._get_xla_sharding_spec(data['x'])) |
| 107 | + self.assertEqual(annotation, |
| 108 | + torch_xla._XLAC._get_xla_sharding_spec(data['y'])) |
| 109 | + |
| 110 | + @unittest.skipUnless(xr.global_runtime_device_count() > 1, |
| 111 | + "Multiple devices required for tupled partition spec") |
| 112 | + def test_error_missing_keys(self): |
| 113 | + device = xm.xla_device() |
| 114 | + batch = {'x': torch.randn((16, 128)), 'y': torch.randn((16, 128, 128))} |
| 115 | + train_loader = self.fake_dataloader(batch) |
| 116 | + mesh = xs.get_1d_mesh('x') |
| 117 | + train_loader = pl.MpDeviceLoader( |
| 118 | + train_loader, |
| 119 | + device, |
| 120 | + input_sharding={'x': xs.ShardingSpec(mesh, ('x', None))}) |
| 121 | + train_loader = iter(train_loader) |
| 122 | + with self.assertRaises(KeyError): |
| 123 | + data = next(train_loader) |
| 124 | + |
| 125 | + @unittest.skipUnless(xr.global_runtime_device_count() > 1, |
| 126 | + "Multiple devices required for tupled partition spec") |
| 127 | + def test_input_sharding_not_dict(self): |
| 128 | + device = xm.xla_device() |
| 129 | + num_devices = xr.global_runtime_device_count() |
| 130 | + batch = {'x': torch.randn((16, 128)), 'y': torch.randn((16, 128))} |
| 131 | + train_loader = self.fake_dataloader(batch) |
| 132 | + mesh = xs.get_1d_mesh('x') |
| 133 | + train_loader = pl.MpDeviceLoader( |
| 134 | + train_loader, device, input_sharding=xs.ShardingSpec(mesh, ('x', None))) |
| 135 | + train_loader = iter(train_loader) |
| 136 | + data = next(train_loader) |
| 137 | + annotation_x = '{devices=[%d,1]%s}' % (num_devices, ','.join( |
| 138 | + [str(i) for i in range(num_devices)])) |
| 139 | + annotation_y = '{devices=[%d,1]%s}' % (num_devices, ','.join( |
| 140 | + [str(i) for i in range(num_devices)])) |
| 141 | + self.assertEqual(annotation_x, |
| 142 | + torch_xla._XLAC._get_xla_sharding_spec(data['x'])) |
| 143 | + self.assertEqual(annotation_y, |
| 144 | + torch_xla._XLAC._get_xla_sharding_spec(data['y'])) |
| 145 | + |
| 146 | + |
| 147 | +if __name__ == '__main__': |
| 148 | + test = unittest.main() |
| 149 | + sys.exit(0 if test.result.wasSuccessful() else 1) |
0 commit comments