vSphere vSAN Volume creation time out : "VolumeDriver.Create: context deadline exceeded"

[ShaharukShaikh]

1. We have over 300 volumes created
2. When we try to create volume of smaller size like 10gb (thin provisioned) the creation operation goes through successfully.
   root@ubuntu1604:~/repo/DCEServices# docker volume create --driver=vsphere --name=vol1 -o size=10gb -o diskformat=thin
   vol1
3. But when we try to create volume of size 1TB (thin provisined), the creation times out
   root@ubuntu1604:/repo/DCEServices# docker volume create --driver=vsphere --name=vol2 -o size=1000gb -o diskformat=thin
   Error response from daemon: create vol2: Post http://%2Frun%2Fdocker%2Fplugins%2F8034a87ce4b03163e12c79fda97f3cdfb1ea10be9650132aab364dff735c6730%2Fvsphere.sock/VolumeDriver.Create: context deadline exceeded
   root@ubuntu1604:/repo/DCEServices# docker plugin ls
   ID NAME DESCRIPTION ENABLED
   8034a87ce4b0 vsphere:latest VMWare vSphere Docker Volume plugin true

root@ubuntu1604:~/repo/DCEServices# docker plugin inspect 8034a87ce4b0
[
{
"Config": {
"Args": {
"Description": "",
"Name": "",
"Settable": null,
"Value": null
},
"Description": "VMWare vSphere Docker Volume plugin",
"DockerVersion": "17.12.0-ce",
"Documentation": "http://vmware.github.io/vsphere-storage-for-docker/documentation",
"Entrypoint": [
"/usr/bin/vsphere-storage-for-docker",
"--config",
"/etc/vsphere-storage-for-docker.conf"
],
"Env": [
{
"Description": "Log level - info, warn, error or debug",
"Name": "VDVS_LOG_LEVEL",
"Settable": [
"value"
],
"Value": "info"
},
{
"Description": "Release version of VDVS plugin",
"Name": "VDVS_VERSION",
"Settable": [
"value"
],
"Value": "latest"
},
{
"Description": "Client protocol version used in test",
"Name": "VDVS_TEST_PROTOCOL_VERSION",
"Settable": [
"value"
],
"Value": ""
},
{
"Description": "Timeout value in second used by vFILE plugin",
"Name": "VFILE_TIMEOUT_IN_SECOND",
"Settable": [
"value"
],
"Value": ""
},
{
"Description": "Group ID of the socket file for the plugin",
"Name": "VDVS_SOCKET_GID",
"Settable": [
"value"
],
"Value": "root"
},
{
"Description": "Etcd client port number used by vFILE plugin",
"Name": "VFILE_ETCD_CLIENT_PORT",
"Settable": [
"value"
],
"Value": ""
},
{
"Description": "Etcd peer port number used by vFILE plugin",
"Name": "VFILE_ETCD_PEER_PORT",
"Settable": [
"value"
],
"Value": ""
}
],
"Interface": {
"Socket": "vsphere.sock",
"Types": [
"docker.volumedriver/1.0"
]
},
"IpcHost": false,
"Linux": {
"AllowAllDevices": true,
"Capabilities": [
"CAP_SYS_ADMIN"
],
"Devices": null
},
"Mounts": [
{
"Description": "The plugin uses dev to mount volumes and watch for attaches",
"Destination": "/dev",
"Name": "",
"Options": [
"rbind",
"shared"
],
"Settable": null,
"Source": "/dev",
"Type": "bind"
},
{
"Description": "The plugin uses /var/run/docker.sock to ask Docker some questions",
"Destination": "/var/run",
"Name": "",
"Options": [
"rbind"
],
"Settable": null,
"Source": "/var/run",
"Type": "bind"
},
{
"Description": "Location to look for config file (/etc/vsphere-storage-for-docker.conf)",
"Destination": "/etc",
"Name": "",
"Options": [
"rbind"
],
"Settable": null,
"Source": "/etc",
"Type": "bind"
},
{
"Description": "Expose plugin logs in /var/log/docker-volume-vshpere.log",
"Destination": "/var/log",
"Name": "",
"Options": [
"rbind"
],
"Settable": null,
"Source": "/var/log",
"Type": "bind"
}
],
"Network": {
"Type": ""
},
"PidHost": false,
"PropagatedMount": "/mnt/vmdk",
"User": {},
"WorkDir": "",
"rootfs": {
"diff_ids": [
"sha256:dd5a292474e64426536c1671f8bb4c23941f8873f01cd7d7933f40db00869a55"
],
"type": "layers"
}
},
"Enabled": true,
"Id": "8034a87ce4b03163e12c79fda97f3cdfb1ea10be9650132aab364dff735c6730",
"Name": "vsphere:latest",
"PluginReference": "docker.io/vmware/vsphere-storage-for-docker:latest",
"Settings": {
"Args": [],
"Devices": [],
"Env": [
"VDVS_LOG_LEVEL=info",
"VDVS_VERSION=latest",
"VDVS_TEST_PROTOCOL_VERSION=",
"VFILE_TIMEOUT_IN_SECOND=",
"VDVS_SOCKET_GID=root",
"VFILE_ETCD_CLIENT_PORT=",
"VFILE_ETCD_PEER_PORT="
],
"Mounts": [
{
"Description": "The plugin uses dev to mount volumes and watch for attaches",
"Destination": "/dev",
"Name": "",
"Options": [
"rbind",
"shared"
],
"Settable": null,
"Source": "/dev",
"Type": "bind"
},
{
"Description": "The plugin uses /var/run/docker.sock to ask Docker some questions",
"Destination": "/var/run",
"Name": "",
"Options": [
"rbind"
],
"Settable": null,
"Source": "/var/run",
"Type": "bind"
},
{
"Description": "Location to look for config file (/etc/vsphere-storage-for-docker.conf)",
"Destination": "/etc",
"Name": "",
"Options": [
"rbind"
],
"Settable": null,
"Source": "/etc",
"Type": "bind"
},
{
"Description": "Expose plugin logs in /var/log/docker-volume-vshpere.log",
"Destination": "/var/log",
"Name": "",
"Options": [
"rbind"
],
"Settable": null,
"Source": "/var/log",
"Type": "bind"
}
]
}
}
]

root@ubuntu1604:~/repo/DCEServices# docker info
Containers: 0
Running: 0
Paused: 0
Stopped: 0
Images: 34
Server Version: 18.09.3
Storage Driver: overlay2
Backing Filesystem: extfs
Supports d_type: true
Native Overlay Diff: true
Logging Driver: json-file
Cgroup Driver: cgroupfs
Plugins:
Volume: local
Network: bridge host macvlan null overlay
Log: awslogs fluentd gcplogs gelf journald json-file local logentries splunk syslog
Swarm: active
NodeID: rwle6pieva9wlki1knlyifre3
Is Manager: true
ClusterID: u256kfiab8zcvpjx7c1mdcalr
Managers: 2
Nodes: 2
Default Address Pool: 10.0.0.0/8
SubnetSize: 24
Orchestration:
Task History Retention Limit: 5
Raft:
Snapshot Interval: 10000
Number of Old Snapshots to Retain: 0
Heartbeat Tick: 1
Election Tick: 10
Dispatcher:
Heartbeat Period: 5 seconds
CA Configuration:
Expiry Duration: 3 months
Force Rotate: 0
Autolock Managers: false
Root Rotation In Progress: false
Node Address: 172.21.75.69
Manager Addresses:
172.21.75.68:2377
172.21.75.69:2377
Runtimes: runc
Default Runtime: runc
Init Binary: docker-init
containerd version: 9754871865f7fe2f4e74d43e2fc7ccd237edcbce
runc version: 09c8266bf2fcf9519a651b04ae54c967b9ab86ec
init version: fec3683
Security Options:
apparmor
seccomp
Profile: default
Kernel Version: 4.4.0-131-generic
Operating System: Ubuntu 16.04.5 LTS
OSType: linux
Architecture: x86_64
CPUs: 16
Total Memory: 70.78GiB
Name: ubuntu1604
ID: TOUY:GER6:UQNU:WYPP:6HZC:3NWY:MKFX:KA75:ZGIU:FZJH:TA32:OYBC
Docker Root Dir: /var/lib/docker
Debug Mode (client): false
Debug Mode (server): false
Username: nitishmowall
Registry: https://index.docker.io/v1/
Labels:
Experimental: false
Insecure Registries:
127.0.0.0/8
Live Restore Enabled: false
Product License: Community Engine