draid: add failure domains support

Currently, the only way to tolerate the failure of the whole
enclosure is to configure several draid vdevs in the pool, each
vdev having disks from different enclosures. But this essentially
degrades draid to raidz and defeats the purpose having fast
sequential resilvering on wide pools with draid.

This patch allows to configure several children groups in the same
row in one draid vdev. In each such group, let's call it failure
group, the user can configure disks belonging to different
enclosures - failure domains. For example, in case of 10
enclosures with 10 disks each, the user can put 1st disk from each
enclosure into 1st group, 2nd disk from each enclosure into 2nd
group, and so on. If one enclosure fails, only one disk from each
group would fail, which won't affect draid operation, and each
group would have enough redundancy to recover the stored data. Of
course, in case of draid2 - two enclosures can fail at a time, in
case of draid3 - three enclosures (provided there are no other
disk failures in each group).

In order to preserve fast sequential resilvering in case of a disk
failure, the groups much share all disks between themselves, and
this is achieved by shuffling the disks between the groups. But
only i-th disks in each group are shuffled between themselves,
i.e. the disks from the same enclosures, after that they are
shuffled within each group, like it is done today in an ordinary
draid. Thus, no more than one disk from any enclosure can appear
in any failure group as a result of this shuffling.

For example, here's how the pool status output looks like in
case of two `draid1:2d:4c:1s` groups:

    NAME                        STATE     READ WRITE CKSUM
    pool1                       ONLINE       0     0     0
      draid1:2d:4c:1s:8w-0      ONLINE       0     0     0
        enc0d0                  ONLINE       0     0     0
        enc1d0                  ONLINE       0     0     0
        enc2d0                  ONLINE       0     0     0
        enc3d0                  ONLINE       0     0     0
        enc0d1                  ONLINE       0     0     0
        enc1d1                  ONLINE       0     0     0
        enc2d1                  ONLINE       0     0     0
        enc3d1                  ONLINE       0     0     0
    spares
      draid1-0-0                AVAIL
      draid1-0-1                AVAIL

The number of failure groups is specified indirectly via the new
width parameter in draid vdev configuration descriptor, which is
the total number of disks and which is multiple of children in
each group. This multiple is the number of groups (width /
children). Doing it this way allows the user conveniently see how
many disks draid has in an instant.

Spare disks are evenly distributed among failure groups, so the
number of spares should be multiple of the number of groups, and
they are shared by all groups. However, to support domain failure,
we cannot have more than nparity - 1 failed disks in any group, no
matter if they are rebuilt to draid spares or not (the blocks of
those spares can be mapped to the disks from the failed domain
(enclosure), and we cannot tolerate more than nparity failures in
any failure group).

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Alexander Motin <alexander.motin@TrueNAS.com>
Signed-off-by: Andriy Tkachuk <andriy.tkachuk@seagate.com>

Closes #11969.

Author: andriytk

cmd/zpool/zpool_main.c

@@ -3528,6 +3528,11 @@ show_import(nvlist_t *config, boolean_t report_error)
 		    "accessed by another system.\n"));
 		break;
 
+	case ZPOOL_STATUS_FAULTED_FDOM_R:
+		(void) printf_color(ANSI_YELLOW, gettext("One or more failure "
+		    " domains are faulted.\n"));
+		break;
+
 	case ZPOOL_STATUS_FAULTED_DEV_R:
 	case ZPOOL_STATUS_FAULTED_DEV_NR:
 		(void) printf_color(ANSI_YELLOW, gettext("One or more devices "
@@ -8030,7 +8035,7 @@ zpool_do_online(int argc, char **argv)
 
 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL) {
 		(void) fprintf(stderr, gettext("failed to open pool "
-		    "\"%s\""), poolname);
+		    "\"%s\"\n"), poolname);
 		return (1);
 	}
 
@@ -8174,7 +8179,7 @@ zpool_do_offline(int argc, char **argv)
 
 	if ((zhp = zpool_open(g_zfs, poolname)) == NULL) {
 		(void) fprintf(stderr, gettext("failed to open pool "
-		    "\"%s\""), poolname);
+		    "\"%s\"\n"), poolname);
 		return (1);
 	}
 
@@ -10715,6 +10720,18 @@ print_status_reason(zpool_handle_t *zhp, status_cbdata_t *cbp,
 		    "or use 'zpool clear' to mark the device\n\trepaired.\n"));
 		break;
 
+	case ZPOOL_STATUS_FAULTED_FDOM_R:
+		(void) snprintf(status, ST_SIZE,
+		    gettext("One or more failure domains are faulted. "
+		    "The storage devices may be\n\tintact. Sufficient "
+		    "replicas exist for the pool to continue functioning\n\t"
+		    "in a degraded state.\n"));
+		(void) snprintf(action, AC_SIZE,
+		    gettext("Replace the faulted domain device, "
+		    "or use 'zpool clear' to mark domain\n\tstorage devices "
+		    "repaired.\n"));
+		break;
+
 	case ZPOOL_STATUS_FAULTED_DEV_NR:
 		(void) snprintf(status, ST_SIZE,
 		    gettext("One or more devices are "

cmd/zpool/zpool_vdev.c

@@ -1323,36 +1323,44 @@ is_grouping(const char *type, int *mindev, int *maxdev)
  * Extract the configuration parameters encoded in the dRAID type and
  * use them to generate a dRAID configuration.  The expected format is:
  *
- * draid[<parity>][:<data><d|D>][:<children><c|C>][:<spares><s|S>]
+ * draid[<parity>][:<data>d][:<children>c][:<spares>s][:<width>w]
  *
  * The intent is to be able to generate a good configuration when no
  * additional information is provided.  The only mandatory component
  * of the 'type' is the 'draid' prefix.  If a value is not provided
  * then reasonable defaults are used.  The optional components may
- * appear in any order but the d/s/c suffix is required.
+ * appear in any order but the d/s/c/w suffix is required.
  *
  * Valid inputs:
  * - data:     number of data devices per group (1-255)
- * - parity:   number of parity blocks per group (1-3)
- * - spares:   number of distributed spare (0-100)
- * - children: total number of devices (1-255)
+ * - parity:   number of parity devices per group (1-3)
+ * - children: total number of devices in slice (1-255)
+ * - width:    total number of devices, multiple of children (1-255 for now)
+ * - spares:   number of distributed spare devices (0-100), must be
+ *             multiple of failure groups (width / children)
  *
  * Examples:
  * - zpool create tank draid <devices...>
  * - zpool create tank draid2:8d:51c:2s <devices...>
+ * - zpool create tank draid2:8d:12c:96w:8s <devices...>
  */
 static int
-draid_config_by_type(nvlist_t *nv, const char *type, uint64_t children)
+draid_config_by_type(nvlist_t *nv, const char *type, uint64_t width,
+    int nfgroup, int nfdomain)
 {
 	uint64_t nparity;
 	uint64_t nspares = 0;
 	uint64_t ndata = UINT64_MAX;
 	uint64_t ngroups = 1;
+	uint64_t children = 0;
 	long value;
 
 	if (strncmp(type, VDEV_TYPE_DRAID, strlen(VDEV_TYPE_DRAID)) != 0)
 		return (EINVAL);
 
+	if (nfgroup && nfdomain) /* must be only one of two or none */
+		return (EINVAL);
+
 	nparity = (uint64_t)get_parity(type);
 	if (nparity == 0 || nparity > VDEV_DRAID_MAXPARITY) {
 		fprintf(stderr,
@@ -1376,24 +1384,35 @@ draid_config_by_type(nvlist_t *nv, const char *type, uint64_t children)
 			return (EINVAL);
 		}
 
-		/* Expected non-zero value with c/d/s suffix */
+		/* Expected non-zero value with c/d/s/w suffix */
 		value = strtol(p, &end, 10);
 		char suffix = tolower(*end);
 		if (errno != 0 ||
-		    (suffix != 'c' && suffix != 'd' && suffix != 's')) {
+		    (suffix != 'c' && suffix != 'd' && suffix != 's' &&
+		    suffix != 'w')) {
 			(void) fprintf(stderr, gettext("invalid dRAID "
-			    "syntax; expected [:<number><c|d|s>] not '%s'\n"),
-			    type);
+			    "syntax; expected [:<number><c|d|s|w>], "
+			    "not '%s'\n"), type);
 			return (EINVAL);
 		}
 
 		if (suffix == 'c') {
-			if ((uint64_t)value != children) {
+			if ((uint64_t)value > width ||
+			    width % (uint64_t)value != 0) {
 				fprintf(stderr,
-				    gettext("invalid number of dRAID children; "
+				    gettext("invalid number of dRAID disks; "
+				    "multiple of %llu required but %llu "
+				    "provided\n"), (u_longlong_t)value,
+				    (u_longlong_t)width);
+				return (EINVAL);
+			}
+			children = value;
+		} else if (suffix == 'w') {
+			if ((uint64_t)value != width) {
+				fprintf(stderr,
+				    gettext("invalid number of dRAID disks; "
 				    "%llu required but %llu provided\n"),
-				    (u_longlong_t)value,
-				    (u_longlong_t)children);
+				    (u_longlong_t)value, (u_longlong_t)width);
 				return (EINVAL);
 			}
 		} else if (suffix == 'd') {
@@ -1405,6 +1424,42 @@ draid_config_by_type(nvlist_t *nv, const char *type, uint64_t children)
 		}
 	}
 
+	if (!children && nfgroup)
+		children = width / nfgroup;
+	if (!children && nfdomain)
+		children = nfdomain;
+	if (!children)
+		children = width;
+
+	int fgrps = width / children;
+
+	if ((nspares % fgrps) != 0) {
+		fprintf(stderr, gettext("invalid number of distributed spares "
+		    "%llu, must be multiple of failure groups %d\n"),
+		    (u_longlong_t)nspares, fgrps);
+		return (EINVAL);
+	}
+
+	if (fgrps == 1 && (nfgroup || nfdomain)) {
+		fprintf(stderr, gettext("failure domains are not set "
+		    "in dRAID vdev descriptor\n"));
+		return (EINVAL);
+	}
+
+	if (fgrps > 1 && nfgroup && fgrps != nfgroup) {
+		fprintf(stderr, gettext("invalid number of failure groups "
+		    "%d, must be %d\n"), nfgroup, fgrps);
+		return (EINVAL);
+	}
+
+	if (fgrps > 1 && nfdomain && nfdomain != children) {
+		fprintf(stderr, gettext("invalid number of failure domains "
+		    "%d, must be %lu\n"), nfdomain, children);
+		return (EINVAL);
+	}
+
+	nspares /= fgrps;
+
 	/*
 	 * When a specific number of data disks is not provided limit a
 	 * redundancy group to 8 data disks.  This value was selected to
@@ -1414,8 +1469,8 @@ draid_config_by_type(nvlist_t *nv, const char *type, uint64_t children)
 		if (children > nspares + nparity) {
 			ndata = MIN(children - nspares - nparity, 8);
 		} else {
-			fprintf(stderr, gettext("request number of "
-			    "distributed spares %llu and parity level %llu\n"
+			fprintf(stderr, gettext("requested number of "
+			    "distributed spares %llu and parity level %llu "
 			    "leaves no disks available for data\n"),
 			    (u_longlong_t)nspares, (u_longlong_t)nparity);
 			return (EINVAL);
@@ -1450,7 +1505,7 @@ draid_config_by_type(nvlist_t *nv, const char *type, uint64_t children)
 		    (u_longlong_t)(ndata + nparity + nspares));
 	}
 
-	if (children > VDEV_DRAID_MAX_CHILDREN) {
+	if (width > VDEV_DRAID_MAX_CHILDREN) {
 		fprintf(stderr, gettext("%llu disks were provided, but "
 		    "dRAID only supports up to %u disks"),
 		    (u_longlong_t)children, VDEV_DRAID_MAX_CHILDREN);
@@ -1467,8 +1522,9 @@ draid_config_by_type(nvlist_t *nv, const char *type, uint64_t children)
 	/* Store the basic dRAID configuration. */
 	fnvlist_add_uint64(nv, ZPOOL_CONFIG_NPARITY, nparity);
 	fnvlist_add_uint64(nv, ZPOOL_CONFIG_DRAID_NDATA, ndata);
-	fnvlist_add_uint64(nv, ZPOOL_CONFIG_DRAID_NSPARES, nspares);
+	fnvlist_add_uint64(nv, ZPOOL_CONFIG_DRAID_NSPARES, nspares * fgrps);
 	fnvlist_add_uint64(nv, ZPOOL_CONFIG_DRAID_NGROUPS, ngroups);
+	fnvlist_add_uint64(nv, ZPOOL_CONFIG_DRAID_NCHILDREN, children);
 
 	return (0);
 }
@@ -1606,10 +1662,41 @@ construct_spec(nvlist_t *props, int argc, char **argv)
 				nlogs++;
 			}
 
+			int nfdomain = 0, nfgroup = 0;
+			int fdndev = 0, fgndev = 0;
+			int fdndev_prev = 0, fgndev_prev = 0;
+
 			for (c = 1; c < argc; c++) {
 				if (is_grouping(argv[c], NULL, NULL) != NULL)
 					break;
 
+				if (strcmp(argv[c], "fgroup") == 0 ||
+				    strcmp(argv[c], "failure_group") == 0) {
+					if (fgndev_prev &&
+					    fgndev_prev != fgndev)
+						break;
+					fgndev_prev = fgndev;
+					fgndev = 0;
+					nfgroup++;
+					continue;
+				}
+
+				if (strcmp(argv[c], "fdomain") == 0 ||
+				    strcmp(argv[c], "failure_domain") == 0) {
+					if (fdndev_prev &&
+					    fdndev_prev != fdndev)
+						break;
+					fdndev_prev = fdndev;
+					fdndev = 0;
+					nfdomain++;
+					continue;
+				}
+
+				if (nfgroup)
+					fgndev++;
+				if (nfdomain)
+					fdndev++;
+
 				children++;
 				child = realloc(child,
 				    children * sizeof (nvlist_t *));
@@ -1647,6 +1734,81 @@ construct_spec(nvlist_t *props, int argc, char **argv)
 				goto spec_out;
 			}
 
+			if ((nfdomain || nfgroup) &&
+			    strcmp(type, VDEV_TYPE_DRAID) != 0) {
+				(void) fprintf(stderr, gettext("invalid vdev "
+				    "specification: %s is not dRAID and cannot "
+				    "have failure domains\n"), argv[0]);
+				for (c = 0; c < children; c++)
+					nvlist_free(child[c]);
+				free(child);
+				goto spec_out;
+			}
+
+			if (nfgroup && nfdomain) {
+				(void) fprintf(stderr, gettext("invalid vdev "
+				    "specification: %s has mixed configuration "
+				    "of %d failure groups and %d failure "
+				    "domains, it must have either fgroups or "
+				    "fdomains, not both\n"), argv[0],
+				    nfgroup, nfdomain);
+				for (c = 0; c < children; c++)
+					nvlist_free(child[c]);
+				free(child);
+				goto spec_out;
+			}
+
+			if (nfgroup == 1 || nfdomain == 1) {
+				(void) fprintf(stderr, gettext("invalid vdev "
+				    "specification: %s has only one failure %s "
+				    "configured, it must be more than one\n"),
+				    argv[0], nfgroup ? "group" : "domain");
+				for (c = 0; c < children; c++)
+					nvlist_free(child[c]);
+				free(child);
+				goto spec_out;
+			}
+
+			if (fgndev_prev != fgndev) {
+				(void) fprintf(stderr, gettext("invalid vdev "
+				    "specification: %s has different number of "
+				    "devices in failure group %d than in "
+				    "previous group: %d != %d\n"), argv[0],
+				    nfgroup, fgndev, fgndev_prev);
+				for (c = 0; c < children; c++)
+					nvlist_free(child[c]);
+				free(child);
+				goto spec_out;
+			}
+
+			if (fdndev_prev != fdndev) {
+				(void) fprintf(stderr, gettext("invalid vdev "
+				    "specification: %s has different number of "
+				    "devices in failure domain %d than in "
+				    "previous domain: %d != %d\n"), argv[0],
+				    nfdomain, fdndev, fdndev_prev);
+				for (c = 0; c < children; c++)
+					nvlist_free(child[c]);
+				free(child);
+				goto spec_out;
+			}
+
+			if (nfdomain) {
+				/* Put children in the right order */
+				nvlist_t **ch = NULL;
+				ch = realloc(ch,
+				    children * sizeof (nvlist_t *));
+				if (ch == NULL)
+					zpool_no_memory();
+				int dlen = children / nfdomain;
+				int i = 0;
+				for (int g = 0; g < dlen; g++)
+					for (int d = 0; d < nfdomain; d++)
+						ch[i++] = child[g + (d * dlen)];
+				free(child);
+				child = ch;
+			}
+
 			argc -= c;
 			argv += c;
 
@@ -1692,7 +1854,8 @@ construct_spec(nvlist_t *props, int argc, char **argv)
 				}
 				if (strcmp(type, VDEV_TYPE_DRAID) == 0) {
 					if (draid_config_by_type(nv,
-					    fulltype, children) != 0) {
+					    fulltype, children, nfgroup,
+					    nfdomain) != 0) {
 						for (c = 0; c < children; c++)
 							nvlist_free(child[c]);
 						free(child);

include/libzfs.h

@@ -443,6 +443,7 @@ typedef enum {
 	 * checksum errors) has been lost.
 	 */
 	ZPOOL_STATUS_FAULTED_DEV_R,	/* faulted device with replicas */
+	ZPOOL_STATUS_FAULTED_FDOM_R,	/* faulted fdomain with replicas */
 	ZPOOL_STATUS_FAULTED_DEV_NR,	/* faulted device with no replicas */
 
 	/*

include/sys/fs/zfs.h

@@ -389,6 +389,8 @@ typedef enum {
 	VDEV_PROP_SIT_OUT,
 	VDEV_PROP_AUTOSIT,
 	VDEV_PROP_SLOW_IO_EVENTS,
+	VDEV_PROP_FDOMAIN,
+	VDEV_PROP_FGROUP,
 	VDEV_NUM_PROPS
 } vdev_prop_t;
 
@@ -907,6 +909,7 @@ typedef struct zpool_load_policy {
 #define	ZPOOL_CONFIG_DRAID_NDATA	"draid_ndata"
 #define	ZPOOL_CONFIG_DRAID_NSPARES	"draid_nspares"
 #define	ZPOOL_CONFIG_DRAID_NGROUPS	"draid_ngroups"
+#define	ZPOOL_CONFIG_DRAID_NCHILDREN	"draid_nchildren"
 
 #define	VDEV_TYPE_ROOT			"root"
 #define	VDEV_TYPE_MIRROR		"mirror"

include/sys/vdev_draid.h

@@ -68,9 +68,10 @@ typedef struct vdev_draid_config {
 	 */
 	uint64_t vdc_ndata;		/* # of data devices in group */
 	uint64_t vdc_nparity;		/* # of parity devices in group */
-	uint64_t vdc_nspares;		/* # of distributed spares */
+	uint64_t vdc_nspares;		/* # of distributed spares in slice */
 	uint64_t vdc_children;		/* # of children */
 	uint64_t vdc_ngroups;		/* # groups per slice */
+	uint64_t vdc_width;		/* # multiple of children */
 
 	/*
 	 * Immutable derived constants.
@@ -103,7 +104,9 @@ extern nvlist_t *vdev_draid_read_config_spare(vdev_t *);
 /* Functions for dRAID distributed spares. */
 extern vdev_t *vdev_draid_spare_get_child(vdev_t *, uint64_t);
 extern vdev_t *vdev_draid_spare_get_parent(vdev_t *);
-extern int vdev_draid_spare_create(nvlist_t *, vdev_t *, uint64_t *, uint64_t);
+extern int vdev_draid_spare_create(nvlist_t *, vdev_t *, uint64_t *, uint64_t *,
+    uint64_t);
+extern boolean_t vdev_draid_fail_domain_allowed(vdev_t *);
 
 #ifdef  __cplusplus
 }