Skip to content

Commit 32dec67

Browse files
nicholasciolinicholascioli
committed
feat(lvm-driver): enable RAID support
Add support for LVM2 RAID types and parameters, with sane defaults for backwards compatibility. lvm-driver now assumes that a non- specified RAID type corresponds to the previous default of linear RAID, where data is packed onto disk until it runs out of space, continuing to the next as necessary. Tests have been added to cover the main supported RAID types (e.g. raid0, raid1, raid5, raid6, and raid10), but technically any valid LVM RAID type should work as well. Fixes #164 Signed-off-by: Nicholas Cioli <[email protected]>
1 parent 9e0ac5b commit 32dec67

File tree

15 files changed

+648
-2
lines changed

15 files changed

+648
-2
lines changed

‎buildscripts/build.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,12 +96,13 @@ output_name="bin/${PNAME}/"$GOOS"_"$GOARCH"/"$CTLNAME
9696
if [ $GOOS = "windows" ]; then
9797
output_name+='.exe'
9898
fi
99-
env GOOS=$GOOS GOARCH=$GOARCH go build -ldflags \
99+
env GOOS=$GOOS GOARCH=$GOARCH CGO_ENABLED=0 go build -ldflags \
100100
"-X github.com/openebs/lvm-localpv/pkg/version.GitCommit=${GIT_COMMIT} \
101101
-X main.CtlName='${CTLNAME}' \
102102
-X github.com/openebs/lvm-localpv/pkg/version.Version=${VERSION} \
103103
-X github.com/openebs/lvm-localpv/pkg/version.VersionMeta=${VERSION_META}"\
104104
-o $output_name\
105+
-installsuffix cgo \
105106
./cmd
106107

107108
echo ""

‎changelogs/unreleased/164-nicholascioli‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
add support for LVM raid options

‎ci/ci-test.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,12 @@ fi
3333
FOREIGN_LVM_SYSTEMID="openebs-ci-test-system"
3434
FOREIGN_LVM_CONFIG="global{system_id_source=lvmlocal}local{system_id=${FOREIGN_LVM_SYSTEMID}}"
3535

36+
# RAID info for corresponding tests
37+
RAID_COUNT=5
38+
39+
# RAID info for corresponding tests
40+
RAID_COUNT=5
41+
3642
# Clean up generated resources for successive tests.
3743
cleanup_loopdev() {
3844
sudo losetup -l | grep '(deleted)' | awk '{print $1}' \
@@ -60,13 +66,28 @@ cleanup_foreign_lvmvg() {
6066
cleanup_loopdev
6167
}
6268

69+
cleanup_raidvg() {
70+
sudo vgremove raidvg -y || true
71+
72+
for IMG in `seq ${RAID_COUNT}`
73+
do
74+
if [ -f /tmp/openebs_ci_raid_disk_${IMG}.img ]
75+
then
76+
rm /tmp/openebs_ci_raid_disk_${IMG}.img
77+
fi
78+
done
79+
80+
cleanup_loopdev
81+
}
82+
6383
cleanup() {
6484
set +e
6585

6686
echo "Cleaning up test resources"
6787

6888
cleanup_lvmvg
6989
cleanup_foreign_lvmvg
90+
cleanup_raidvg
7091

7192
kubectl delete pvc -n openebs lvmpv-pvc
7293
kubectl delete -f "${SNAP_CLASS}"
@@ -93,10 +114,40 @@ foreign_disk="$(sudo losetup -f /tmp/openebs_ci_foreign_disk.img --show)"
93114
sudo pvcreate "${foreign_disk}"
94115
sudo vgcreate foreign_lvmvg "${foreign_disk}" --config="${FOREIGN_LVM_CONFIG}"
95116

117+
# setup a RAID volume group
118+
cleanup_raidvg
119+
raid_disks=()
120+
for IMG in `seq ${RAID_COUNT}`
121+
do
122+
truncate -s 1024G /tmp/openebs_ci_raid_disk_${IMG}.img
123+
raid_disk="$(sudo losetup -f /tmp/openebs_ci_raid_disk_${IMG}.img --show)"
124+
sudo pvcreate "${raid_disk}"
125+
126+
raid_disks+=("${raid_disk}")
127+
done
128+
sudo vgcreate raidvg "${raid_disks[@]}"
129+
130+
# setup a RAID volume group
131+
cleanup_raidvg
132+
raid_disks=()
133+
for IMG in `seq ${RAID_COUNT}`
134+
do
135+
truncate -s 1024G /tmp/openebs_ci_raid_disk_${IMG}.img
136+
raid_disk="$(sudo losetup -f /tmp/openebs_ci_raid_disk_${IMG}.img --show)"
137+
sudo pvcreate "${raid_disk}"
138+
139+
raid_disks+=("${raid_disk}")
140+
done
141+
sudo vgcreate raidvg "${raid_disks[@]}"
142+
96143
# install snapshot and thin volume module for lvm
97144
sudo modprobe dm-snapshot
98145
sudo modprobe dm_thin_pool
99146

147+
# install RAID modules for lvm
148+
sudo modprobe dm_raid
149+
sudo modprobe dm_integrity
150+
100151
# Prepare env for running BDD tests
101152
# Minikube is already running
102153
kubectl apply -f "${LVM_OPERATOR}"

‎deploy/lvm-operator.yaml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,12 +95,47 @@ spec:
9595
description: Capacity of the volume
9696
minLength: 1
9797
type: string
98+
integrity:
99+
description: Integrity specifies whether logical volumes should be
100+
checked for integrity. If it is set to "yes", then the LVM LocalPV
101+
Driver will enable DM integrity for the logical volume
102+
enum:
103+
- "yes"
104+
- "no"
105+
type: string
106+
lvcreateoptions:
107+
description: LvCreateOptions are extra options for creating a volume.
108+
Options should be separated by ; e.g. "--vdo;--readahead;auto"
109+
type: string
110+
mirrors:
111+
description: Mirrors specifies the mirror count for a RAID configuration.
112+
minimum: 0
113+
type: integer
114+
nosync:
115+
description: NoSync enables the `--nosync` option of a RAID volume.
116+
If it is set to "yes", then LVM will skip drive sync when creating
117+
the mirrors. Defaults to "no"
118+
enum:
119+
- "yes"
120+
- "no"
121+
type: string
98122
ownerNodeID:
99123
description: OwnerNodeID is the Node ID where the volume group is
100124
present which is where the volume has been provisioned. OwnerNodeID
101125
can not be edited after the volume has been provisioned.
102126
minLength: 1
103127
type: string
128+
raidtype:
129+
description: RaidType specifies the type of RAID for the logical volume.
130+
Defaults to linear, if unspecified.
131+
enum:
132+
- linear
133+
- raid0
134+
- raid1
135+
- raid5
136+
- raid6
137+
- raid10
138+
type: string
104139
shared:
105140
description: Shared specifies whether the volume can be shared among
106141
multiple pods. If it is not set to "yes", then the LVM LocalPV Driver
@@ -109,6 +144,18 @@ spec:
109144
- "yes"
110145
- "no"
111146
type: string
147+
stripecount:
148+
description: StripeCount specifies the stripe count for a RAID configuration.
149+
This is equal to the number of physical volumes to scatter the logical
150+
volume
151+
minimum: 0
152+
type: integer
153+
stripesize:
154+
description: StripeSize specifies the size of a stripe for a RAID
155+
configuration. Must be a power of 2 but must not exceed the physical
156+
extent size
157+
minimum: 0
158+
type: integer
112159
thinProvision:
113160
description: ThinProvision specifies whether logical volumes can be
114161
thinly provisioned. If it is set to "yes", then the LVM LocalPV
@@ -129,6 +176,7 @@ spec:
129176
required:
130177
- capacity
131178
- ownerNodeID
179+
- raidtype
132180
- vgPattern
133181
- volGroup
134182
type: object

‎deploy/yamls/lvmvolume-crd.yaml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,12 +74,47 @@ spec:
7474
description: Capacity of the volume
7575
minLength: 1
7676
type: string
77+
integrity:
78+
description: Integrity specifies whether logical volumes should be
79+
checked for integrity. If it is set to "yes", then the LVM LocalPV
80+
Driver will enable DM integrity for the logical volume
81+
enum:
82+
- "yes"
83+
- "no"
84+
type: string
85+
lvcreateoptions:
86+
description: LvCreateOptions are extra options for creating a volume.
87+
Options should be separated by ; e.g. "--vdo;--readahead;auto"
88+
type: string
89+
mirrors:
90+
description: Mirrors specifies the mirror count for a RAID configuration.
91+
minimum: 0
92+
type: integer
93+
nosync:
94+
description: NoSync enables the `--nosync` option of a RAID volume.
95+
If it is set to "yes", then LVM will skip drive sync when creating
96+
the mirrors. Defaults to "no"
97+
enum:
98+
- "yes"
99+
- "no"
100+
type: string
77101
ownerNodeID:
78102
description: OwnerNodeID is the Node ID where the volume group is
79103
present which is where the volume has been provisioned. OwnerNodeID
80104
can not be edited after the volume has been provisioned.
81105
minLength: 1
82106
type: string
107+
raidtype:
108+
description: RaidType specifies the type of RAID for the logical volume.
109+
Defaults to linear, if unspecified.
110+
enum:
111+
- linear
112+
- raid0
113+
- raid1
114+
- raid5
115+
- raid6
116+
- raid10
117+
type: string
83118
shared:
84119
description: Shared specifies whether the volume can be shared among
85120
multiple pods. If it is not set to "yes", then the LVM LocalPV Driver
@@ -88,6 +123,18 @@ spec:
88123
- "yes"
89124
- "no"
90125
type: string
126+
stripecount:
127+
description: StripeCount specifies the stripe count for a RAID configuration.
128+
This is equal to the number of physical volumes to scatter the logical
129+
volume
130+
minimum: 0
131+
type: integer
132+
stripesize:
133+
description: StripeSize specifies the size of a stripe for a RAID
134+
configuration. Must be a power of 2 but must not exceed the physical
135+
extent size
136+
minimum: 0
137+
type: integer
91138
thinProvision:
92139
description: ThinProvision specifies whether logical volumes can be
93140
thinly provisioned. If it is set to "yes", then the LVM LocalPV
@@ -108,6 +155,7 @@ spec:
108155
required:
109156
- capacity
110157
- ownerNodeID
158+
- raidtype
111159
- vgPattern
112160
- volGroup
113161
type: object

‎design/lvm/storageclass-parameters/raid.md‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
---
2+
title: LVM-LocalPV RAID
3+
authors:
4+
- "@nicholascioli"
5+
owners: []
6+
creation-date: 2023-11-04
7+
last-updated: 2023-11-04
8+
status: Implemented
9+
---
10+
11+
# LVM-LocalPV RAID
12+
13+
## Table of Contents
14+
- [LVM-LocalPV RAID](#lvm-localpv-raid)
15+
- [Table of Contents](#table-of-contents)
16+
- [Summary](#summary)
17+
- [Motivation](#motivation)
18+
- [Goals](#goals)
19+
- [Non Goals](#non-goals)
20+
- [Proposal](#proposal)
21+
- [User Stories](#user-stories)
22+
- [Implementation Details](#implementation-details)
23+
- [Usage details](#usage-details)
24+
- [Test Plan](#test-plan)
25+
- [Graduation Criteria](#graduation-criteria)
26+
- [Drawbacks](#drawbacks)
27+
- [Alternatives](#alternatives)
28+
29+
30+
## Summary
31+
32+
This proposal charts out the workflow details to support creation of RAID volumes.
33+
34+
## Motivation
35+
36+
### Goals
37+
38+
- Able to provision RAID volumes in a VolumeGroup.
39+
- Able to specify VolumeGroup-specific RAID options for all sub volumes.
40+
- Able to specify extra options for all volumes in a VolumeGroup.
41+
42+
### Non Goals
43+
44+
- Validating combinations of RAID types / options.
45+
46+
## Proposal
47+
48+
### User Stories
49+
50+
- RAIDed volumes provide data redundancy and can mitigate data loss due to individual drive failures.
51+
- Ability to specify extra arguments for VolumeGroups allow for user customizations without needing
52+
to rework k8s schemas.
53+
54+
### Implementation Details
55+
56+
- User/Admin has to set RAID-sepcific options under storageclass parameters which
57+
are used when creating volumes in the VolumeGroup.
58+
- During volume provisioning time external-provisioner will read all key-value pairs
59+
that are specified under referenced storageclass and pass information to CSI
60+
driver as payload for `CreateVolume` gRPC request.
61+
- After receiving the `CreateVolume` request CSI driver will pick appropriate node based
62+
on scheduling attributes(like topology information, matching VG name and available capacity)
63+
and creates LVM volume resource by setting `Spec.RaidType` to a valid type along with other properties.
64+
- Once the LVMVolume resource is created corresponding node LVM volume controller reconcile
65+
LVM volume resource in the following way:
66+
- LVM controller will check `Spec.RaidType` field, if the field is set to anything other
67+
than `linear`, then the controller will perform following operations:
68+
- Fetch information about existence of matching VolumeGroup.
69+
- If there is a VolumeGroup with <vg_name> name then controller will create a volume.
70+
Command used to create thin volume: `lvcreate --type <RAID_TYPE> --raidintegrity <INTEGRITY> --nosync ... <LVCREATEOPTIONS> -y`
71+
- If volume creation is successfull then controller will LVM volume resource as `Ready`.
72+
- After watching `Ready` status CSI driver will return success response to `CreateVolume` gRPC
73+
request.
74+
75+
### Usage details
76+
77+
1. User/Admin can configure the following options under the storageclass parameters.
78+
79+
Option | Required | Valid Values | Description
80+
-------|----------|--------------|-------------------
81+
`type` | `true` | `raid0` / `stripe`, `raid` / `raid1` / `mirror`, `raid5`, `raid6`, `raid10` | The RAID type of the volume.
82+
`integrity` | `false` | `true`, `false` | Whether or not to enable DM integrity for the volume. Defaults to `false`.
83+
`mirrors` | depends | [0, ∞) | Mirror count. Certain RAID configurations require this to be set.
84+
`nosync` | `false` | `true`, `false` | Whether or not to disable the initial sync. Defaults to false.
85+
`stripecount` | depends | [0, ∞) | Stripe count. Certain RAID configurations require this to be set.
86+
`stripesize` | `false` | [0, ∞) (but must be a power of 2) | The size of each stripe. If not specified, LVM will choose a sane default.
87+
`lvcreateoptions` | `false` | String, delimited by `;` | Extra options to be passed to LVM when creating volumes.
88+
89+
An example is shown below
90+
```yaml
91+
apiVersion: storage.k8s.io/v1
92+
kind: StorageClass
93+
metadata:
94+
name: openebs-lvm
95+
provisioner: local.csi.openebs.io
96+
parameters:
97+
storage: "lvm"
98+
volgroup: "lvmvg"
99+
raidType: "raid1"
100+
lvcreateoptions: "--vdo;--readahead auto"
101+
```
102+
103+
### Test Plan
104+
- Provision an application on various RAID configurations, verify volume accessibility from application,
105+
and verify that `lvs` reports correct RAID information.
106+
107+
## Graduation Criteria
108+
109+
All testcases mentioned in [Test Plan](#test-plan) section need to be automated
110+
111+
## Drawbacks
112+
113+
- Since the RAID options exist at the storageclass level, changes to the storage
114+
class RAID options is not possible without custom logic per RAID type or manual
115+
operator interactions.
116+
- Validation of the RAID options depend on the version of LVM2 installed as well as
117+
the type of RAID used and its options. This is outside of the scope of these changes
118+
and will cause users to have to debug issues with a finer comb to see why certain
119+
options do not work together or on their specific machine.
120+
121+
## Alternatives
122+
123+
RAID can be done in either software or hardware, with many off-the-shelf products
124+
including built-in hardware solutions. There are also other software RAID alternatives
125+
that can be used below LVM, such as mdadm.
126+
127+
This unfortunately requires operators to decouple
128+
the SotrageClass from the RAID configuration, but does simplify the amount of code maintained by
129+
this project.

0 commit comments

Comments
 (0)