Skip to content

Commit 82e2c06

Browse files
robertmurobertmu
committed
refactor(meta): Adapt resource group DDL from GPDB<7 sources to be target-aware
Refactors the generation of DDL statements for resource groups when migrating data from Greenplum Database (GPDB) versions prior to 7. This change enhances compatibility and accuracy by making the DDL generation sensitive to the target database version. Key changes in `PrintCreateResourceGroupStatementsBefore7`: - The function now dispatches to new helper functions based on whether the `destDBVersion` is GPDB 7+ / CloudberryDB or an older GPDB version. - For migrations from GPDB < 7 to GPDB 7+ or CloudberryDB targets: - Resource group attributes are translated to their modern syntax (e.g., `CPU_RATE_LIMIT` to `CPU_MAX_PERCENT`, addition of `CPU_WEIGHT`). - System groups (`default_group`, `admin_group`, `system_group`) are modified using `ALTER RESOURCE GROUP`. - User-defined groups are created using `CREATE RESOURCE GROUP` with the updated syntax. - For migrations from GPDB < 7 to older GPDB < 7 targets: - The existing DDL generation logic is largely preserved, maintaining compatibility with older GPDB features and syntax (e.g., GPDB 5.x specific `memory_spill_ratio` handling, `CPU_RATE_LIMIT`, `MEMORY_AUDITOR`). Additionally, the logic in `PrintResetResourceGroupStatements` for selecting which default values to use for resetting resource groups (e.g., `CPU_RATE_LIMIT` vs. `CPU_MAX_PERCENT` based parameters) has been modified. This selection continues to be based on `srcDBVersion`.
1 parent 0a89ff2 commit 82e2c06

File tree

2 files changed

+119
-73
lines changed

2 files changed

+119
-73
lines changed

‎meta/builtin/metadata_globals.go‎

Lines changed: 118 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -99,11 +99,11 @@ func PrintCreateResourceQueueStatements(metadataFile *utils.FileWithByteCount, t
9999

100100
func PrintResetResourceGroupStatements(metadataFile *utils.FileWithByteCount, tocfile *toc.TOC) {
101101
/*
102-
* total cpu_rate_limit and memory_limit should less than 100, so clean
103-
* them before we seting new memory_limit and cpu_rate_limit.
102+
* Total cpu_rate_limit and memory_limit should be less than 100, so clean
103+
* them before setting new memory_limit and cpu_rate_limit.
104104
*
105105
* Minimal memory_limit is adjusted from 1 to 0 since 5.20, however for
106-
* backward compatibility we still use 1 as the minimal value. The only
106+
* backward compatibility we still use 1 as the minimal value. The only
107107
* failing case is that default_group has memory_limit=100 and admin_group
108108
* has memory_limit=0, but this should not happen in real world.
109109
*/
@@ -115,12 +115,14 @@ func PrintResetResourceGroupStatements(metadataFile *utils.FileWithByteCount, to
115115
}
116116
defSettings := make([]DefSetting, 0)
117117

118-
if srcDBVersion.IsGPDB() && srcDBVersion.Before("7") {
118+
if destDBVersion.IsGPDB() && destDBVersion.Before("7") {
119+
// Target is GPDB 6 or earlier
119120
defSettings = append(defSettings, DefSetting{"admin_group", "SET CPU_RATE_LIMIT 1"})
120121
defSettings = append(defSettings, DefSetting{"admin_group", "SET MEMORY_LIMIT 1"})
121122
defSettings = append(defSettings, DefSetting{"default_group", "SET CPU_RATE_LIMIT 1"})
122123
defSettings = append(defSettings, DefSetting{"default_group", "SET MEMORY_LIMIT 1"})
123-
} else { // GPDB7+
124+
} else {
125+
// Target is GPDB 7+ or CloudberryDB
124126
defSettings = append(defSettings, DefSetting{"admin_group", "SET CPU_MAX_PERCENT 1"})
125127
defSettings = append(defSettings, DefSetting{"admin_group", "SET CPU_WEIGHT 100"})
126128
defSettings = append(defSettings, DefSetting{"default_group", "SET CPU_MAX_PERCENT 1"})
@@ -195,88 +197,131 @@ func PrintCreateResourceGroupStatementsAtLeast7(metadataFile *utils.FileWithByte
195197
}
196198
}
197199

198-
func PrintCreateResourceGroupStatementsBefore7(metadataFile *utils.FileWithByteCount, toc *toc.TOC, resGroups []ResourceGroupBefore7, resGroupMetadata MetadataMap) {
199-
for _, resGroup := range resGroups {
200+
// printResGroupStmtsBefore7ForTarget7OrLater handles printing resource group statements
201+
// when the source is GPDB before 7, and the target is GPDB 7+ or CloudberryDB.
202+
func printResGroupStmtsBefore7ForTarget7OrLater(metadataFile *utils.FileWithByteCount, toc *toc.TOC, resGroup ResourceGroupBefore7, resGroupMetadata MetadataMap) {
203+
var start uint64
204+
section, entry := resGroup.GetMetadataEntry()
200205

201-
// temporarily special case for 5x resource groups #temp5xResGroup
202-
memorySpillRatio := resGroup.MemorySpillRatio
206+
if resGroup.Name == "default_group" || resGroup.Name == "admin_group" || resGroup.Name == "system_group" {
207+
start = metadataFile.ByteCount
208+
metadataFile.MustPrintf("\n\nALTER RESOURCE GROUP %s SET CONCURRENCY %s;", resGroup.Name, resGroup.Concurrency)
209+
toc.AddMetadataEntry(section, entry, start, metadataFile.ByteCount)
203210

204-
if srcDBVersion.IsGPDB() && srcDBVersion.Is("5") {
205-
/*
206-
* memory_spill_ratio can be set in absolute value format since 5.20,
207-
* such as '1 MB', it has to be set as a quoted string, otherwise set
208-
* it without quotes.
209-
*/
210-
if _, err := strconv.Atoi(memorySpillRatio); err != nil {
211-
/* memory_spill_ratio is in absolute value format, set it with quotes */
211+
start = metadataFile.ByteCount
212+
metadataFile.MustPrintf("\n\nALTER RESOURCE GROUP %s SET CPU_WEIGHT %s;", resGroup.Name, "100")
213+
toc.AddMetadataEntry(section, entry, start, metadataFile.ByteCount)
212214

213-
memorySpillRatio = "'" + memorySpillRatio + "'"
214-
}
215+
start = metadataFile.ByteCount
216+
if !strings.HasPrefix(resGroup.CPURateLimit, "-") {
217+
metadataFile.MustPrintf("\n\nALTER RESOURCE GROUP %s SET CPU_MAX_PERCENT %s;", resGroup.Name, resGroup.CPURateLimit)
218+
} else {
219+
metadataFile.MustPrintf("\n\nALTER RESOURCE GROUP %s SET CPUSET '%s';", resGroup.Name, resGroup.Cpuset)
215220
}
221+
toc.AddMetadataEntry(section, entry, start, metadataFile.ByteCount)
216222

217-
var start uint64
218-
section, entry := resGroup.GetMetadataEntry()
219-
if resGroup.Name == "default_group" || resGroup.Name == "admin_group" {
220-
resGroupList := []struct {
221-
setting string
222-
value string
223-
}{
224-
{"MEMORY_LIMIT", resGroup.MemoryLimit},
225-
{"MEMORY_SHARED_QUOTA", resGroup.MemorySharedQuota},
226-
{"MEMORY_SPILL_RATIO", memorySpillRatio},
227-
{"CONCURRENCY", resGroup.Concurrency},
228-
}
229-
for _, property := range resGroupList {
230-
start = metadataFile.ByteCount
231-
metadataFile.MustPrintf("\n\nALTER RESOURCE GROUP %s SET %s %s;", resGroup.Name, property.setting, property.value)
223+
PrintObjectMetadata(metadataFile, toc, resGroupMetadata[resGroup.GetUniqueID()], resGroup, "")
232224

233-
toc.AddMetadataEntry(section, entry, start, metadataFile.ByteCount)
234-
}
225+
} else {
226+
start = metadataFile.ByteCount
227+
attributes := make([]string, 0)
235228

236-
/* special handling for cpu properties */
237-
start = metadataFile.ByteCount
238-
if !strings.HasPrefix(resGroup.CPURateLimit, "-") {
239-
/* cpu rate mode */
240-
metadataFile.MustPrintf("\n\nALTER RESOURCE GROUP %s SET CPU_RATE_LIMIT %s;", resGroup.Name, resGroup.CPURateLimit)
241-
} else if (srcDBVersion.IsGPDB() && srcDBVersion.AtLeast("5.9.0")) || srcDBVersion.IsCBDBFamily() {
242-
/* cpuset mode */
243-
metadataFile.MustPrintf("\n\nALTER RESOURCE GROUP %s SET CPUSET '%s';", resGroup.Name, resGroup.Cpuset)
244-
}
229+
attributes = append(attributes, fmt.Sprintf("CONCURRENCY=%s", resGroup.Concurrency))
230+
attributes = append(attributes, fmt.Sprintf("CPU_WEIGHT=%s", "100"))
245231

246-
toc.AddMetadataEntry(section, entry, start, metadataFile.ByteCount)
247-
PrintObjectMetadata(metadataFile, toc, resGroupMetadata[resGroup.GetUniqueID()], resGroup, "")
232+
if !strings.HasPrefix(resGroup.CPURateLimit, "-") {
233+
attributes = append(attributes, fmt.Sprintf("CPU_MAX_PERCENT=%s", resGroup.CPURateLimit))
248234
} else {
249-
start = metadataFile.ByteCount
250-
attributes := make([]string, 0)
235+
// Assuming Cpuset is applicable/desired here as it was in the original logic for CREATE
236+
attributes = append(attributes, fmt.Sprintf("CPUSET='%s'", resGroup.Cpuset))
237+
}
251238

252-
/* special handling for cpu properties */
253-
if !strings.HasPrefix(resGroup.CPURateLimit, "-") {
254-
/* cpu rate mode */
255-
attributes = append(attributes, fmt.Sprintf("CPU_RATE_LIMIT=%s", resGroup.CPURateLimit))
256-
} else if (srcDBVersion.IsGPDB() && srcDBVersion.AtLeast("5.9.0")) || srcDBVersion.IsCBDBFamily() {
257-
/* cpuset mode */
258-
attributes = append(attributes, fmt.Sprintf("CPUSET='%s'", resGroup.Cpuset))
259-
}
239+
metadataFile.MustPrintf("\n\nCREATE RESOURCE GROUP %s WITH (%s);", resGroup.Name, strings.Join(attributes, ", "))
240+
toc.AddMetadataEntry(section, entry, start, metadataFile.ByteCount)
241+
PrintObjectMetadata(metadataFile, toc, resGroupMetadata[resGroup.GetUniqueID()], resGroup, "")
242+
}
243+
}
260244

261-
/*
262-
* Possible values of memory_auditor:
263-
* - "1": cgroup
264-
* - "0": vmtracker (default)
265-
*/
266-
if resGroup.MemoryAuditor == "1" {
267-
attributes = append(attributes, "MEMORY_AUDITOR=cgroup")
268-
} else if (srcDBVersion.IsGPDB() && srcDBVersion.AtLeast("5.8.0")) || srcDBVersion.IsCBDBFamily() {
269-
attributes = append(attributes, "MEMORY_AUDITOR=vmtracker")
270-
}
245+
// printResGroupStmtsBefore7ForTargetBefore7 handles printing resource group statements
246+
// when the source is GPDB before 7, and the target is also GPDB before 7.
247+
func printResGroupStmtsBefore7ForTargetBefore7(metadataFile *utils.FileWithByteCount, toc *toc.TOC, resGroup ResourceGroupBefore7, resGroupMetadata MetadataMap) {
248+
var start uint64
249+
section, entry := resGroup.GetMetadataEntry()
271250

272-
attributes = append(attributes, fmt.Sprintf("MEMORY_LIMIT=%s", resGroup.MemoryLimit))
273-
attributes = append(attributes, fmt.Sprintf("MEMORY_SHARED_QUOTA=%s", resGroup.MemorySharedQuota))
274-
attributes = append(attributes, fmt.Sprintf("MEMORY_SPILL_RATIO=%s", memorySpillRatio))
275-
attributes = append(attributes, fmt.Sprintf("CONCURRENCY=%s", resGroup.Concurrency))
276-
metadataFile.MustPrintf("\n\nCREATE RESOURCE GROUP %s WITH (%s);", resGroup.Name, strings.Join(attributes, ", "))
251+
memorySpillRatio := resGroup.MemorySpillRatio
252+
// Check if source DB is GPDB version 5. GPDB 5 has a memory_spill_ratio 'gpmemlimit' which is not an int
253+
if srcDBVersion.IsGPDB() && srcDBVersion.Is("5") {
254+
if _, err := strconv.Atoi(resGroup.MemorySpillRatio); err != nil {
255+
memorySpillRatio = "'" + resGroup.MemorySpillRatio + "'"
256+
}
257+
}
277258

259+
if resGroup.Name == "default_group" || resGroup.Name == "admin_group" {
260+
resGroupList := []struct {
261+
setting string
262+
value string
263+
}{
264+
{"MEMORY_LIMIT", resGroup.MemoryLimit},
265+
{"MEMORY_SHARED_QUOTA", resGroup.MemorySharedQuota},
266+
{"MEMORY_SPILL_RATIO", memorySpillRatio},
267+
{"CONCURRENCY", resGroup.Concurrency},
268+
}
269+
for _, property := range resGroupList {
270+
start = metadataFile.ByteCount
271+
metadataFile.MustPrintf("\n\nALTER RESOURCE GROUP %s SET %s %s;", resGroup.Name, property.setting, property.value)
278272
toc.AddMetadataEntry(section, entry, start, metadataFile.ByteCount)
279-
PrintObjectMetadata(metadataFile, toc, resGroupMetadata[resGroup.GetUniqueID()], resGroup, "")
273+
}
274+
275+
start = metadataFile.ByteCount
276+
// For CPU_RATE_LIMIT vs CPUSET
277+
if !strings.HasPrefix(resGroup.CPURateLimit, "-") {
278+
metadataFile.MustPrintf("\n\nALTER RESOURCE GROUP %s SET CPU_RATE_LIMIT %s;", resGroup.Name, resGroup.CPURateLimit)
279+
} else if srcDBVersion.IsGPDB() && srcDBVersion.AtLeast("5.9.0") { // CPUSET available from GPDB 5.9.0
280+
metadataFile.MustPrintf("\n\nALTER RESOURCE GROUP %s SET CPUSET '%s';", resGroup.Name, resGroup.Cpuset)
281+
}
282+
toc.AddMetadataEntry(section, entry, start, metadataFile.ByteCount)
283+
PrintObjectMetadata(metadataFile, toc, resGroupMetadata[resGroup.GetUniqueID()], resGroup, "")
284+
} else {
285+
start = metadataFile.ByteCount
286+
attributes := make([]string, 0)
287+
288+
// For CPU_RATE_LIMIT vs CPUSET
289+
if !strings.HasPrefix(resGroup.CPURateLimit, "-") {
290+
attributes = append(attributes, fmt.Sprintf("CPU_RATE_LIMIT=%s", resGroup.CPURateLimit))
291+
} else if srcDBVersion.IsGPDB() && srcDBVersion.AtLeast("5.9.0") { // CPUSET available from GPDB 5.9.0
292+
attributes = append(attributes, fmt.Sprintf("CPUSET='%s'", resGroup.Cpuset))
293+
}
294+
295+
// Memory Auditor: cgroup (value 1) vs vmtracker (available from GPDB 5.8.0)
296+
if resGroup.MemoryAuditor == "1" { // Assuming "1" means cgroup
297+
attributes = append(attributes, "MEMORY_AUDITOR=cgroup")
298+
} else if srcDBVersion.IsGPDB() && srcDBVersion.AtLeast("5.8.0") {
299+
attributes = append(attributes, "MEMORY_AUDITOR=vmtracker")
300+
}
301+
302+
attributes = append(attributes, fmt.Sprintf("MEMORY_LIMIT=%s", resGroup.MemoryLimit))
303+
attributes = append(attributes, fmt.Sprintf("MEMORY_SHARED_QUOTA=%s", resGroup.MemorySharedQuota))
304+
attributes = append(attributes, fmt.Sprintf("MEMORY_SPILL_RATIO=%s", memorySpillRatio))
305+
attributes = append(attributes, fmt.Sprintf("CONCURRENCY=%s", resGroup.Concurrency))
306+
metadataFile.MustPrintf("\n\nCREATE RESOURCE GROUP %s WITH (%s);", resGroup.Name, strings.Join(attributes, ", "))
307+
308+
toc.AddMetadataEntry(section, entry, start, metadataFile.ByteCount)
309+
PrintObjectMetadata(metadataFile, toc, resGroupMetadata[resGroup.GetUniqueID()], resGroup, "")
310+
}
311+
}
312+
313+
func PrintCreateResourceGroupStatementsBefore7(metadataFile *utils.FileWithByteCount, toc *toc.TOC, resGroups []ResourceGroupBefore7, resGroupMetadata MetadataMap) {
314+
// Determine if the target database version is GPDB 7 or later, or any CloudberryDB family database.
315+
// This flag helps in deciding which syntax or features to use for resource groups.
316+
isTargetGpdb7OrLater := (destDBVersion.IsGPDB() && destDBVersion.AtLeast("7")) || destDBVersion.IsCBDBFamily()
317+
318+
for _, resGroup := range resGroups {
319+
if isTargetGpdb7OrLater {
320+
// If the target is GPDB 7+ or CloudberryDB, use the specific logic for these versions.
321+
printResGroupStmtsBefore7ForTarget7OrLater(metadataFile, toc, resGroup, resGroupMetadata)
322+
} else {
323+
// Otherwise, use the logic for older GPDB versions (before 7).
324+
printResGroupStmtsBefore7ForTargetBefore7(metadataFile, toc, resGroup, resGroupMetadata)
280325
}
281326
}
282327
}

‎meta/builtin/metadata_globals_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ GRANT TEMPORARY,CONNECT ON DATABASE testdb TO testrole;`,
162162
defaultGroup := builtin.ResourceGroupBefore7{ResourceGroup: builtin.ResourceGroup{Oid: 1, Name: "default_group", Concurrency: "15"}, CPURateLimit: "10", MemoryLimit: "20", MemorySharedQuota: "25", MemorySpillRatio: "30"}
163163
resGroups := []builtin.ResourceGroupBefore7{defaultGroup}
164164

165+
SetDBVersion(connectionPool, "6.0.0")
165166
builtin.PrintCreateResourceGroupStatementsBefore7(backupfile, tocfile, resGroups, emptyResGroupMetadata)
166167
testutils.ExpectEntry(tocfile.GlobalEntries, 0, "", "", "default_group", "RESOURCE GROUP")
167168
testutils.AssertBufferContents(tocfile.GlobalEntries, buffer,

0 commit comments

Comments
 (0)