Skip to content

Commit 65fdf82

Browse files
LexLuthrLexLuthr
authored
keep base layer updated with new config (#405)
* keep base layer updated with new config * use transaction * ensure new keys are not removed by old nodes * remove commented code * harmony retry
1 parent b98675c commit 65fdf82

File tree

2 files changed

+255
-0
lines changed

2 files changed

+255
-0
lines changed

‎deps/deps.go

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
package deps
33

44
import (
5+
"bytes"
56
"context"
67
"crypto/rand"
78
"database/sql"
@@ -429,6 +430,11 @@ func LoadConfigWithUpgrades(text string, curioConfigWithDefaults *config.CurioCo
429430
return meta, err
430431
}
431432
func GetConfig(ctx context.Context, layers []string, db *harmonydb.DB) (*config.CurioConfig, error) {
433+
err := updateBaseLayer(ctx, db)
434+
if err != nil {
435+
return nil, err
436+
}
437+
432438
curioConfig := config.DefaultCurioConfig()
433439
have := []string{}
434440
layers = append([]string{"base"}, layers...) // Always stack on top of "base" layer
@@ -462,6 +468,131 @@ func GetConfig(ctx context.Context, layers []string, db *harmonydb.DB) (*config.
462468
return curioConfig, nil
463469
}
464470

471+
func updateBaseLayer(ctx context.Context, db *harmonydb.DB) error {
472+
_, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
473+
// Get existing base from DB
474+
text := ""
475+
err = tx.QueryRow(`SELECT config FROM harmony_config WHERE title=$1`, "base").Scan(&text)
476+
if err != nil {
477+
if strings.Contains(err.Error(), sql.ErrNoRows.Error()) {
478+
return false, fmt.Errorf("missing layer 'base' ")
479+
}
480+
return false, fmt.Errorf("could not read layer 'base': %w", err)
481+
}
482+
483+
// Load the existing configuration
484+
cfg := config.DefaultCurioConfig()
485+
metadata, err := LoadConfigWithUpgrades(text, cfg)
486+
if err != nil {
487+
return false, fmt.Errorf("could not read base layer, bad toml %s: %w", text, err)
488+
}
489+
490+
// Capture unknown fields
491+
keys := removeUnknownEntries(metadata.Keys(), metadata.Undecoded())
492+
unrecognizedFields := extractUnknownFields(keys, text)
493+
494+
// Convert the updated config back to TOML string
495+
cb, err := config.ConfigUpdate(cfg, config.DefaultCurioConfig(), config.Commented(true), config.DefaultKeepUncommented(), config.NoEnv())
496+
if err != nil {
497+
return false, xerrors.Errorf("cannot update base config: %w", err)
498+
}
499+
500+
// Merge unknown fields back into the updated config
501+
finalConfig, err := mergeUnknownFields(string(cb), unrecognizedFields)
502+
if err != nil {
503+
return false, xerrors.Errorf("cannot merge unknown fields: %w", err)
504+
}
505+
506+
// Check if we need to update the DB
507+
if text == finalConfig {
508+
return false, nil
509+
}
510+
511+
// Save the updated base with merged comments
512+
_, err = tx.Exec("UPDATE harmony_config SET config=$1 WHERE title='base'", finalConfig)
513+
if err != nil {
514+
return false, xerrors.Errorf("cannot update base config: %w", err)
515+
}
516+
517+
return true, nil
518+
}, harmonydb.OptionRetry())
519+
520+
if err != nil {
521+
return err
522+
}
523+
524+
return nil
525+
}
526+
527+
func extractUnknownFields(knownKeys []toml.Key, originalConfig string) map[string]interface{} {
528+
// Parse the original config into a raw map
529+
var rawConfig map[string]interface{}
530+
err := toml.Unmarshal([]byte(originalConfig), &rawConfig)
531+
if err != nil {
532+
log.Warnw("Failed to parse original config for unknown fields", "error", err)
533+
return nil
534+
}
535+
536+
// Collect all recognized keys
537+
recognizedKeys := map[string]struct{}{}
538+
for _, key := range knownKeys {
539+
recognizedKeys[strings.Join(key, ".")] = struct{}{}
540+
}
541+
542+
// Identify unrecognized fields
543+
unrecognizedFields := map[string]interface{}{}
544+
for key, value := range rawConfig {
545+
if _, recognized := recognizedKeys[key]; !recognized {
546+
unrecognizedFields[key] = value
547+
}
548+
}
549+
return unrecognizedFields
550+
}
551+
552+
func removeUnknownEntries(array1, array2 []toml.Key) []toml.Key {
553+
// Create a set from array2 for fast lookup
554+
toRemove := make(map[string]struct{}, len(array2))
555+
for _, key := range array2 {
556+
toRemove[key.String()] = struct{}{}
557+
}
558+
559+
// Filter array1, keeping only elements not in toRemove
560+
var result []toml.Key
561+
for _, key := range array1 {
562+
if _, exists := toRemove[key.String()]; !exists {
563+
result = append(result, key)
564+
}
565+
}
566+
567+
return result
568+
}
569+
570+
func mergeUnknownFields(updatedConfig string, unrecognizedFields map[string]interface{}) (string, error) {
571+
// Parse the updated config into a raw map
572+
var updatedConfigMap map[string]interface{}
573+
err := toml.Unmarshal([]byte(updatedConfig), &updatedConfigMap)
574+
if err != nil {
575+
return "", fmt.Errorf("failed to parse updated config: %w", err)
576+
}
577+
578+
// Merge unrecognized fields
579+
for key, value := range unrecognizedFields {
580+
if _, exists := updatedConfigMap[key]; !exists {
581+
updatedConfigMap[key] = value
582+
}
583+
}
584+
585+
// Convert back into TOML
586+
b := new(bytes.Buffer)
587+
encoder := toml.NewEncoder(b)
588+
err = encoder.Encode(updatedConfigMap)
589+
if err != nil {
590+
return "", fmt.Errorf("failed to marshal final config: %w", err)
591+
}
592+
593+
return b.String(), nil
594+
}
595+
465596
func GetDefaultConfig(comment bool) (string, error) {
466597
c := config.DefaultCurioConfig()
467598
cb, err := config.ConfigUpdate(c, nil, config.Commented(comment), config.DefaultKeepUncommented(), config.NoEnv())

‎deps/deps_test.go

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
package deps
2+
3+
import (
4+
"bytes"
5+
"testing"
6+
7+
"github.com/BurntSushi/toml"
8+
)
9+
10+
type ExampleConfig struct {
11+
Subsystems struct {
12+
EnableWindowPost bool
13+
WindowPostMaxTasks int
14+
}
15+
Fees struct {
16+
DefaultMaxFee string
17+
}
18+
}
19+
20+
// An original TOML configuration that has both recognized and unknown fields.
21+
const originalTOML = `
22+
[Subsystems]
23+
EnableWindowPost = true
24+
WindowPostMaxTasks = 5
25+
26+
[Fees]
27+
DefaultMaxFee = "0.07 FIL"
28+
29+
[UnknownSection]
30+
SomeUnknownKey = "whatever"
31+
AnotherField = 123
32+
33+
[AnotherUnknownSection.Nested]
34+
NestedValue = "I am nested"
35+
`
36+
37+
func TestExtractAndMergeUnknownFields(t *testing.T) {
38+
//----------------------------------------------------------------------
39+
// Step 1: Decode original TOML into recognized struct & collect MetaData
40+
//----------------------------------------------------------------------
41+
var recognized ExampleConfig
42+
meta, err := toml.Decode(originalTOML, &recognized)
43+
if err != nil {
44+
t.Fatalf("failed to decode recognized fields: %v", err)
45+
}
46+
47+
keys := removeUnknownEntries(meta.Keys(), meta.Undecoded())
48+
49+
//----------------------------------------------------------------------
50+
// Step 2: Extract the unknown fields using extractUnknownFields
51+
//----------------------------------------------------------------------
52+
unknownFields := extractUnknownFields(keys, originalTOML)
53+
if len(unknownFields) == 0 {
54+
t.Errorf("expected unknown fields, got none")
55+
}
56+
57+
//----------------------------------------------------------------------
58+
// Step 3: Update recognized fields in the struct
59+
//----------------------------------------------------------------------
60+
recognized.Subsystems.EnableWindowPost = false // flip the boolean
61+
recognized.Subsystems.WindowPostMaxTasks = 10 // change from 5 to 10
62+
recognized.Fees.DefaultMaxFee = "0.08 FIL" // update the fee
63+
64+
//----------------------------------------------------------------------
65+
// Step 4: Re-encode recognized fields back to TOML
66+
//----------------------------------------------------------------------
67+
var buf bytes.Buffer
68+
if err := toml.NewEncoder(&buf).Encode(recognized); err != nil {
69+
t.Fatalf("failed to marshal updated recognized config: %v", err)
70+
}
71+
updatedConfig := buf.String()
72+
73+
//----------------------------------------------------------------------
74+
// Step 5: Merge unknown fields back
75+
//----------------------------------------------------------------------
76+
finalConfig, err := mergeUnknownFields(updatedConfig, unknownFields)
77+
if err != nil {
78+
t.Fatalf("failed to merge unknown fields: %v", err)
79+
}
80+
81+
//----------------------------------------------------------------------
82+
// Assertions: Check recognized fields have changed & unknown remain
83+
//----------------------------------------------------------------------
84+
// 5a. Parse final config into a map to check contents
85+
var finalMap map[string]interface{}
86+
if err := toml.Unmarshal([]byte(finalConfig), &finalMap); err != nil {
87+
t.Fatalf("failed to parse final config: %v\nFinal Config:\n%s", err, finalConfig)
88+
}
89+
90+
// 5b. Check recognized fields updated
91+
subsystems, ok := finalMap["Subsystems"].(map[string]interface{})
92+
if !ok {
93+
t.Fatalf("expected 'Subsystems' in final config")
94+
}
95+
96+
if enable, _ := subsystems["EnableWindowPost"].(bool); enable {
97+
t.Errorf("expected Subsystems.EnableWindowPost = false, got true")
98+
}
99+
if tasks, _ := subsystems["WindowPostMaxTasks"].(int64); tasks != 10 {
100+
t.Errorf("expected Subsystems.WindowPostMaxTasks = 10, got %d", tasks)
101+
}
102+
103+
fees, ok := finalMap["Fees"].(map[string]interface{})
104+
if !ok {
105+
t.Fatalf("expected 'Fees' in final config")
106+
}
107+
if defaultFee, _ := fees["DefaultMaxFee"].(string); defaultFee != "0.08 FIL" {
108+
t.Errorf("expected Fees.DefaultMaxFee = '0.08 FIL', got '%s'", defaultFee)
109+
}
110+
111+
// 5c. Check unknown fields remain
112+
if _, exists := finalMap["UnknownSection"]; !exists {
113+
t.Errorf("expected UnknownSection to remain in final config, but not found")
114+
}
115+
if anotherUnknown, exists := finalMap["AnotherUnknownSection"]; !exists {
116+
t.Errorf("expected AnotherUnknownSection to remain in final config, but not found")
117+
} else {
118+
// Inside nested
119+
nested := anotherUnknown.(map[string]interface{})["Nested"].(map[string]interface{})
120+
if val, ok := nested["NestedValue"].(string); !ok || val != "I am nested" {
121+
t.Errorf("expected AnotherUnknownSection.Nested.NestedValue = 'I am nested', got '%v'", val)
122+
}
123+
}
124+
}

0 commit comments

Comments
 (0)