Skip to content

Commit b7f48d7

Browse files
committed
feat: memory storage implementation
1 parent 3bad5b6 commit b7f48d7

File tree

3 files changed

+289
-17
lines changed

3 files changed

+289
-17
lines changed

account_test.go

+17-17
Original file line numberDiff line numberDiff line change
@@ -29,25 +29,25 @@ import (
2929
"go.uber.org/zap"
3030
)
3131

32-
// memoryStorage is an in-memory storage implementation with known contents *and* fixed iteration order for List.
33-
type memoryStorage struct {
34-
contents []memoryStorageItem
32+
// testingMemoryStorage is an in-memory storage implementation with known contents *and* fixed iteration order for List.
33+
type testingMemoryStorage struct {
34+
contents []testingMemoryStorageItem
3535
}
3636

37-
type memoryStorageItem struct {
37+
type testingMemoryStorageItem struct {
3838
key string
3939
data []byte
4040
}
4141

42-
func (m *memoryStorage) lookup(_ context.Context, key string) *memoryStorageItem {
42+
func (m *testingMemoryStorage) lookup(_ context.Context, key string) *testingMemoryStorageItem {
4343
for _, item := range m.contents {
4444
if item.key == key {
4545
return &item
4646
}
4747
}
4848
return nil
4949
}
50-
func (m *memoryStorage) Delete(ctx context.Context, key string) error {
50+
func (m *testingMemoryStorage) Delete(ctx context.Context, key string) error {
5151
for i, item := range m.contents {
5252
if item.key == key {
5353
m.contents = append(m.contents[:i], m.contents[i+1:]...)
@@ -56,14 +56,14 @@ func (m *memoryStorage) Delete(ctx context.Context, key string) error {
5656
}
5757
return fs.ErrNotExist
5858
}
59-
func (m *memoryStorage) Store(ctx context.Context, key string, value []byte) error {
60-
m.contents = append(m.contents, memoryStorageItem{key: key, data: value})
59+
func (m *testingMemoryStorage) Store(ctx context.Context, key string, value []byte) error {
60+
m.contents = append(m.contents, testingMemoryStorageItem{key: key, data: value})
6161
return nil
6262
}
63-
func (m *memoryStorage) Exists(ctx context.Context, key string) bool {
63+
func (m *testingMemoryStorage) Exists(ctx context.Context, key string) bool {
6464
return m.lookup(ctx, key) != nil
6565
}
66-
func (m *memoryStorage) List(ctx context.Context, path string, recursive bool) ([]string, error) {
66+
func (m *testingMemoryStorage) List(ctx context.Context, path string, recursive bool) ([]string, error) {
6767
if recursive {
6868
panic("unimplemented")
6969
}
@@ -88,22 +88,22 @@ nextitem:
8888
}
8989
return result, nil
9090
}
91-
func (m *memoryStorage) Load(ctx context.Context, key string) ([]byte, error) {
91+
func (m *testingMemoryStorage) Load(ctx context.Context, key string) ([]byte, error) {
9292
if item := m.lookup(ctx, key); item != nil {
9393
return item.data, nil
9494
}
9595
return nil, fs.ErrNotExist
9696
}
97-
func (m *memoryStorage) Stat(ctx context.Context, key string) (KeyInfo, error) {
97+
func (m *testingMemoryStorage) Stat(ctx context.Context, key string) (KeyInfo, error) {
9898
if item := m.lookup(ctx, key); item != nil {
9999
return KeyInfo{Key: key, Size: int64(len(item.data))}, nil
100100
}
101101
return KeyInfo{}, fs.ErrNotExist
102102
}
103-
func (m *memoryStorage) Lock(ctx context.Context, name string) error { panic("unimplemented") }
104-
func (m *memoryStorage) Unlock(ctx context.Context, name string) error { panic("unimplemented") }
103+
func (m *testingMemoryStorage) Lock(ctx context.Context, name string) error { panic("unimplemented") }
104+
func (m *testingMemoryStorage) Unlock(ctx context.Context, name string) error { panic("unimplemented") }
105105

106-
var _ Storage = (*memoryStorage)(nil)
106+
var _ Storage = (*testingMemoryStorage)(nil)
107107

108108
type recordingStorage struct {
109109
Storage
@@ -293,7 +293,7 @@ func TestGetAccountAlreadyExistsSkipsBroken(t *testing.T) {
293293
am := &ACMEIssuer{CA: dummyCA, Logger: zap.NewNop(), mu: new(sync.Mutex)}
294294
testConfig := &Config{
295295
Issuers: []Issuer{am},
296-
Storage: &memoryStorage{},
296+
Storage: &testingMemoryStorage{},
297297
Logger: defaultTestLogger,
298298
certCache: new(Cache),
299299
}
@@ -342,7 +342,7 @@ func TestGetAccountWithEmailAlreadyExists(t *testing.T) {
342342
am := &ACMEIssuer{CA: dummyCA, Logger: zap.NewNop(), mu: new(sync.Mutex)}
343343
testConfig := &Config{
344344
Issuers: []Issuer{am},
345-
Storage: &recordingStorage{Storage: &memoryStorage{}},
345+
Storage: &recordingStorage{Storage: &testingMemoryStorage{}},
346346
Logger: defaultTestLogger,
347347
certCache: new(Cache),
348348
}

memorystorage.go

+200
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
// Copyright 2015 Matthew Holt
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package certmagic
16+
17+
import (
18+
"context"
19+
"os"
20+
"path"
21+
"strings"
22+
"sync"
23+
"time"
24+
25+
"golang.org/x/sync/semaphore"
26+
)
27+
28+
type storageEntry struct {
29+
i KeyInfo
30+
d []byte
31+
}
32+
33+
// memoryStorage is a Storage implemention that exists only in memory
34+
// it is intended for testing and one-time-deploys where no persistence is needed
35+
type memoryStorage struct {
36+
m map[string]storageEntry
37+
mu sync.RWMutex
38+
39+
kmu *keyMutex
40+
}
41+
42+
func NewMemoryStorage() Storage {
43+
return &memoryStorage{
44+
m: map[string]storageEntry{},
45+
kmu: newKeyMutex(),
46+
}
47+
}
48+
49+
// Exists returns true if key exists in s.
50+
func (s *memoryStorage) Exists(ctx context.Context, key string) bool {
51+
ans, err := s.List(ctx, key, true)
52+
if err != nil {
53+
return false
54+
}
55+
return len(ans) != 0
56+
}
57+
58+
// Store saves value at key.
59+
func (s *memoryStorage) Store(_ context.Context, key string, value []byte) error {
60+
s.mu.Lock()
61+
defer s.mu.Unlock()
62+
s.m[key] = storageEntry{
63+
i: KeyInfo{
64+
Key: key,
65+
Modified: time.Now(),
66+
Size: int64(len(value)),
67+
IsTerminal: true,
68+
},
69+
d: value,
70+
}
71+
return nil
72+
}
73+
74+
// Load retrieves the value at key.
75+
func (s *memoryStorage) Load(_ context.Context, key string) ([]byte, error) {
76+
s.mu.Lock()
77+
defer s.mu.Unlock()
78+
val, ok := s.m[key]
79+
if !ok {
80+
return nil, os.ErrNotExist
81+
}
82+
return val.d, nil
83+
}
84+
85+
// Delete deletes the value at key.
86+
func (s *memoryStorage) Delete(_ context.Context, key string) error {
87+
s.mu.Lock()
88+
defer s.mu.Unlock()
89+
delete(s.m, key)
90+
return nil
91+
}
92+
93+
// List returns all keys that match prefix.
94+
func (s *memoryStorage) List(ctx context.Context, prefix string, recursive bool) ([]string, error) {
95+
s.mu.Lock()
96+
defer s.mu.Unlock()
97+
return s.list(ctx, prefix, recursive)
98+
}
99+
100+
func (s *memoryStorage) list(ctx context.Context, prefix string, recursive bool) ([]string, error) {
101+
var keyList []string
102+
103+
keys := make([]string, 0, len(s.m))
104+
for k := range s.m {
105+
if strings.HasPrefix(k, prefix) {
106+
keys = append(keys, k)
107+
}
108+
}
109+
// adapted from https://github.com/pberkel/caddy-storage-redis/blob/main/storage.go#L369
110+
// Iterate over each child key
111+
for _, k := range keys {
112+
// Directory keys will have a "/" suffix
113+
trimmedKey := strings.TrimSuffix(k, "/")
114+
// Reconstruct the full path of child key
115+
fullPathKey := path.Join(prefix, trimmedKey)
116+
// If current key is a directory
117+
if recursive && k != trimmedKey {
118+
// Recursively traverse all child directories
119+
childKeys, err := s.list(ctx, fullPathKey, recursive)
120+
if err != nil {
121+
return keyList, err
122+
}
123+
keyList = append(keyList, childKeys...)
124+
} else {
125+
keyList = append(keyList, fullPathKey)
126+
}
127+
}
128+
129+
return keys, nil
130+
}
131+
132+
// Stat returns information about key.
133+
func (s *memoryStorage) Stat(_ context.Context, key string) (KeyInfo, error) {
134+
s.mu.Lock()
135+
defer s.mu.Unlock()
136+
val, ok := s.m[key]
137+
if !ok {
138+
return KeyInfo{}, os.ErrNotExist
139+
}
140+
return val.i, nil
141+
}
142+
143+
// Lock obtains a lock named by the given name. It blocks
144+
// until the lock can be obtained or an error is returned.
145+
func (s *memoryStorage) Lock(ctx context.Context, name string) error {
146+
return s.kmu.LockKey(ctx, name)
147+
}
148+
149+
// Unlock releases the lock for name.
150+
func (s *memoryStorage) Unlock(_ context.Context, name string) error {
151+
return s.kmu.UnlockKey(name)
152+
}
153+
154+
func (s *memoryStorage) String() string {
155+
return "memoryStorage"
156+
}
157+
158+
// Interface guard
159+
var _ Storage = (*memoryStorage)(nil)
160+
161+
type keyMutex struct {
162+
m map[string]*semaphore.Weighted
163+
mu sync.Mutex
164+
}
165+
166+
func newKeyMutex() *keyMutex {
167+
return &keyMutex{
168+
m: map[string]*semaphore.Weighted{},
169+
}
170+
}
171+
172+
func (km *keyMutex) LockKey(ctx context.Context, id string) error {
173+
select {
174+
case <-ctx.Done():
175+
// as a special case, caddy allows for the cancelled context to be used for a trylock.
176+
if km.mutex(id).TryAcquire(1) {
177+
return nil
178+
}
179+
return ctx.Err()
180+
default:
181+
return km.mutex(id).Acquire(ctx, 1)
182+
}
183+
}
184+
185+
// Releases the lock associated with the specified ID.
186+
func (km *keyMutex) UnlockKey(id string) error {
187+
km.mutex(id).Release(1)
188+
return nil
189+
}
190+
191+
func (km *keyMutex) mutex(id string) *semaphore.Weighted {
192+
km.mu.Lock()
193+
defer km.mu.Unlock()
194+
val, ok := km.m[id]
195+
if !ok {
196+
val = semaphore.NewWeighted(1)
197+
km.m[id] = val
198+
}
199+
return val
200+
}

memorystorage_test.go

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
package certmagic_test
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"os"
7+
"testing"
8+
9+
"github.com/caddyserver/certmagic"
10+
"github.com/caddyserver/certmagic/internal/testutil"
11+
)
12+
13+
func TestMemoryStorageStoreLoad(t *testing.T) {
14+
ctx := context.Background()
15+
tmpDir, err := os.MkdirTemp(os.TempDir(), "certmagic*")
16+
testutil.RequireNoError(t, err, "allocating tmp dir")
17+
defer os.RemoveAll(tmpDir)
18+
s := certmagic.NewMemoryStorage()
19+
err = s.Store(ctx, "foo", []byte("bar"))
20+
testutil.RequireNoError(t, err)
21+
dat, err := s.Load(ctx, "foo")
22+
testutil.RequireNoError(t, err)
23+
testutil.RequireEqualValues(t, dat, []byte("bar"))
24+
}
25+
26+
func TestMemoryStorageStoreLoadRace(t *testing.T) {
27+
ctx := context.Background()
28+
tmpDir, err := os.MkdirTemp(os.TempDir(), "certmagic*")
29+
testutil.RequireNoError(t, err, "allocating tmp dir")
30+
defer os.RemoveAll(tmpDir)
31+
s := certmagic.NewMemoryStorage()
32+
a := bytes.Repeat([]byte("a"), 4096*1024)
33+
b := bytes.Repeat([]byte("b"), 4096*1024)
34+
err = s.Store(ctx, "foo", a)
35+
testutil.RequireNoError(t, err)
36+
done := make(chan struct{})
37+
go func() {
38+
err := s.Store(ctx, "foo", b)
39+
testutil.RequireNoError(t, err)
40+
close(done)
41+
}()
42+
dat, err := s.Load(ctx, "foo")
43+
<-done
44+
testutil.RequireNoError(t, err)
45+
testutil.RequireEqualValues(t, 4096*1024, len(dat))
46+
}
47+
48+
func TestMemoryStorageWriteLock(t *testing.T) {
49+
ctx := context.Background()
50+
tmpDir, err := os.MkdirTemp(os.TempDir(), "certmagic*")
51+
testutil.RequireNoError(t, err, "allocating tmp dir")
52+
defer os.RemoveAll(tmpDir)
53+
s := certmagic.NewMemoryStorage()
54+
// cctx is a cancelled ctx. so if we can't immediately get the lock, it will fail
55+
cctx, cn := context.WithCancel(ctx)
56+
cn()
57+
// should success
58+
err = s.Lock(cctx, "foo")
59+
testutil.RequireNoError(t, err)
60+
// should fail
61+
err = s.Lock(cctx, "foo")
62+
testutil.RequireError(t, err)
63+
64+
err = s.Unlock(cctx, "foo")
65+
testutil.RequireNoError(t, err)
66+
// shouldn't fail
67+
err = s.Lock(cctx, "foo")
68+
testutil.RequireNoError(t, err)
69+
70+
err = s.Unlock(cctx, "foo")
71+
testutil.RequireNoError(t, err)
72+
}

0 commit comments

Comments
 (0)