Skip to content

Commit dcce9ba

Browse files
authored
feat: Serverless client side schema generation (#4263)
## Relevant issue(s) Resolves #2982 ## Description Adds a new `sdl` command which contains a `generate` subcommand. This command takes as input one or more user defined schema types and outputs the fully generated defradb schema definition (SDL). **Question**: I opted to include `include-searchable-encryption` as an user argument, since the `SchemManager.Generate` takes it as a parameter. We could just hardcode this to true, and it always output schema that supports searchable encryption types (even though the actual user type might not have any `@encryptedIndex` defined) ## How has this been tested? Added embedded CLI tests, along with manual testing. Happy to add more if there's a preference. Specify the platform(s) on which this was tested: - Ubuntu (WSL2)
1 parent b7abc99 commit dcce9ba

25 files changed

Lines changed: 9925 additions & 148 deletions
Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@ on:
2424
- develop
2525

2626
jobs:
27-
# This job runs the JS tests to ensure they are able to build and run.
27+
# This job runs the tests that depend on npx to ensure they are able to build and run.
2828
test-js:
29-
name: Test JS build job
29+
name: Test NPX/JS build job
3030
runs-on: ubuntu-latest
3131

3232
steps:
@@ -36,5 +36,5 @@ jobs:
3636
- name: Setup defradb
3737
uses: ./.github/composites/setup-defradb
3838

39-
- name: Test Introspection JS Client
40-
run: make test:introspectionjs
39+
- name: Run NPX/JS dependent tests
40+
run: make test:npx

Makefile

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -377,9 +377,20 @@ test\:changes:
377377
test\:js:
378378
GOOS=js GOARCH=wasm gotestsum --format testname -- $(JS_TEST_DIRS) $(JS_TEST_FLAGS)
379379

380-
.PHONY: test\:introspectionjs
381-
test\:introspectionjs:
382-
go test -tags nodejs -run ^TestIntrospectionResult$$ ./internal/request/graphql/schema
380+
# This test scans all the test files to find ones that include the npx build tag
381+
# then runs only those tests and their respective packages
382+
# Note: simply include `-tags npx` isnt sufficient since go test still includes
383+
# all the other tests and packages that arent tagged.
384+
.PHONY: test\:npx
385+
test\:npx:
386+
@npx_files=$$(grep -rl --include='*_test.go' -E '^//go:build.*\bnpx\b|^// \+build.*\bnpx\b' .); \
387+
if [ -z "$$npx_files" ]; then \
388+
echo "No npx-tagged tests found"; \
389+
exit 0; \
390+
fi; \
391+
packages=$$(echo "$$npx_files" | xargs -n1 dirname | sort -u | sed 's|^\./||' | sed 's|^|./|'); \
392+
test_pattern=$$(echo "$$npx_files" | xargs grep -h -E '^func (Test[A-Za-z0-9_]+)' | sed -E 's/^func (Test[A-Za-z0-9_]+).*/\1/' | paste -sd '|' -); \
393+
echo "$$packages" | xargs gotestsum --format pkgname -- -tags=npx -run "^($$test_pattern)$$"
383394

384395
.PHONY: validate\:codecov
385396
validate\:codecov:

cli/cli.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,11 +199,17 @@ func NewDefraCommand(ctx context.Context) *cobra.Command {
199199
MakeIdentityNewCommand(ctx),
200200
)
201201

202+
sdl := MakeSDLCommand(ctx)
203+
sdl.AddCommand(
204+
MakeSDLGenerateCommand(ctx),
205+
)
206+
202207
root := MakeRootCommand(ctx)
203208
root.AddCommand(
204209
client,
205210
keyring,
206211
identity,
212+
sdl,
207213
MakeStartCommand(ctx),
208214
MakeServerDumpCmd(),
209215
MakeVersionCommand(ctx),

cli/errors.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ var (
3838
ErrMissingKeyringSecret = errors.New("missing keyring secret")
3939
ErrEmptySchemaString = errors.New(errEmptySchemaString)
4040
ErrNegativeReplicatorRetryIntervals = errors.New("replicator retry intervals must only contain positive integers")
41+
ErrStdinSingleInputOnly = errors.New("stdin only allowed as single input")
42+
ErrReadingInput = errors.New("reading input")
43+
ErrParsingSDL = errors.New("parsing SDL")
44+
ErrGeneratingSDL = errors.New("generating SDL")
4145
ErrPurgeForceFlagRequired = errors.New("run this command again with --force if you " +
4246
"really want to purge all data")
4347
)

cli/sdl.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
// Copyright 2025 Democratized Data Foundation
2+
//
3+
// Use of this software is governed by the Business Source License
4+
// included in the file licenses/BSL.txt.
5+
//
6+
// As of the Change Date specified in that file, in accordance with
7+
// the Business Source License, use of this software will be governed
8+
// by the Apache License, Version 2.0, included in the file
9+
// licenses/APL.txt.
10+
11+
package cli
12+
13+
import (
14+
"context"
15+
16+
"github.com/spf13/cobra"
17+
)
18+
19+
func MakeSDLCommand(ctx context.Context) *cobra.Command {
20+
var cmd = &cobra.Command{
21+
Use: "sdl",
22+
Short: "Utilities to interact with the DefraDB SDL",
23+
Long: `Utilities to interact with the DefraDB Schema Definition Language.`,
24+
}
25+
26+
return cmd
27+
}

cli/sdl_generate.go

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
// Copyright 2025 Democratized Data Foundation
2+
//
3+
// Use of this software is governed by the Business Source License
4+
// included in the file licenses/BSL.txt.
5+
//
6+
// As of the Change Date specified in that file, in accordance with
7+
// the Business Source License, use of this software will be governed
8+
// by the Apache License, Version 2.0, included in the file
9+
// licenses/APL.txt.
10+
11+
package cli
12+
13+
import (
14+
"context"
15+
"io"
16+
"os"
17+
"strings"
18+
19+
"github.com/spf13/cobra"
20+
21+
"github.com/sourcenetwork/defradb/client"
22+
"github.com/sourcenetwork/defradb/errors"
23+
"github.com/sourcenetwork/defradb/internal/db/description"
24+
"github.com/sourcenetwork/defradb/internal/request/graphql/schema"
25+
)
26+
27+
var (
28+
defaultOutputPath = "schema.gen.graphql"
29+
fileLineSeperator = "\n\n"
30+
)
31+
32+
func MakeSDLGenerateCommand(ctx context.Context) *cobra.Command {
33+
var outputFile string
34+
var yesOverwrite bool
35+
var searchableEncryption bool
36+
var cmd = &cobra.Command{
37+
Use: "generate --output schema.graphql <input schema files...>",
38+
Short: "Generate full GraphQL formatted schema.",
39+
Long: `Generates the fully formatted GraphQL schema from a given user type definition(s).
40+
41+
Accepts multiple input files as well as "-" to use stdin.
42+
`,
43+
Args: cobra.MinimumNArgs(1),
44+
RunE: func(cmd *cobra.Command, args []string) error {
45+
var sdlBuf string
46+
47+
// Either we use stdin or we concat all the file
48+
// arguments
49+
if len(args) == 1 && args[0] == "-" {
50+
sdlByteBuf, err := io.ReadAll(cmd.InOrStdin())
51+
if err != nil {
52+
return err
53+
}
54+
sdlBuf = string(sdlByteBuf)
55+
} else {
56+
var fileInputBuf strings.Builder
57+
for i, arg := range args {
58+
if arg == "-" {
59+
return ErrStdinSingleInputOnly
60+
}
61+
fileBuf, err := os.ReadFile(arg)
62+
if err != nil {
63+
return err
64+
}
65+
66+
if i != 0 {
67+
fileInputBuf.WriteString(fileLineSeperator)
68+
}
69+
fileInputBuf.Write(fileBuf)
70+
}
71+
sdlBuf = fileInputBuf.String()
72+
}
73+
74+
var outWriter io.Writer
75+
if outputFile == "-" {
76+
outWriter = cmd.OutOrStdout()
77+
} else {
78+
// check if the file exists, if so check for the overwrite
79+
// flag
80+
ofinfo, err := os.Stat(outputFile)
81+
if err != nil && !errors.Is(err, os.ErrNotExist) {
82+
return err
83+
}
84+
if ofinfo != nil && !yesOverwrite {
85+
return errors.New("output file path already exists. If you want to overwrite use -y")
86+
}
87+
88+
f, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
89+
if err != nil {
90+
return err
91+
}
92+
defer f.Close() //nolint:errcheck
93+
outWriter = f
94+
}
95+
96+
schemaManager, err := schema.NewSchemaManager(searchableEncryption)
97+
if err != nil {
98+
return err
99+
}
100+
101+
cols, err := schemaManager.ParseSDL(sdlBuf)
102+
if err != nil {
103+
return errors.Join(ErrParsingSDL, err)
104+
}
105+
106+
collections := make([]client.CollectionVersion, len(cols))
107+
for i, c := range cols {
108+
collections[i] = c.Definition
109+
}
110+
111+
cache := description.NewCollectionCache()
112+
cache.AddAll(collections)
113+
ctx := description.ContextWithCollectionCache(ctx, cache)
114+
115+
_, err = schemaManager.Generator.Generate(ctx, collections)
116+
if err != nil {
117+
return errors.Join(ErrGeneratingSDL, err)
118+
}
119+
120+
return schemaManager.WriteSDL(outWriter)
121+
},
122+
}
123+
124+
EmbedCLIExample(ctx, cmd, "Generate SDL",
125+
`defradb sdl generate foo.graphql`)
126+
127+
EmbedCLIExample(ctx, cmd, "Generate Multiple SDLs",
128+
`defradb sdl generate foo.graphql bar.graphql`)
129+
130+
EmbedCLIExample(ctx, cmd, "Generate SDL and overwrite output",
131+
`defradb sdl generate foo.graphql bar.graphql --output schema.graphql -y`)
132+
133+
cmd.PersistentFlags().StringVarP(&outputFile, "output", "o", defaultOutputPath,
134+
"The output file to write the generated schema. Accepts '-' to write to stdout")
135+
136+
EmbedCLIExample(ctx, cmd, "Generate SDL with Searchable Encryption type definitions",
137+
`defradb sdl generate foo.graphql -s`)
138+
139+
cmd.PersistentFlags().BoolVarP(&yesOverwrite, "overwrite", "y", false,
140+
"Overwrite any existing matching output file paths")
141+
142+
cmd.PersistentFlags().BoolVarP(&searchableEncryption, "include-searchable-encryption", "s",
143+
false, "Include the schema type definitions to support Searchable Encryption")
144+
145+
return cmd
146+
}

0 commit comments

Comments
 (0)