Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
77e6c5e
Update dependencies
carabasdaniel Apr 10, 2025
7b179a1
Fix sdk linting issues
carabasdaniel Apr 11, 2025
bd76d0e
Fix linting for openapi, okta and ldap plugins
carabasdaniel Apr 11, 2025
fbfa2de
Fix linting in jumpcloud, google and fusionauth plugins
carabasdaniel Apr 14, 2025
2336137
Fix linting for azuread, azuredb2c and cognito plugins
carabasdaniel Apr 14, 2025
27587c4
Fix auth0 and cli linting errors
carabasdaniel Apr 14, 2025
8ba9f1c
Update ci go linter and go releaser versions
carabasdaniel Apr 14, 2025
11440b6
Use linting rules from topaz and update formatting with gofumpt
carabasdaniel Apr 15, 2025
fdba13a
Update error messages and address review comments
carabasdaniel Apr 15, 2025
7c4eece
Merge commit 'f46d21fedde94c6732608fab3845d95f60d09e77' into updates
carabasdaniel Apr 15, 2025
1363ce7
Update cognito fetch groups func
carabasdaniel Apr 15, 2025
f20611d
Update azuread plugin fetcher
carabasdaniel Apr 16, 2025
b3320c7
Update azureadb2c fetcher
carabasdaniel Apr 16, 2025
ba5768e
Update google and azure plugin fetchers
carabasdaniel Apr 17, 2025
cd92aed
Update auth0 plugin fetcher and httpclient
carabasdaniel Apr 22, 2025
9e30c50
Add fetcher helpers to sdk
carabasdaniel Apr 23, 2025
4ac7be3
Remove BASE_DIR from makefile
carabasdaniel Apr 23, 2025
69c5807
Address review comments
carabasdaniel Apr 24, 2025
0170d18
Fix generic fetcher map yield helper
carabasdaniel Apr 24, 2025
ebcb245
Create a custom error writer in sdk
carabasdaniel Apr 25, 2025
ea1a067
Update error writer
carabasdaniel Apr 29, 2025
670337f
ErrorWriter embeds an io.Writer
ronenh Apr 30, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion plugins/auth0/pkg/app/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/aserto-dev/ds-load/plugins/auth0/pkg/auth0client"
"github.com/aserto-dev/ds-load/plugins/auth0/pkg/fetch"
"github.com/aserto-dev/ds-load/sdk/common"
"github.com/aserto-dev/ds-load/sdk/common/cc"
)

Expand Down Expand Up @@ -45,5 +46,5 @@ func (f *FetchCmd) Run(ctx *cc.CommonCtx) error {
WithSAML(f.SAML).
WithConnectionName(f.ConnectionName)

return fetcher.Fetch(ctx.Context, os.Stdout, os.Stderr)
return fetcher.Fetch(ctx.Context, os.Stdout, common.ErrorWriter{Writer: os.Stderr})
}
16 changes: 7 additions & 9 deletions plugins/auth0/pkg/fetch/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ func (f *Fetcher) WithConnectionName(connectionName string) *Fetcher {
return f
}

func (f *Fetcher) Fetch(ctx context.Context, outputWriter, errorWriter io.Writer) error {
func (f *Fetcher) Fetch(ctx context.Context, outputWriter io.Writer, errorWriter common.ErrorWriter) error {
writer := js.NewJSONArrayWriter(outputWriter)
defer writer.Close()

Expand All @@ -78,7 +78,7 @@ func (f *Fetcher) Fetch(ctx context.Context, outputWriter, errorWriter io.Writer
return f.fetchUsers(ctx, writer, errorWriter)
}

func (f *Fetcher) fetchUsers(ctx context.Context, outputWriter *js.JSONArrayWriter, errorWriter io.Writer) error {
func (f *Fetcher) fetchUsers(ctx context.Context, outputWriter *js.JSONArrayWriter, errorWriter common.ErrorWriter) error {
page := 0

for {
Expand All @@ -89,15 +89,13 @@ func (f *Fetcher) fetchUsers(ctx context.Context, outputWriter *js.JSONArrayWrit

users, more, err := f.fetchUsersList(ctx, opts)
if err != nil {
common.WriteErrorWithExitCode(errorWriter, err, 1)
errorWriter.Error(err)
return err
}

for _, user := range users {
obj, err := f.userToMap(ctx, user)
if err != nil {
common.WriteErrorWithExitCode(errorWriter, err, 1)
}
errorWriter.Error(err)

if obj == nil {
continue
Expand Down Expand Up @@ -156,7 +154,7 @@ func (f *Fetcher) userToMap(ctx context.Context, user *management.User) (map[str
return obj, nil
}

func (f *Fetcher) fetchGroups(ctx context.Context, outputWriter *js.JSONArrayWriter, errorWriter io.Writer) error {
func (f *Fetcher) fetchGroups(ctx context.Context, outputWriter *js.JSONArrayWriter, errorWriter common.ErrorWriter) error {
page := 0

for f.Roles {
Expand All @@ -168,7 +166,7 @@ func (f *Fetcher) fetchGroups(ctx context.Context, outputWriter *js.JSONArrayWri

roles, more, err := f.fetchRoles(ctx, opts)
if err != nil {
common.WriteErrorWithExitCode(errorWriter, err, 1)
errorWriter.Error(err)
return err
}

Expand All @@ -178,7 +176,7 @@ func (f *Fetcher) fetchGroups(ctx context.Context, outputWriter *js.JSONArrayWri
var obj map[string]any

if err := json.Unmarshal([]byte(res), &obj); err != nil {
common.WriteErrorWithExitCode(errorWriter, err, 1)
errorWriter.Error(err)
continue
}

Expand Down
3 changes: 2 additions & 1 deletion plugins/azuread/pkg/app/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"os"

"github.com/aserto-dev/ds-load/plugins/azuread/pkg/fetch"
"github.com/aserto-dev/ds-load/sdk/common"
"github.com/aserto-dev/ds-load/sdk/common/cc"
)

Expand All @@ -29,5 +30,5 @@ func (cmd *FetchCmd) Run(ctx *cc.CommonCtx) error {
return err
}

return fetcher.WithGroups(cmd.Groups).Fetch(ctx.Context, os.Stdout, os.Stderr)
return fetcher.WithGroups(cmd.Groups).Fetch(ctx.Context, os.Stdout, common.ErrorWriter{Writer: os.Stderr})
}
20 changes: 7 additions & 13 deletions plugins/azuread/pkg/fetch/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,30 +33,24 @@ func (f *Fetcher) WithGroups(groups bool) *Fetcher {
return f
}

func (f *Fetcher) Fetch(ctx context.Context, outputWriter, errorWriter io.Writer) error {
func (f *Fetcher) Fetch(ctx context.Context, outputWriter io.Writer, errorWriter common.ErrorWriter) error {
jsonWriter := js.NewJSONArrayWriter(outputWriter)
defer jsonWriter.Close()

if f.Groups {
for obj, err := range f.fetchGroups(ctx) {
if err != nil {
common.WriteErrorWithExitCode(errorWriter, err, 1)
}
errorWriter.Error(err)

if err := jsonWriter.Write(obj); err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
}
err := jsonWriter.Write(obj)
errorWriter.ErrorNoExitCode(err)
}
}

for user, err := range f.fetchUsers(ctx) {
if err != nil {
common.WriteErrorWithExitCode(errorWriter, err, 1)
}
errorWriter.Error(err)

if err := jsonWriter.Write(user); err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
}
err := jsonWriter.Write(user)
errorWriter.ErrorNoExitCode(err)
}

return nil
Expand Down
3 changes: 2 additions & 1 deletion plugins/azureadb2c/pkg/app/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"os"

"github.com/aserto-dev/ds-load/plugins/azureadb2c/pkg/fetch"
"github.com/aserto-dev/ds-load/sdk/common"
"github.com/aserto-dev/ds-load/sdk/common/cc"
)

Expand All @@ -29,5 +30,5 @@ func (cmd *FetchCmd) Run(ctx *cc.CommonCtx) error {
return err
}

return fetcher.WithGroups(cmd.Groups).Fetch(ctx.Context, os.Stdout, os.Stderr)
return fetcher.WithGroups(cmd.Groups).Fetch(ctx.Context, os.Stdout, common.ErrorWriter{Writer: os.Stderr})
}
20 changes: 7 additions & 13 deletions plugins/azureadb2c/pkg/fetch/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,30 +33,24 @@ func (f *Fetcher) WithGroups(groups bool) *Fetcher {
return f
}

func (f *Fetcher) Fetch(ctx context.Context, outputWriter, errorWriter io.Writer) error {
func (f *Fetcher) Fetch(ctx context.Context, outputWriter io.Writer, errorWriter common.ErrorWriter) error {
jsonWriter := js.NewJSONArrayWriter(outputWriter)
defer jsonWriter.Close()

if f.Groups {
for obj, err := range f.fetchGroups(ctx) {
if err != nil {
common.WriteErrorWithExitCode(errorWriter, err, 1)
}
errorWriter.Error(err)

if err := jsonWriter.Write(obj); err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
}
err := jsonWriter.Write(obj)
errorWriter.ErrorNoExitCode(err)
}
}

for user, err := range f.fetchUsers(ctx) {
if err != nil {
common.WriteErrorWithExitCode(errorWriter, err, 1)
}
errorWriter.Error(err)

if err := jsonWriter.Write(user); err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
}
err := jsonWriter.Write(user)
errorWriter.ErrorNoExitCode(err)
}

return nil
Expand Down
3 changes: 2 additions & 1 deletion plugins/cognito/pkg/app/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

"github.com/aserto-dev/ds-load/plugins/cognito/pkg/cognitoclient"
"github.com/aserto-dev/ds-load/plugins/cognito/pkg/fetch"
"github.com/aserto-dev/ds-load/sdk/common"
"github.com/aserto-dev/ds-load/sdk/common/cc"
)

Expand All @@ -29,5 +30,5 @@ func (cmd *FetchCmd) Run(ctx *cc.CommonCtx) error {

fetcher = fetcher.WithGroups(cmd.Groups)

return fetcher.Fetch(ctx.Context, os.Stdout, os.Stderr)
return fetcher.Fetch(ctx.Context, os.Stdout, common.ErrorWriter{Writer: os.Stderr})
}
36 changes: 14 additions & 22 deletions plugins/cognito/pkg/fetch/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,26 +29,21 @@ func (f *Fetcher) WithGroups(groups bool) *Fetcher {
return f
}

func (f *Fetcher) Fetch(ctx context.Context, outputWriter, errorWriter io.Writer) error {
func (f *Fetcher) Fetch(ctx context.Context, outputWriter io.Writer, errorWriter common.ErrorWriter) error {
writer := js.NewJSONArrayWriter(outputWriter)
defer writer.Close()

if f.groups {
for obj, err := range f.fetchGroups() {
if err != nil {
common.WriteErrorWithExitCode(errorWriter, err, 1)
}
errorWriter.Error(err)

if err := writer.Write(obj); err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
}
err := writer.Write(obj)
errorWriter.ErrorNoExitCode(err)
}
}

users, err := f.cognitoClient.ListUsers(ctx)
if err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
}
errorWriter.ErrorNoExitCode(err)

for _, user := range users {
attributes := make(map[string]string)
Expand All @@ -58,41 +53,38 @@ func (f *Fetcher) Fetch(ctx context.Context, outputWriter, errorWriter io.Writer

userBytes, err := json.Marshal(user)
if err != nil {
common.WriteErrorWithExitCode(errorWriter, err, 1)
errorWriter.Error(err)
return err
}

var obj map[string]any
if err := json.Unmarshal(userBytes, &obj); err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
}
err = json.Unmarshal(userBytes, &obj)
errorWriter.ErrorNoExitCode(err)

obj["Attributes"] = attributes

if f.groups {
groups, err := f.cognitoClient.GetGroupsForUser(*user.Username)
if err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
errorWriter.ErrorNoExitCode(err)
continue
}

groupBytes, err := json.Marshal(groups.Groups)
if err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
errorWriter.ErrorNoExitCode(err)
return err
}

var grps []map[string]string
if err := json.Unmarshal(groupBytes, &grps); err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
}
err = json.Unmarshal(groupBytes, &grps)
errorWriter.ErrorNoExitCode(err)

obj["Groups"] = grps
}

if err := writer.Write(obj); err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
}
err = writer.Write(obj)
errorWriter.ErrorNoExitCode(err)
}

return nil
Expand Down
3 changes: 2 additions & 1 deletion plugins/fusionauth/pkg/app/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

"github.com/aserto-dev/ds-load/plugins/fusionauth/pkg/fetch"
"github.com/aserto-dev/ds-load/plugins/fusionauth/pkg/fusionauthclient"
"github.com/aserto-dev/ds-load/sdk/common"
"github.com/aserto-dev/ds-load/sdk/common/cc"
)

Expand All @@ -27,5 +28,5 @@ func (cmd *FetchCmd) Run(ctx *cc.CommonCtx) error {

fetcher = fetcher.WithGroups(cmd.Groups).WithHost(cmd.HostURL)

return fetcher.Fetch(ctx.Context, os.Stdout, os.Stderr)
return fetcher.Fetch(ctx.Context, os.Stdout, common.ErrorWriter{Writer: os.Stderr})
}
25 changes: 10 additions & 15 deletions plugins/fusionauth/pkg/fetch/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"io"

"github.com/aserto-dev/ds-load/plugins/fusionauth/pkg/fusionauthclient"
"github.com/aserto-dev/ds-load/sdk/common"
"github.com/aserto-dev/ds-load/sdk/common/js"
)

Expand All @@ -32,50 +33,44 @@ func (f *Fetcher) WithHost(host string) *Fetcher {
return f
}

func (f *Fetcher) Fetch(ctx context.Context, outputWriter, errorWriter io.Writer) error {
func (f *Fetcher) Fetch(ctx context.Context, outputWriter io.Writer, errorWriter common.ErrorWriter) error {
writer := js.NewJSONArrayWriter(outputWriter)
defer writer.Close()

users, err := f.fusionauthClient.ListUsers(ctx)
if err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
}
errorWriter.ErrorNoExitCode(err)

for i := range users {
user := &users[i]

userBytes, err := json.Marshal(user)
if err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
errorWriter.ErrorNoExitCode(err)
return err
}

var obj map[string]any
if err := json.Unmarshal(userBytes, &obj); err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
errorWriter.ErrorNoExitCode(err)
return err
}

if user.ImageUrl != "" {
obj["picture"] = fmt.Sprintf("%s%s", f.host, user.ImageUrl)
}

if err := writer.Write(obj); err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
}
err = writer.Write(obj)
errorWriter.ErrorNoExitCode(err)
}

if f.groups {
groups, err := f.fusionauthClient.ListGroups(ctx)
if err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
}
errorWriter.ErrorNoExitCode(err)

for i := range groups {
group := &groups[i]
if err := writer.Write(group); err != nil {
_, _ = errorWriter.Write([]byte(err.Error()))
}
err := writer.Write(group)
errorWriter.ErrorNoExitCode(err)
}
}

Expand Down
3 changes: 2 additions & 1 deletion plugins/google/pkg/app/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

"github.com/aserto-dev/ds-load/plugins/google/pkg/fetch"
"github.com/aserto-dev/ds-load/plugins/google/pkg/googleclient"
"github.com/aserto-dev/ds-load/sdk/common"
"github.com/aserto-dev/ds-load/sdk/common/cc"
)

Expand All @@ -29,5 +30,5 @@ func (cmd *FetchCmd) Run(ctx *cc.CommonCtx) error {

fetcher = fetcher.WithGroups(cmd.Groups)

return fetcher.Fetch(ctx.Context, os.Stdout, os.Stderr)
return fetcher.Fetch(ctx.Context, os.Stdout, common.ErrorWriter{Writer: os.Stderr})
}
Loading
Loading