Skip to content

Commit c1abb93

Browse files
charlie-haleycharlie-haley
committed
add metadata and tags to terms and teams
1 parent 1da5210 commit c1abb93

21 files changed

Lines changed: 1087 additions & 176 deletions

File tree

‎docs/docs.go‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4412,14 +4412,11 @@ const docTemplate = `{
44124412
1000000000,
44134413
60000000000,
44144414
3600000000000,
4415-
-9223372036854775808,
4416-
9223372036854775807,
44174415
1,
44184416
1000,
44194417
1000000,
44204418
1000000000,
4421-
60000000000,
4422-
3600000000000
4419+
60000000000
44234420
],
44244421
"x-enum-varnames": [
44254422
"minDuration",
@@ -4430,14 +4427,11 @@ const docTemplate = `{
44304427
"Second",
44314428
"Minute",
44324429
"Hour",
4433-
"minDuration",
4434-
"maxDuration",
44354430
"Nanosecond",
44364431
"Microsecond",
44374432
"Millisecond",
44384433
"Second",
4439-
"Minute",
4440-
"Hour"
4434+
"Minute"
44414435
]
44424436
},
44434437
"ui.BannerResponse": {

‎docs/swagger.json‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4405,14 +4405,11 @@
44054405
1000000000,
44064406
60000000000,
44074407
3600000000000,
4408-
-9223372036854775808,
4409-
9223372036854775807,
44104408
1,
44114409
1000,
44124410
1000000,
44134411
1000000000,
4414-
60000000000,
4415-
3600000000000
4412+
60000000000
44164413
],
44174414
"x-enum-varnames": [
44184415
"minDuration",
@@ -4423,14 +4420,11 @@
44234420
"Second",
44244421
"Minute",
44254422
"Hour",
4426-
"minDuration",
4427-
"maxDuration",
44284423
"Nanosecond",
44294424
"Microsecond",
44304425
"Millisecond",
44314426
"Second",
4432-
"Minute",
4433-
"Hour"
4427+
"Minute"
44344428
]
44354429
},
44364430
"ui.BannerResponse": {

‎docs/swagger.yaml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -999,14 +999,11 @@ definitions:
999999
- 1000000000
10001000
- 60000000000
10011001
- 3600000000000
1002-
- -9223372036854775808
1003-
- 9223372036854775807
10041002
- 1
10051003
- 1000
10061004
- 1000000
10071005
- 1000000000
10081006
- 60000000000
1009-
- 3600000000000
10101007
type: integer
10111008
x-enum-varnames:
10121009
- minDuration
@@ -1017,14 +1014,11 @@ definitions:
10171014
- Second
10181015
- Minute
10191016
- Hour
1020-
- minDuration
1021-
- maxDuration
10221017
- Nanosecond
10231018
- Microsecond
10241019
- Millisecond
10251020
- Second
10261021
- Minute
1027-
- Hour
10281022
ui.BannerResponse:
10291023
properties:
10301024
dismissible:

‎internal/api/v1/search/handler.go‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,26 +8,30 @@ import (
88
"github.com/marmotdata/marmot/internal/core/auth"
99
"github.com/marmotdata/marmot/internal/core/search"
1010
"github.com/marmotdata/marmot/internal/core/user"
11+
"github.com/marmotdata/marmot/internal/metrics"
1112
)
1213

1314
type Handler struct {
14-
searchService search.Service
15-
userService user.Service
16-
authService auth.Service
17-
config *config.Config
15+
searchService search.Service
16+
userService user.Service
17+
authService auth.Service
18+
metricsService *metrics.Service
19+
config *config.Config
1820
}
1921

2022
func NewHandler(
2123
searchService search.Service,
2224
userService user.Service,
2325
authService auth.Service,
26+
metricsService *metrics.Service,
2427
config *config.Config,
2528
) *Handler {
2629
return &Handler{
27-
searchService: searchService,
28-
userService: userService,
29-
authService: authService,
30-
config: config,
30+
searchService: searchService,
31+
userService: userService,
32+
authService: authService,
33+
metricsService: metricsService,
34+
config: config,
3135
}
3236
}
3337

‎internal/api/v1/search/search.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,5 +98,17 @@ func (h *Handler) search(w http.ResponseWriter, r *http.Request) {
9898
return
9999
}
100100

101+
if query != "" && response.Total > 0 {
102+
recorder := h.metricsService.GetRecorder()
103+
queryType := "full_text"
104+
if len(filter.Types) > 0 || len(filter.AssetTypes) > 0 || len(filter.Providers) > 0 || len(filter.Tags) > 0 {
105+
queryType = "filtered"
106+
}
107+
err = recorder.RecordSearchQuery(r.Context(), queryType, query)
108+
if err != nil {
109+
log.Error().Err(err)
110+
}
111+
}
112+
101113
common.RespondJSON(w, http.StatusOK, response)
102114
}

‎internal/api/v1/server.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ func New(config *config.Config, db *pgxpool.Pool) *Server {
101101
runs.NewHandler(runsSvc, userSvc, authSvc, config),
102102
glossary.NewHandler(glossarySvc, userSvc, authSvc, config),
103103
teams.NewHandler(teamSvc, userSvc, authSvc, config),
104-
searchAPI.NewHandler(searchSvc, userSvc, authSvc, config),
104+
searchAPI.NewHandler(searchSvc, userSvc, authSvc, metricsService, config),
105105
ui.NewHandler(config),
106106
}
107107

‎internal/api/v1/teams/handler.go‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -251,21 +251,18 @@ func (h *Handler) updateTeam(w http.ResponseWriter, r *http.Request) {
251251
id := r.PathValue("id")
252252

253253
var req struct {
254-
Name string `json:"name"`
255-
Description string `json:"description"`
254+
Name *string `json:"name,omitempty"`
255+
Description *string `json:"description,omitempty"`
256+
Metadata map[string]interface{} `json:"metadata,omitempty"`
257+
Tags []string `json:"tags,omitempty"`
256258
}
257259

258260
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
259261
common.RespondError(w, http.StatusBadRequest, "Invalid request body")
260262
return
261263
}
262264

263-
if req.Name == "" {
264-
common.RespondError(w, http.StatusBadRequest, "Name is required")
265-
return
266-
}
267-
268-
err := h.teamService.UpdateTeam(r.Context(), id, req.Name, req.Description)
265+
err := h.teamService.UpdateTeamFields(r.Context(), id, req.Name, req.Description, req.Metadata, req.Tags)
269266
if err != nil {
270267
if err == team.ErrTeamNotFound {
271268
common.RespondError(w, http.StatusNotFound, "Team not found")

‎internal/core/asset/store.go‎

Lines changed: 122 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -465,6 +465,39 @@ func (r *PostgresRepository) GetMetadataFields(ctx context.Context) ([]MetadataF
465465
466466
UNION ALL
467467
468+
SELECT
469+
key as path,
470+
key as field,
471+
value,
472+
jsonb_typeof(value) as type,
473+
1 as depth,
474+
ARRAY[key] as path_parts,
475+
ARRAY[jsonb_typeof(value)] as types
476+
FROM glossary_terms,
477+
jsonb_each(metadata)
478+
WHERE metadata IS NOT NULL
479+
AND metadata != '{}'::jsonb
480+
AND jsonb_typeof(metadata) = 'object'
481+
AND deleted_at IS NULL
482+
483+
UNION ALL
484+
485+
SELECT
486+
key as path,
487+
key as field,
488+
value,
489+
jsonb_typeof(value) as type,
490+
1 as depth,
491+
ARRAY[key] as path_parts,
492+
ARRAY[jsonb_typeof(value)] as types
493+
FROM teams,
494+
jsonb_each(metadata)
495+
WHERE metadata IS NOT NULL
496+
AND metadata != '{}'::jsonb
497+
AND jsonb_typeof(metadata) = 'object'
498+
499+
UNION ALL
500+
468501
SELECT
469502
mk.path || '.' || e.key,
470503
e.key,
@@ -497,8 +530,16 @@ func (r *PostgresRepository) GetMetadataFieldsWithContext(ctx context.Context, q
497530
SELECT id FROM assets
498531
WHERE search_text @@ websearch_to_tsquery('english', $1) AND is_stub = FALSE
499532
),
533+
matching_glossary AS (
534+
SELECT id FROM glossary_terms
535+
WHERE search_text @@ websearch_to_tsquery('english', $1) AND deleted_at IS NULL
536+
),
537+
matching_teams AS (
538+
SELECT id FROM teams
539+
WHERE search_text @@ websearch_to_tsquery('english', $1)
540+
),
500541
metadata_stats AS (
501-
SELECT
542+
SELECT
502543
key as field,
503544
jsonb_typeof(value) as type,
504545
COUNT(*) as count,
@@ -508,15 +549,42 @@ func (r *PostgresRepository) GetMetadataFieldsWithContext(ctx context.Context, q
508549
jsonb_each(metadata)
509550
WHERE metadata != '{}'::jsonb
510551
GROUP BY key, jsonb_typeof(value)
552+
553+
UNION ALL
554+
555+
SELECT
556+
key as field,
557+
jsonb_typeof(value) as type,
558+
COUNT(*) as count,
559+
MODE() WITHIN GROUP (ORDER BY value) as example
560+
FROM glossary_terms g
561+
JOIN matching_glossary mg ON g.id = mg.id,
562+
jsonb_each(metadata)
563+
WHERE metadata != '{}'::jsonb
564+
GROUP BY key, jsonb_typeof(value)
565+
566+
UNION ALL
567+
568+
SELECT
569+
key as field,
570+
jsonb_typeof(value) as type,
571+
COUNT(*) as count,
572+
MODE() WITHIN GROUP (ORDER BY value) as example
573+
FROM teams t
574+
JOIN matching_teams mt ON t.id = mt.id,
575+
jsonb_each(metadata)
576+
WHERE metadata != '{}'::jsonb
577+
GROUP BY key, jsonb_typeof(value)
511578
)
512-
SELECT
579+
SELECT
513580
field,
514581
type,
515-
count,
516-
example,
582+
SUM(count)::bigint as count,
583+
(array_agg(example ORDER BY count DESC))[1] as example,
517584
ARRAY[field] as path_parts,
518585
ARRAY[type] as types
519586
FROM metadata_stats
587+
GROUP BY field, type
520588
ORDER BY count DESC, field ASC`
521589

522590
return r.scanMetadataFields(ctx, query, queryContext.Query)
@@ -615,16 +683,30 @@ func (r *PostgresRepository) GetMetadataValues(ctx context.Context, field string
615683
return r.scanMetadataValues(ctx, query, prefix, limit)
616684

617685
default:
618-
// Regular metadata field query
619686
pathArray := strings.Split(field, ".")
620687
query := `
621-
WITH RECURSIVE MetadataValues AS (
688+
WITH MetadataValues AS (
622689
SELECT
623-
a.id,
624-
jsonb_extract_path(a.metadata, VARIADIC $1::text[]) as value,
625-
1 as level
690+
a.id::text,
691+
jsonb_extract_path(a.metadata, VARIADIC $1::text[]) as value
626692
FROM assets a
627693
WHERE jsonb_typeof(jsonb_extract_path(a.metadata, VARIADIC $1::text[])) != 'null' AND is_stub = FALSE
694+
695+
UNION ALL
696+
697+
SELECT
698+
g.id::text,
699+
jsonb_extract_path(g.metadata, VARIADIC $1::text[]) as value
700+
FROM glossary_terms g
701+
WHERE jsonb_typeof(jsonb_extract_path(g.metadata, VARIADIC $1::text[])) != 'null' AND deleted_at IS NULL
702+
703+
UNION ALL
704+
705+
SELECT
706+
t.id::text,
707+
jsonb_extract_path(t.metadata, VARIADIC $1::text[]) as value
708+
FROM teams t
709+
WHERE jsonb_typeof(jsonb_extract_path(t.metadata, VARIADIC $1::text[])) != 'null'
628710
)
629711
SELECT
630712
value::text,
@@ -725,21 +807,50 @@ func (r *PostgresRepository) GetMetadataValuesWithContext(ctx context.Context, f
725807
return r.scanMetadataValues(ctx, query, queryContext.Query, prefix, limit)
726808

727809
default:
728-
// Regular metadata field query
729810
query := `
730811
WITH matching_assets AS (
731812
SELECT id FROM assets
732813
WHERE search_text @@ websearch_to_tsquery('english', $1) AND is_stub = FALSE
733814
),
815+
matching_glossary AS (
816+
SELECT id FROM glossary_terms
817+
WHERE search_text @@ websearch_to_tsquery('english', $1) AND deleted_at IS NULL
818+
),
819+
matching_teams AS (
820+
SELECT id FROM teams
821+
WHERE search_text @@ websearch_to_tsquery('english', $1)
822+
),
734823
MetadataValues AS (
735824
SELECT
736-
a.id,
825+
a.id::text,
737826
je.key,
738827
je.value
739828
FROM assets a
740829
JOIN matching_assets ma ON a.id = ma.id
741830
CROSS JOIN LATERAL jsonb_each(a.metadata) AS je
742831
WHERE a.metadata IS NOT NULL
832+
833+
UNION ALL
834+
835+
SELECT
836+
g.id::text,
837+
je.key,
838+
je.value
839+
FROM glossary_terms g
840+
JOIN matching_glossary mg ON g.id = mg.id
841+
CROSS JOIN LATERAL jsonb_each(g.metadata) AS je
842+
WHERE g.metadata IS NOT NULL
843+
844+
UNION ALL
845+
846+
SELECT
847+
t.id::text,
848+
je.key,
849+
je.value
850+
FROM teams t
851+
JOIN matching_teams mt ON t.id = mt.id
852+
CROSS JOIN LATERAL jsonb_each(t.metadata) AS je
853+
WHERE t.metadata IS NOT NULL
743854
)
744855
SELECT
745856
value::text,

0 commit comments

Comments
 (0)