perf(query): reduce user query duration (zitadel#10037)

# Which Problems Are Solved

The resource usage to query user(s) on the database was high and
therefore could have performance impact.

# How the Problems Are Solved

Database queries involving the users and loginnames table were improved
and an index was added for user by email query.

# Additional Changes

- spellchecks
- updated apis on load tests

# additional info

needs cherry pick to v3

Author: adlerhurst

cmd/setup/58.go

@@ -0,0 +1,49 @@
+package setup
+
+import (
+	"context"
+	"database/sql"
+	"embed"
+	"fmt"
+
+	"github.com/zitadel/logging"
+
+	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/eventstore"
+)
+
+var (
+	//go:embed 58/*.sql
+	replaceLoginNames3View embed.FS
+)
+
+type ReplaceLoginNames3View struct {
+	dbClient *database.DB
+}
+
+func (mig *ReplaceLoginNames3View) Execute(ctx context.Context, _ eventstore.Event) error {
+	var exists bool
+	err := mig.dbClient.QueryRowContext(ctx, func(r *sql.Row) error {
+		return r.Scan(&exists)
+	}, "SELECT exists(SELECT 1 from information_schema.views WHERE table_schema = 'projections' AND table_name = 'login_names3')")
+
+	if err != nil || !exists {
+		return err
+	}
+
+	statements, err := readStatements(replaceLoginNames3View, "58")
+	if err != nil {
+		return err
+	}
+	for _, stmt := range statements {
+		logging.WithFields("file", stmt.file, "migration", mig.String()).Info("execute statement")
+		if _, err := mig.dbClient.ExecContext(ctx, stmt.query); err != nil {
+			return fmt.Errorf("%s %s: %w", mig.String(), stmt.file, err)
+		}
+	}
+	return nil
+}
+
+func (mig *ReplaceLoginNames3View) String() string {
+	return "58_replace_login_names3_view"
+}

cmd/setup/58/01_update_login_names3_view.sql

@@ -0,0 +1,36 @@
+CREATE OR REPLACE VIEW projections.login_names3 AS
+    SELECT
+        u.id AS user_id
+        , CASE
+            WHEN p.must_be_domain THEN CONCAT(u.user_name, '@', d.name)
+            ELSE u.user_name
+        END AS login_name
+        , COALESCE(d.is_primary, TRUE) AS is_primary
+        , u.instance_id
+    FROM
+        projections.login_names3_users AS u
+    LEFT JOIN LATERAL (
+        SELECT
+            must_be_domain
+            , is_default
+        FROM
+            projections.login_names3_policies AS p
+        WHERE
+            (
+                p.instance_id = u.instance_id
+                AND NOT p.is_default
+                AND p.resource_owner = u.resource_owner
+            ) OR (
+                p.instance_id = u.instance_id
+                AND p.is_default
+            )
+        ORDER BY
+            p.is_default -- custom first
+        LIMIT 1
+    ) AS p ON TRUE
+    LEFT JOIN 
+        projections.login_names3_domains d
+        ON 
+            p.must_be_domain 
+            AND u.resource_owner = d.resource_owner 
+            AND u.instance_id = d.instance_id

cmd/setup/58/02_create_index.sql

@@ -0,0 +1 @@
+CREATE INDEX CONCURRENTLY IF NOT EXISTS login_names3_policies_is_default_owner_idx ON projections.login_names3_policies (instance_id, is_default, resource_owner) INCLUDE (must_be_domain)

cmd/setup/config.go

@@ -154,6 +154,7 @@ type Steps struct {
 	s55ExecutionHandlerStart                *ExecutionHandlerStart
 	s56IDPTemplate6SAMLFederatedLogout      *IDPTemplate6SAMLFederatedLogout
 	s57CreateResourceCounts                 *CreateResourceCounts
+	s58ReplaceLoginNames3View               *ReplaceLoginNames3View
 }
 
 func MustNewSteps(v *viper.Viper) *Steps {

cmd/setup/setup.go

@@ -216,6 +216,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 	steps.s55ExecutionHandlerStart = &ExecutionHandlerStart{dbClient: dbClient}
 	steps.s56IDPTemplate6SAMLFederatedLogout = &IDPTemplate6SAMLFederatedLogout{dbClient: dbClient}
 	steps.s57CreateResourceCounts = &CreateResourceCounts{dbClient: dbClient}
+	steps.s58ReplaceLoginNames3View = &ReplaceLoginNames3View{dbClient: dbClient}
 
 	err = projection.Create(ctx, dbClient, eventstoreClient, config.Projections, nil, nil, nil)
 	logging.OnError(err).Fatal("unable to start projections")
@@ -262,6 +263,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 		steps.s55ExecutionHandlerStart,
 		steps.s56IDPTemplate6SAMLFederatedLogout,
 		steps.s57CreateResourceCounts,
+		steps.s58ReplaceLoginNames3View,
 	} {
 		setupErr = executeMigration(ctx, eventstoreClient, step, "migration failed")
 		if setupErr != nil {

internal/query/oidc_settings.go

@@ -84,7 +84,7 @@ func (q *Queries) OIDCSettingsByAggID(ctx context.Context, aggregateID string) (
 		OIDCSettingsColumnInstanceID.identifier():  authz.GetInstance(ctx).InstanceID(),
 	}).ToSql()
 	if err != nil {
-		return nil, zerrors.ThrowInternal(err, "QUERY-s9nle", "Errors.Query.SQLStatment")
+		return nil, zerrors.ThrowInternal(err, "QUERY-s9nle", "Errors.Query.SQLStatement")
 	}
 
 	err = q.client.QueryRowContext(ctx, func(row *sql.Row) error {

internal/query/org_metadata.go

@@ -103,7 +103,7 @@ func (q *Queries) GetOrgMetadataByKey(ctx context.Context, shouldTriggerBulk boo
 	}
 	stmt, args, err := query.Where(eq).ToSql()
 	if err != nil {
-		return nil, zerrors.ThrowInternal(err, "QUERY-aDaG2", "Errors.Query.SQLStatment")
+		return nil, zerrors.ThrowInternal(err, "QUERY-aDaG2", "Errors.Query.SQLStatement")
 	}
 
 	err = q.client.QueryRowContext(ctx, func(row *sql.Row) error {
@@ -133,7 +133,7 @@ func (q *Queries) SearchOrgMetadata(ctx context.Context, shouldTriggerBulk bool,
 	query, scan := prepareOrgMetadataListQuery()
 	stmt, args, err := queries.toQuery(query).Where(eq).ToSql()
 	if err != nil {
-		return nil, zerrors.ThrowInternal(err, "QUERY-Egbld", "Errors.Query.SQLStatment")
+		return nil, zerrors.ThrowInternal(err, "QUERY-Egbld", "Errors.Query.SQLStatement")
 	}
 
 	err = q.client.QueryContext(ctx, func(rows *sql.Rows) error {

internal/query/project.go

@@ -211,7 +211,7 @@ func (q *Queries) ProjectByID(ctx context.Context, shouldTriggerBulk bool, id st
 	}
 	query, args, err := stmt.Where(eq).ToSql()
 	if err != nil {
-		return nil, zerrors.ThrowInternal(err, "QUERY-2m00Q", "Errors.Query.SQLStatment")
+		return nil, zerrors.ThrowInternal(err, "QUERY-2m00Q", "Errors.Query.SQLStatement")
 	}
 
 	err = q.client.QueryRowContext(ctx, func(row *sql.Row) error {

internal/query/project_grant.go

@@ -167,7 +167,7 @@ func (q *Queries) ProjectGrantByID(ctx context.Context, shouldTriggerBulk bool,
 	}
 	query, args, err := stmt.Where(eq).ToSql()
 	if err != nil {
-		return nil, zerrors.ThrowInternal(err, "QUERY-Nf93d", "Errors.Query.SQLStatment")
+		return nil, zerrors.ThrowInternal(err, "QUERY-Nf93d", "Errors.Query.SQLStatement")
 	}
 
 	err = q.client.QueryRowContext(ctx, func(row *sql.Row) error {
@@ -189,7 +189,7 @@ func (q *Queries) ProjectGrantByIDAndGrantedOrg(ctx context.Context, id, granted
 	}
 	query, args, err := stmt.Where(eq).ToSql()
 	if err != nil {
-		return nil, zerrors.ThrowInternal(err, "QUERY-MO9fs", "Errors.Query.SQLStatment")
+		return nil, zerrors.ThrowInternal(err, "QUERY-MO9fs", "Errors.Query.SQLStatement")
 	}
 
 	err = q.client.QueryRowContext(ctx, func(row *sql.Row) error {

internal/query/projection/login_name.go

@@ -2,9 +2,7 @@ package projection
 
 import (
 	"context"
-	"strings"
-
-	sq "github.com/Masterminds/squirrel"
+	_ "embed"
 
 	"github.com/zitadel/zitadel/internal/eventstore"
 	old_handler "github.com/zitadel/zitadel/internal/eventstore/handler"
@@ -58,105 +56,8 @@ const (
 	LoginNamePoliciesInstanceIDCol    = "instance_id"
 )
 
-var (
-	policyUsers = sq.Select(
-		alias(
-			col(usersAlias, LoginNameUserIDCol),
-			LoginNameUserCol,
-		),
-		col(usersAlias, LoginNameUserUserNameCol),
-		col(usersAlias, LoginNameUserInstanceIDCol),
-		col(usersAlias, LoginNameUserResourceOwnerCol),
-		alias(
-			coalesce(col(policyCustomAlias, LoginNamePoliciesMustBeDomainCol), col(policyDefaultAlias, LoginNamePoliciesMustBeDomainCol)),
-			LoginNamePoliciesMustBeDomainCol,
-		),
-	).From(alias(LoginNameUserProjectionTable, usersAlias)).
-		LeftJoin(
-			leftJoin(LoginNamePolicyProjectionTable, policyCustomAlias,
-				eq(col(policyCustomAlias, LoginNamePoliciesResourceOwnerCol), col(usersAlias, LoginNameUserResourceOwnerCol)),
-				eq(col(policyCustomAlias, LoginNamePoliciesInstanceIDCol), col(usersAlias, LoginNameUserInstanceIDCol)),
-			),
-		).
-		LeftJoin(
-			leftJoin(LoginNamePolicyProjectionTable, policyDefaultAlias,
-				eq(col(policyDefaultAlias, LoginNamePoliciesIsDefaultCol), "true"),
-				eq(col(policyDefaultAlias, LoginNamePoliciesInstanceIDCol), col(usersAlias, LoginNameUserInstanceIDCol)),
-			),
-		)
-
-	loginNamesTable = sq.Select(
-		col(policyUsersAlias, LoginNameUserCol),
-		col(policyUsersAlias, LoginNameUserUserNameCol),
-		col(policyUsersAlias, LoginNameUserResourceOwnerCol),
-		alias(col(policyUsersAlias, LoginNameUserInstanceIDCol),
-			LoginNameInstanceIDCol),
-		col(policyUsersAlias, LoginNamePoliciesMustBeDomainCol),
-		alias(col(domainsAlias, LoginNameDomainNameCol),
-			domainAlias),
-		col(domainsAlias, LoginNameDomainIsPrimaryCol),
-	).FromSelect(policyUsers, policyUsersAlias).
-		LeftJoin(
-			leftJoin(LoginNameDomainProjectionTable, domainsAlias,
-				col(policyUsersAlias, LoginNamePoliciesMustBeDomainCol),
-				eq(col(policyUsersAlias, LoginNameUserResourceOwnerCol), col(domainsAlias, LoginNameDomainResourceOwnerCol)),
-				eq(col(policyUsersAlias, LoginNamePoliciesInstanceIDCol), col(domainsAlias, LoginNameDomainInstanceIDCol)),
-			),
-		)
-
-	viewStmt, _ = sq.Select(
-		LoginNameUserCol,
-		alias(
-			whenThenElse(
-				LoginNamePoliciesMustBeDomainCol,
-				concat(LoginNameUserUserNameCol, "'@'", domainAlias),
-				LoginNameUserUserNameCol),
-			LoginNameCol),
-		alias(coalesce(LoginNameDomainIsPrimaryCol, "true"),
-			LoginNameIsPrimaryCol),
-		LoginNameInstanceIDCol,
-	).FromSelect(loginNamesTable, LoginNameTableAlias).MustSql()
-)
-
-func col(table, name string) string {
-	return table + "." + name
-}
-
-func alias(col, alias string) string {
-	return col + " AS " + alias
-}
-
-func coalesce(values ...string) string {
-	str := "COALESCE("
-	for i, value := range values {
-		if i > 0 {
-			str += ", "
-		}
-		str += value
-	}
-	str += ")"
-	return str
-}
-
-func eq(first, second string) string {
-	return first + " = " + second
-}
-
-func leftJoin(table, alias, on string, and ...string) string {
-	st := table + " " + alias + " ON " + on
-	for _, a := range and {
-		st += " AND " + a
-	}
-	return st
-}
-
-func concat(strs ...string) string {
-	return "CONCAT(" + strings.Join(strs, ", ") + ")"
-}
-
-func whenThenElse(when, then, el string) string {
-	return "(CASE WHEN " + when + " THEN " + then + " ELSE " + el + " END)"
-}
+//go:embed login_name_query.sql
+var loginNameViewStmt string
 
 type loginNameProjection struct{}
 
@@ -170,7 +71,7 @@ func (*loginNameProjection) Name() string {
 
 func (*loginNameProjection) Init() *old_handler.Check {
 	return handler.NewViewCheck(
-		viewStmt,
+		loginNameViewStmt,
 		handler.NewSuffixedTable(
 			[]*handler.InitColumn{
 				handler.NewColumn(LoginNameUserIDCol, handler.ColumnTypeText),
@@ -229,7 +130,9 @@ func (*loginNameProjection) Init() *old_handler.Check {
 			},
 			handler.NewPrimaryKey(LoginNamePoliciesInstanceIDCol, LoginNamePoliciesResourceOwnerCol),
 			loginNamePolicySuffix,
-			handler.WithIndex(handler.NewIndex("is_default", []string{LoginNamePoliciesResourceOwnerCol, LoginNamePoliciesIsDefaultCol})),
+			// this index is not used anymore, but kept for understanding why the default exists on existing systems, TODO: remove in login_names4
+			// handler.WithIndex(handler.NewIndex("is_default", []string{LoginNamePoliciesResourceOwnerCol, LoginNamePoliciesIsDefaultCol})),
+			handler.WithIndex(handler.NewIndex("is_default_owner", []string{LoginNamePoliciesInstanceIDCol, LoginNamePoliciesIsDefaultCol, LoginNamePoliciesResourceOwnerCol}, handler.WithInclude(LoginNamePoliciesMustBeDomainCol))),
 		),
 	)
 }