PMM-15191 Reject a Change*Agent request for the wrong agent type before it commits. #5703
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,7 +18,6 @@ package inventory | |
|
|
||
| import ( | ||
| "context" | ||
| "strings" | ||
|
|
||
| "github.com/AlekSi/pointer" | ||
|
|
@@ -234,7 +233,7 @@ func (as *AgentsService) ChangeNodeExporter(ctx context.Context, agentID string, | |
| MetricsResolutions: convertMetricsResolutions(p.MetricsResolutions), | ||
| } | ||
|
|
||
| agent, err := as.executeAgentChange(ctx, agentID, models.NodeExporterType, params) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -334,7 +333,7 @@ func (as *AgentsService) ChangeMySQLdExporter(ctx context.Context, agentID strin | |
| ConnectionTimeout: duration.OptionalFromProto(p.ConnectionTimeout), | ||
| } | ||
|
|
||
| agent, err := as.executeAgentChange(ctx, agentID, models.MySQLdExporterType, params) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -438,7 +437,7 @@ func (as *AgentsService) ChangeMongoDBExporter( | |
| ConnectionTimeout: duration.OptionalFromProto(p.ConnectionTimeout), | ||
| } | ||
|
|
||
| agent, err := as.executeAgentChange(ctx, agentID, models.MongoDBExporterType, params) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -541,7 +540,7 @@ func (as *AgentsService) ChangeQANMySQLPerfSchemaAgent( | |
| MetricsResolutions: convertMetricsResolutions(p.MetricsResolutions), | ||
| } | ||
|
|
||
| agent, err := as.executeAgentChange(ctx, agentID, models.QANMySQLPerfSchemaAgentType, params) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -647,7 +646,7 @@ func (as *AgentsService) ChangeQANMySQLSlowlogAgent( | |
| MetricsResolutions: convertMetricsResolutions(p.MetricsResolutions), | ||
| } | ||
|
|
||
| agent, err := as.executeAgentChange(ctx, agentID, models.QANMySQLSlowlogAgentType, params) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -741,7 +740,7 @@ func (as *AgentsService) ChangePostgresExporter( | |
| ConnectionTimeout: duration.OptionalFromProto(p.ConnectionTimeout), | ||
| } | ||
|
|
||
| agent, err := as.executeAgentChange(ctx, agentID, models.PostgresExporterType, params) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -832,7 +831,7 @@ func (as *AgentsService) ChangeValkeyExporter(ctx context.Context, agentID strin | |
| ConnectionTimeout: duration.OptionalFromProto(p.ConnectionTimeout), | ||
| } | ||
|
|
||
| agent, err := as.executeAgentChange(ctx, agentID, models.ValkeyExporterType, params) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -929,7 +928,7 @@ func (as *AgentsService) ChangeQANMongoDBProfilerAgent( | |
| MetricsResolutions: convertMetricsResolutions(p.MetricsResolutions), | ||
| } | ||
|
|
||
| agent, err := as.executeAgentChange(ctx, agentID, models.QANMongoDBProfilerAgentType, params) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -1026,7 +1025,7 @@ func (as *AgentsService) ChangeQANMongoDBMongologAgent( | |
| MetricsResolutions: convertMetricsResolutions(p.MetricsResolutions), | ||
| } | ||
|
|
||
| agent, err := as.executeAgentChange(ctx, agentID, models.QANMongoDBMongologAgentType, params) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -1113,7 +1112,7 @@ func (as *AgentsService) ChangeProxySQLExporter( | |
| ConnectionTimeout: duration.OptionalFromProto(p.ConnectionTimeout), | ||
| } | ||
|
|
||
| agent, err := as.executeAgentChange(ctx, agentID, models.ProxySQLExporterType, params) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -1210,25 +1209,10 @@ func (as *AgentsService) ChangeQANPostgreSQLPgStatementsAgent( | |
| MetricsResolutions: convertMetricsResolutions(p.MetricsResolutions), | ||
| } | ||
|
|
||
| agent, err := as.executeAgentChange(ctx, agentID, models.QANPostgreSQLPgStatementsAgentType, params) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| pgStatementsAgent, ok := agent.(*inventoryv1.QANPostgreSQLPgStatementsAgent) | ||
| if !ok { | ||
|
|
@@ -1324,7 +1308,7 @@ func (as *AgentsService) ChangeQANPostgreSQLPgStatMonitorAgent( | |
| MetricsResolutions: convertMetricsResolutions(p.MetricsResolutions), | ||
| } | ||
|
|
||
| agent, err := as.executeAgentChange(ctx, agentID, models.QANPostgreSQLPgStatMonitorAgentType, params) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -1409,7 +1393,7 @@ func (as *AgentsService) ChangeRDSExporter(ctx context.Context, agentID string, | |
| MetricsResolutions: convertMetricsResolutions(p.MetricsResolutions), | ||
| } | ||
|
|
||
| agent, err := as.executeAgentChange(ctx, agentID, models.RDSExporterType, params) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -1506,7 +1490,7 @@ func (as *AgentsService) ChangeExternalExporter( | |
| MetricsResolutions: convertMetricsResolutions(p.MetricsResolutions), | ||
| } | ||
|
|
||
| agent, err := as.executeAgentChange(ctx, agentID, models.ExternalExporterType, params) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -1593,7 +1577,7 @@ func (as *AgentsService) ChangeAzureDatabaseExporter( | |
| MetricsResolutions: convertMetricsResolutions(p.MetricsResolutions), | ||
| } | ||
|
|
||
| agent, err := as.executeAgentChange(ctx, agentID, models.AzureDatabaseExporterType, params) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -1621,7 +1605,7 @@ func (as *AgentsService) ChangeNomadAgent(ctx context.Context, agentID string, p | |
| SkipConnectionCheck: true, | ||
| } | ||
|
|
||
| agent, err := as.executeAgentChange(ctx, agentID, models.NomadAgentType, changeParams) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -1712,7 +1696,7 @@ func (as *AgentsService) ChangeRTAMongoDBAgent( | |
| changeParams.RTAOptions = models.RTAOptionsFromRequest(p.RtaOptions) | ||
| } | ||
|
|
||
| ag, err := as.executeAgentChange(ctx, agentID, models.RTAMongoDBAgentType, changeParams) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -1768,6 +1752,51 @@ func unexpectedAgentTypeError(agent inventoryv1.Agent) error { | |
| return status.Errorf(codes.Internal, "unexpected agent type %T", agent) | ||
| } | ||
|
|
||
| // checkInternalPgQANEnvOverride rejects a request that would flip the enabled state of the QAN agent | ||
| // of PMM's internal PostgreSQL server away from the state pinned by the PMM_ENABLE_INTERNAL_PG_QAN | ||
| // environment variable. | ||
| // | ||
| // The agent argument is the stored row, before the requested change is applied. | ||
| // | ||
| // The guard deliberately stays in the service layer rather than moving into | ||
| // models.ApplyAgentChange: Server.handleInternalQANToggle is the legitimate actor for this exact | ||
| // state and calls ApplyAgentChange directly, so a guard down there would make the settings API trip | ||
| // over its own pin. | ||
| func checkInternalPgQANEnvOverride(q *reform.Querier, agent *models.Agent, enable *bool) error { | ||
| // Only a request that actually flips the enabled state can contradict the variable. | ||
| if enable == nil || *enable == !agent.Disabled { | ||
| return nil | ||
| } | ||
|
|
||
| // Read before IsInternalPgQANAgent, which costs a query: with the variable unset -- the default | ||
| // -- nothing is pinned and there is nothing to check. | ||
| enabledByEnv, lookupErr := env.LookupBool(env.EnableInternalPgQAN) | ||
| if enabledByEnv == nil && lookupErr == nil { | ||
| return nil | ||
| } | ||
|
|
||
| internal, err := models.IsInternalPgQANAgent(q, agent) | ||
| if err != nil || !internal { | ||
| return err | ||
| } | ||
|
|
||
| if lookupErr != nil { | ||
| // pmm-managed-init rejects an unparsable value before PMM Server starts, so reaching here | ||
| // means that validation was bypassed. The intent to pin is clear even though the value is | ||
| // not, so refuse rather than silently unpin. | ||
| return status.Errorf(codes.FailedPrecondition, "QAN for PMM's internal PostgreSQL server is configured via an environment variable: %s.", lookupErr) | ||
| } | ||
| if *enable == *enabledByEnv { | ||
| return nil | ||
| } | ||
|
|
||
| return status.Errorf( | ||
| codes.FailedPrecondition, | ||
| "QAN for PMM's internal PostgreSQL server is set to %t via an environment variable.", | ||
| *enabledByEnv, | ||
| ) | ||
| } | ||
|
|
||
| // Helper function to convert custom labels from protobuf to model format. | ||
| func convertCustomLabels(customLabels *common.StringMap) *map[string]string { | ||
| if customLabels != nil { | ||
|
|
@@ -1814,11 +1843,36 @@ func convertMetricsResolutions(mrs *common.MetricsResolutions) *models.ChangeMet | |
| } | ||
|
|
||
| // Helper function to execute agent change and build response. | ||
| // | ||
| // The expectedType argument is the agent type that the calling Change*Agent method knows how to | ||
| // convert. The inventory API picks that method from the request payload and not from the type of the | ||
| // agent being changed, so a request can name an agent of any type. Checking the type here, inside | ||
| // the transaction, turns that into a rejected request; without it the change is committed and only | ||
| // then fails the type assertion in the caller, leaving the agent modified, pmm-agent not notified | ||
| // and the client with an internal error. | ||
| // | ||
| // The expectedType argument restates what the caller's own type assertion on the result already says, and the | ||
| // compiler cannot tie the two together: keep them in sync, or a valid request becomes InvalidArgument. | ||
| func (as *AgentsService) executeAgentChange(ctx context.Context, agentID string, expectedType models.AgentType, params *models.ChangeAgentParams) (inventoryv1.Agent, error) { //nolint:ireturn,lll | ||
| var agent inventoryv1.Agent | ||
|
|
||
| err := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { | ||
| // Returning an error rolls the transaction back, so a rejected request leaves the agent untouched. | ||
| currentAgent, err := models.FindAgentByID(tx.Querier, agentID) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if currentAgent.AgentType != expectedType { | ||
|
Member
There was a problem hiding this comment. This precheck, not the guard rework, is what fixes the reported bug — worth reflecting in the PR description. The old guard returned This check converts that into |
||
| return status.Errorf(codes.InvalidArgument, "Agent with ID %s has type %s, expected %s.", agentID, currentAgent.AgentType, expectedType) | ||
| } | ||
|
|
||
| err = checkInternalPgQANEnvOverride(tx.Querier, currentAgent, params.Enabled) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| updatedAgent, err := models.ApplyAgentChange(tx.Querier, currentAgent, params) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This and
FindInternalPgQANAgentdisagree on what "the internal agent" is, and the difference is a silent bypass.Here:
agent_typeANDpmm_agent_id == PMMServerAgentIDAND service name. InFindInternalPgQANAgent(line 388), which is what the settings API acts on: service name AND type, nopmm_agent_id.models.PMMServerAgentIDisn't a constant — it's a mutable process global, reassigned in HA setup (managed/models/database.go:1656) and from the pmm-agent config file (managed/models/database.go:1608).So this sequence leaves an agent that
ChangeSettingshappily toggles but all three guards consider foreign: with the variable unset, remove the internal QAN agent, register a second pmm-agent, re-add the QAN agent forpmm-server-postgresqlunder that pmm-agent (your ownRejectAddingASecondInternalQANAgentUnderAnotherPMMAgenttest proves this shape is constructible), then setPMM_ENABLE_INTERNAL_PG_QAN=true.The
pmm_agent_idconjunct also isn't needed for the reason the comment gives. Service names are unique, so RDS/Azure remote instances necessarily have a differentservice_name— the service-name check alone already excludes them. Dropping the conjunct makes the two functions agree and closes the gap.