Skip to content

Commit ef5741e

Browse files
dtunikovclaudeilidemi
authored
feat(alerting): classify MySQL 1236 max_allowed_packet errors separately (PeerDB-io#4417)
## Problem MySQL error `1236 (HY000)` (`ER_MASTER_FATAL_ERROR_READING_BINLOG`) is currently always classified as `NOTIFY_BINLOG_INVALID`. But one common cause is a single binlog event exceeding the replica's `max_allowed_packet`: ``` failed in pull records when: MySQL execute error: ERROR 1236 (HY000): log event entry exceeded max_allowed_packet; Increase max_allowed_packet on source; the first event 'mysql-bin.168301' at 1789438008, ... ``` This is **not** binlog corruption/purge — the mitigation is different (increase `max_allowed_packet` on the source, resync the affected table, or exclude oversized blob columns), so it deserves its own error class and tailored user notification. ## Change - Add `ErrorNotifyBinlogEventExceededMaxAllowedPacket` (`NOTIFY_BINLOG_EVENT_EXCEEDED_MAX_ALLOWED_PACKET`, action `NotifyUser`). - Inside `case 1236`, sub-classify on the `max_allowed_packet` message signature; everything else still maps to `NOTIFY_BINLOG_INVALID`. - Add a unit test covering both the new case and the generic 1236 fallback. It closes DBI-787. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Ilia Demianenko <ilia.demianenko@clickhouse.com>
1 parent 2c82c16 commit ef5741e

2 files changed

Lines changed: 42 additions & 2 deletions

File tree

flow/alerting/classifier.go

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,9 @@ var (
142142
ErrorNotifyBinlogInvalid = ErrorClass{
143143
Class: "NOTIFY_BINLOG_INVALID", action: NotifyUser,
144144
}
145+
ErrorNotifyBinlogEventExceededMaxAllowedPacket = ErrorClass{
146+
Class: "NOTIFY_BINLOG_EVENT_EXCEEDED_MAX_ALLOWED_PACKET", action: NotifyUser,
147+
}
145148
ErrorNotifyBinlogRowMetadataInvalid = ErrorClass{
146149
Class: "NOTIFY_BINLOG_ROW_METADATA_INVALID", action: NotifyUser,
147150
}
@@ -746,8 +749,13 @@ func GetErrorClass(ctx context.Context, err error) (ErrorClass, ErrorInfo) {
746749
1226, // ER_USER_LIMIT_REACHED
747750
1827: // ER_PASSWORD_FORMAT
748751
return ErrorNotifyConnectivity, myErrorInfo
749-
case 1236, // ER_MASTER_FATAL_ERROR_READING_BINLOG
750-
1373: // ER_UNKNOWN_TARGET_BINLOG
752+
case 1236: // ER_MASTER_FATAL_ERROR_READING_BINLOG
753+
// A single binlog event larger than the replica's max_allowed_packet aborts the binlog stream read
754+
if strings.Contains(myErr.Message, "max_allowed_packet") {
755+
return ErrorNotifyBinlogEventExceededMaxAllowedPacket, myErrorInfo
756+
}
757+
return ErrorNotifyBinlogInvalid, myErrorInfo
758+
case 1373: // ER_UNKNOWN_TARGET_BINLOG
751759
return ErrorNotifyBinlogInvalid, myErrorInfo
752760
case 1105: // ER_UNKNOWN_ERROR
753761
// RDS Aurora MySQL specific errors due to "Zero Downtime Patch" or "Zero Downtime Restart"

flow/alerting/classifier_test.go

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -846,6 +846,38 @@ func TestAuroraMySQLZeroDowntimeRestartErrorShouldBeRecoverable(t *testing.T) {
846846
}, errInfo, "Unexpected error info")
847847
}
848848

849+
func TestMySQLBinlogEventExceededMaxAllowedPacket(t *testing.T) {
850+
// Error 1236 caused by a binlog event larger than max_allowed_packet should be
851+
// classified separately from generic binlog invalidation.
852+
mysqlErr := &mysql.MyError{
853+
Code: 1236, // ER_MASTER_FATAL_ERROR_READING_BINLOG
854+
State: "HY000",
855+
Message: "log event entry exceeded max_allowed_packet; Increase max_allowed_packet on source; " +
856+
"the first event 'mysql-bin.168301' at 1789438008, the last event read from " +
857+
"'/app/work2/binlogs/mysql-bin.168301' at 2333874086, the last byte read from " +
858+
"'/app/work2/binlogs/mysql-bin.168301' at 2333874105.",
859+
}
860+
errorClass, errInfo := GetErrorClass(t.Context(), fmt.Errorf("failed in pull records: %w", mysqlErr))
861+
assert.Equal(t, ErrorNotifyBinlogEventExceededMaxAllowedPacket, errorClass, "Unexpected error class")
862+
assert.Equal(t, ErrorInfo{
863+
Source: ErrorSourceMySQL,
864+
Code: "1236",
865+
}, errInfo, "Unexpected error info")
866+
867+
// A 1236 without the max_allowed_packet signature should still be generic binlog invalidation.
868+
genericErr := &mysql.MyError{
869+
Code: 1236,
870+
State: "HY000",
871+
Message: "Could not find first log file name in binary log index file",
872+
}
873+
errorClass, errInfo = GetErrorClass(t.Context(), fmt.Errorf("mysql error: %w", genericErr))
874+
assert.Equal(t, ErrorNotifyBinlogInvalid, errorClass, "Unexpected error class")
875+
assert.Equal(t, ErrorInfo{
876+
Source: ErrorSourceMySQL,
877+
Code: "1236",
878+
}, errInfo, "Unexpected error info")
879+
}
880+
849881
func TestMySQLExecuteError(t *testing.T) {
850882
err := exceptions.NewMySQLExecuteError(
851883
tls.RecordHeaderError{Msg: "first record does not look like a TLS handshake"})

0 commit comments

Comments
 (0)