Skip to content

fix: handle message too large error on kafka client and emit metric #15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
siddhanta-rath merged 6 commits into from
Aug 21, 2025
Merged

fix: handle message too large error on kafka client and emit metric #15

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d299f35
fix: handle message too large error on kafka client and emit metric
siddhanta-rath Aug 21, 2025
b21b499
debug: add logger statement
siddhanta-rath Aug 21, 2025
5e649eb
fix : error handling on message too large
siddhanta-rath Aug 21, 2025
335b6ed
refactor: error message
siddhanta-rath Aug 21, 2025
3ebb76d
refactor: metric name and tags while producing to kafka
siddhanta-rath Aug 21, 2025
bc0392e
fix: metric tag name for message too large
siddhanta-rath Aug 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions publisher/kafka.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@ import (
"github.com/goto/raccoon/metrics"
)

const (
errUnknownTopic = "Local: Unknown topic" //error msg while producing a message to a topic which is not present in the kafka cluster
errLargeMessageSize = "Broker: Message size too large" //error msg while producing a message which is larger than message.max.bytes config
)

// KafkaProducer Produce data to kafka synchronously
type KafkaProducer interface {
// ProduceBulk message to kafka. Block until all messages are sent. Return array of error. Order is not guaranteed.
Expand Down Expand Up @@ -63,11 +68,19 @@ func (pr *Kafka) ProduceBulk(events []*pb.Event, connGroup string, deliveryChann
err := pr.kp.Produce(message, deliveryChannel)
if err != nil {
metrics.Increment("kafka_messages_delivered_total", fmt.Sprintf("success=false,conn_group=%s,event_type=%s", connGroup, event.Type))
if err.Error() == "Local: Unknown topic" {
switch err.Error() {
case errUnknownTopic:
errors[order] = fmt.Errorf("%v %s", err, topic)
metrics.Increment("kafka_unknown_topic_failure_total",
fmt.Sprintf("topic=%s,event_type=%s,conn_group=%s", topic, event.Type, connGroup))

case errLargeMessageSize:
errors[order] = fmt.Errorf("%v %s", err, topic)
metrics.Increment("kafka_unknown_topic_failure_total", fmt.Sprintf("topic=%s,event_type=%s,conn_group=%s", topic, event.Type, connGroup))

@cpramik cpramik Aug 21, 2025

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

metric should be: kafka_error
with tags: unknown_topic, message_too_large

} else {
metrics.Increment("kafka_message_too_large_total",
fmt.Sprintf("topic=%s,event_type=%s,conn_group=%s", topic, event.Type, connGroup))
default:
errors[order] = err
logger.Errorf("produce to kafka failed due to: %v", err)
}
continue
}
Expand Down
13 changes: 11 additions & 2 deletions publisher/kafka_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,11 +90,20 @@ func TestKafka_ProduceBulk(suite *testing.T) {

t.Run("Should return topic name when unknown topic is returned", func(t *testing.T) {
client := &mockClient{}
client.On("Produce", mock.Anything, mock.Anything).Return(fmt.Errorf("Local: Unknown topic")).Once()
client.On("Produce", mock.Anything, mock.Anything).Return(fmt.Errorf(errUnknownTopic)).Once()
kp := NewKafkaFromClient(client, 10, "%s")

err := kp.ProduceBulk([]*pb.Event{{EventBytes: []byte{}, Type: topic}}, "group1", make(chan kafka.Event, 2))
assert.EqualError(t, err.(BulkError).Errors[0], "Local: Unknown topic "+topic)
assert.EqualError(t, err.(BulkError).Errors[0], errUnknownTopic+" "+topic)
})

t.Run("Should return topic name when message size is too large", func(t *testing.T) {
client := &mockClient{}
client.On("Produce", mock.Anything, mock.Anything).Return(fmt.Errorf(errLargeMessageSize)).Once()
kp := NewKafkaFromClient(client, 10, "%s")

err := kp.ProduceBulk([]*pb.Event{{EventBytes: []byte{}, Type: topic}}, "group1", make(chan kafka.Event, 2))
assert.EqualError(t, err.(BulkError).Errors[0], errLargeMessageSize+" "+topic)
})
})

Expand Down