fetch_request.go

package sarama

import "fmt"

type fetchRequestBlock struct {
	Version int16
	// currentLeaderEpoch contains the current leader epoch of the partition.
	currentLeaderEpoch int32
	// fetchOffset contains the message offset.
	fetchOffset int64
	// logStartOffset contains the earliest available offset of the follower
	// replica.  The field is only used when the request is sent by the
	// follower.
	logStartOffset int64
	// maxBytes contains the maximum bytes to fetch from this partition.  See
	// KIP-74 for cases where this limit may not be honored.
	maxBytes int32
}

func (b *fetchRequestBlock) encode(pe packetEncoder, version int16) error {
	b.Version = version
	if b.Version >= 9 {
		pe.putInt32(b.currentLeaderEpoch)
	}
	pe.putInt64(b.fetchOffset)
	if b.Version >= 5 {
		pe.putInt64(b.logStartOffset)
	}
	pe.putInt32(b.maxBytes)
	return nil
}

func (b *fetchRequestBlock) decode(pd packetDecoder, version int16) (err error) {
	b.Version = version
	if b.Version >= 9 {
		if b.currentLeaderEpoch, err = pd.getInt32(); err != nil {
			return err
		}
	}
	if b.fetchOffset, err = pd.getInt64(); err != nil {
		return err
	}
	if b.Version >= 5 {
		if b.logStartOffset, err = pd.getInt64(); err != nil {
			return err
		}
	}
	if b.maxBytes, err = pd.getInt32(); err != nil {
		return err
	}
	return nil
}

// FetchRequest (API key 1) will fetch Kafka messages. Version 3 introduced the MaxBytes field. See
// https://issues.apache.org/jira/browse/KAFKA-2063 for a discussion of the issues leading up to that.  The KIP is at
// https://cwiki.apache.org/confluence/display/KAFKA/KIP-74%3A+Add+Fetch+Response+Size+Limit+in+Bytes
type FetchRequest struct {
	// Version defines the protocol version to use for encode and decode
	Version int16
	// ReplicaID contains the broker ID of the follower, of -1 if this request
	// is from a consumer.
	// ReplicaID int32
	// MaxWaitTime contains the maximum time in milliseconds to wait for the response.
	MaxWaitTime int32
	// MinBytes contains the minimum bytes to accumulate in the response.
	MinBytes int32
	// MaxBytes contains the maximum bytes to fetch.  See KIP-74 for cases
	// where this limit may not be honored.
	MaxBytes int32
	// Isolation contains a This setting controls the visibility of
	// transactional records. Using READ_UNCOMMITTED (isolation_level = 0)
	// makes all records visible. With READ_COMMITTED (isolation_level = 1),
	// non-transactional and COMMITTED transactional records are visible. To be
	// more concrete, READ_COMMITTED returns all data from offsets smaller than
	// the current LSO (last stable offset), and enables the inclusion of the
	// list of aborted transactions in the result, which allows consumers to
	// discard ABORTED transactional records
	Isolation IsolationLevel
	// SessionID contains the fetch session ID.
	SessionID int32
	// SessionEpoch contains the epoch of the partition leader as known to the
	// follower replica or a consumer.
	SessionEpoch int32
	// blocks contains the topics to fetch.
	blocks map[string]map[int32]*fetchRequestBlock
	// forgotten contains in an incremental fetch request, the partitions to remove.
	forgotten map[string][]int32
	// RackID contains a Rack ID of the consumer making this request
	RackID string
}

type IsolationLevel int8

const (
	ReadUncommitted IsolationLevel = iota
	ReadCommitted
)

func (r *FetchRequest) encode(pe packetEncoder) (err error) {
	metricRegistry := pe.metricRegistry()

	pe.putInt32(-1) // ReplicaID is always -1 for clients
	pe.putInt32(r.MaxWaitTime)
	pe.putInt32(r.MinBytes)
	if r.Version >= 3 {
		pe.putInt32(r.MaxBytes)
	}
	if r.Version >= 4 {
		pe.putInt8(int8(r.Isolation))
	}
	if r.Version >= 7 {
		pe.putInt32(r.SessionID)
		pe.putInt32(r.SessionEpoch)
	}
	err = pe.putArrayLength(len(r.blocks))
	if err != nil {
		return err
	}
	for topic, blocks := range r.blocks {
		err = pe.putString(topic)
		if err != nil {
			return err
		}
		err = pe.putArrayLength(len(blocks))
		if err != nil {
			return err
		}
		for partition, block := range blocks {
			pe.putInt32(partition)
			err = block.encode(pe, r.Version)
			if err != nil {
				return err
			}
		}
		getOrRegisterTopicMeter("consumer-fetch-rate", topic, metricRegistry).Mark(1)
	}
	if r.Version >= 7 {
		err = pe.putArrayLength(len(r.forgotten))
		if err != nil {
			return err
		}
		for topic, partitions := range r.forgotten {
			err = pe.putString(topic)
			if err != nil {
				return err
			}
			err = pe.putArrayLength(len(partitions))
			if err != nil {
				return err
			}
			for _, partition := range partitions {
				pe.putInt32(partition)
			}
		}
	}
	if r.Version >= 11 {
		err = pe.putString(r.RackID)
		if err != nil {
			return err
		}
	}

	return nil
}

func (r *FetchRequest) decode(pd packetDecoder, version int16) (err error) {
	r.Version = version

	if _, err = pd.getInt32(); err != nil {
		return err
	}
	if r.MaxWaitTime, err = pd.getInt32(); err != nil {
		return err
	}
	if r.MinBytes, err = pd.getInt32(); err != nil {
		return err
	}
	if r.Version >= 3 {
		if r.MaxBytes, err = pd.getInt32(); err != nil {
			return err
		}
	}
	if r.Version >= 4 {
		isolation, err := pd.getInt8()
		if err != nil {
			return err
		}
		r.Isolation = IsolationLevel(isolation)
	}
	if r.Version >= 7 {
		r.SessionID, err = pd.getInt32()
		if err != nil {
			return err
		}
		r.SessionEpoch, err = pd.getInt32()
		if err != nil {
			return err
		}
	}
	topicCount, err := pd.getArrayLength()
	if err != nil {
		return err
	}
	if topicCount == 0 {
		return nil
	}
	r.blocks = make(map[string]map[int32]*fetchRequestBlock)
	for i := 0; i < topicCount; i++ {
		topic, err := pd.getString()
		if err != nil {
			return err
		}
		partitionCount, err := pd.getArrayLength()
		if err != nil {
			return err
		}
		r.blocks[topic] = make(map[int32]*fetchRequestBlock)
		for j := 0; j < partitionCount; j++ {
			partition, err := pd.getInt32()
			if err != nil {
				return err
			}
			fetchBlock := &fetchRequestBlock{}
			if err = fetchBlock.decode(pd, r.Version); err != nil {
				return err
			}
			r.blocks[topic][partition] = fetchBlock
		}
	}

	if r.Version >= 7 {
		forgottenCount, err := pd.getArrayLength()
		if err != nil {
			return err
		}
		r.forgotten = make(map[string][]int32)
		for i := 0; i < forgottenCount; i++ {
			topic, err := pd.getString()
			if err != nil {
				return err
			}
			partitionCount, err := pd.getArrayLength()
			if err != nil {
				return err
			}
			if partitionCount < 0 {
				return fmt.Errorf("partitionCount %d is invalid", partitionCount)
			}
			r.forgotten[topic] = make([]int32, partitionCount)

			for j := 0; j < partitionCount; j++ {
				partition, err := pd.getInt32()
				if err != nil {
					return err
				}
				r.forgotten[topic][j] = partition
			}
		}
	}

	if r.Version >= 11 {
		r.RackID, err = pd.getString()
		if err != nil {
			return err
		}
	}

	return nil
}

func (r *FetchRequest) key() int16 {
	return 1
}

func (r *FetchRequest) version() int16 {
	return r.Version
}

func (r *FetchRequest) headerVersion() int16 {
	return 1
}

func (r *FetchRequest) isValidVersion() bool {
	return r.Version >= 0 && r.Version <= 11
}

func (r *FetchRequest) requiredVersion() KafkaVersion {
	switch r.Version {
	case 11:
		return V2_3_0_0
	case 9, 10:
		return V2_1_0_0
	case 8:
		return V2_0_0_0
	case 7:
		return V1_1_0_0
	case 6:
		return V1_0_0_0
	case 4, 5:
		return V0_11_0_0
	case 3:
		return V0_10_1_0
	case 2:
		return V0_10_0_0
	case 1:
		return V0_9_0_0
	case 0:
		return V0_8_2_0
	default:
		return V2_3_0_0
	}
}

func (r *FetchRequest) AddBlock(topic string, partitionID int32, fetchOffset int64, maxBytes int32, leaderEpoch int32) {
	if r.blocks == nil {
		r.blocks = make(map[string]map[int32]*fetchRequestBlock)
	}

	if r.Version >= 7 && r.forgotten == nil {
		r.forgotten = make(map[string][]int32)
	}

	if r.blocks[topic] == nil {
		r.blocks[topic] = make(map[int32]*fetchRequestBlock)
	}

	tmp := new(fetchRequestBlock)
	tmp.Version = r.Version
	tmp.maxBytes = maxBytes
	tmp.fetchOffset = fetchOffset
	if r.Version >= 9 {
		tmp.currentLeaderEpoch = leaderEpoch
	}

	r.blocks[topic][partitionID] = tmp
}