Skip to content

Commit c8446a6

Browse files
ctrlaltlucctrlaltluc
authored
Allow dashes when parsing broker rack (#68)
1 parent 83868b8 commit c8446a6

File tree

2 files changed

+131
-8
lines changed

2 files changed

+131
-8
lines changed

‎pkg/resources/kafka/kafka.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ const (
9595

9696
var (
9797
// kafkaConfigBrokerRackRegex the regex to parse the "broker.rack" Kafka property used in read-only configs
98-
kafkaConfigBrokerRackRegex = regexp.MustCompile(`broker\.rack\s*=\s*(\w+)`)
98+
kafkaConfigBrokerRackRegex = regexp.MustCompile(`broker\.rack\s*=\s*([\w-]+)`)
9999
)
100100

101101
// Reconciler implements the Component Reconciler
@@ -219,6 +219,8 @@ func (r *Reconciler) Reconcile(log logr.Logger) error {
219219

220220
log.V(1).Info("Reconciling")
221221

222+
log.Info("broker rack map", "kafkaBrokerAvailabilityZoneMap", r.kafkaBrokerAvailabilityZoneMap)
223+
222224
ctx := context.Background()
223225
if err := k8sutil.UpdateBrokerConfigurationBackup(r.Client, r.KafkaCluster); err != nil {
224226
log.Error(err, "failed to update broker configuration backup")
@@ -926,6 +928,9 @@ func (r *Reconciler) handleRollingUpgrade(log logr.Logger, desiredPod, currentPo
926928
// Check if we support multiple broker restarts and restart only in same AZ, otherwise restart only 1 broker at once
927929
concurrentBrokerRestartsAllowed := r.getConcurrentBrokerRestartsAllowed()
928930
terminatingOrPendingPods := getPodsInTerminatingOrPendingState(podList.Items)
931+
if len(terminatingOrPendingPods) > 0 {
932+
log.Info("terminating or pending pods", "terminatingOrPendingPods", terminatingOrPendingPods)
933+
}
929934
if len(terminatingOrPendingPods) >= concurrentBrokerRestartsAllowed {
930935
return errorfactory.New(errorfactory.ReconcileRollingUpgrade{}, errors.New(strconv.Itoa(concurrentBrokerRestartsAllowed)+" pod(s) is still terminating or creating"), "rolling upgrade in progress")
931936
}

‎pkg/resources/kafka/kafka_test.go‎

Lines changed: 125 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1269,9 +1269,9 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) {
12691269
desiredPod: &corev1.Pod{},
12701270
currentPod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "kafka-201"}},
12711271
pods: []corev1.Pod{
1272-
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-101", DeletionTimestamp: &metav1.Time{Time: time.Now()}}},
1273-
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-201"}},
1274-
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-301"}},
1272+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-101", Labels: map[string]string{"brokerId": "101"}, DeletionTimestamp: &metav1.Time{Time: time.Now()}}},
1273+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", Labels: map[string]string{"brokerId": "201"}}},
1274+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}},
12751275
},
12761276
errorExpected: true,
12771277
},
@@ -1293,9 +1293,9 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) {
12931293
desiredPod: &corev1.Pod{},
12941294
currentPod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "kafka-301"}},
12951295
pods: []corev1.Pod{
1296-
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-101", DeletionTimestamp: &metav1.Time{Time: time.Now()}}},
1297-
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", DeletionTimestamp: &metav1.Time{Time: time.Now()}}},
1298-
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-301"}},
1296+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-101", Labels: map[string]string{"brokerId": "101"}, DeletionTimestamp: &metav1.Time{Time: time.Now()}}},
1297+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", Labels: map[string]string{"brokerId": "201"}, DeletionTimestamp: &metav1.Time{Time: time.Now()}}},
1298+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}},
12991299
},
13001300
errorExpected: true,
13011301
},
@@ -1531,6 +1531,124 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) {
15311531
outOfSyncReplicas: []int32{101},
15321532
errorExpected: false,
15331533
},
1534+
{
1535+
testName: "Pod is not deleted if pod is restarting in another AZ, if brokers per AZ < tolerated failures",
1536+
kafkaCluster: v1beta1.KafkaCluster{
1537+
ObjectMeta: metav1.ObjectMeta{
1538+
Name: "kafka",
1539+
Namespace: "kafka",
1540+
},
1541+
Spec: v1beta1.KafkaClusterSpec{
1542+
Brokers: []v1beta1.Broker{
1543+
{Id: 101, ReadOnlyConfig: "broker.rack=az1"},
1544+
{Id: 201, ReadOnlyConfig: "broker.rack=az2"},
1545+
{Id: 301, ReadOnlyConfig: "broker.rack=az3"},
1546+
},
1547+
RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{
1548+
FailureThreshold: 2,
1549+
ConcurrentBrokerRestartsAllowed: 2,
1550+
},
1551+
},
1552+
Status: v1beta1.KafkaClusterStatus{State: v1beta1.KafkaClusterRollingUpgrading},
1553+
},
1554+
desiredPod: &corev1.Pod{},
1555+
currentPod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", Labels: map[string]string{"brokerId": "201"}}},
1556+
pods: []corev1.Pod{
1557+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-101", Labels: map[string]string{"brokerId": "101"}, DeletionTimestamp: &metav1.Time{Time: time.Now()}}},
1558+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", Labels: map[string]string{"brokerId": "201"}}},
1559+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}},
1560+
},
1561+
errorExpected: true,
1562+
},
1563+
{
1564+
testName: "Pod is not deleted if there are out-of-sync replicas in another AZ, if brokers per AZ < tolerated failures",
1565+
kafkaCluster: v1beta1.KafkaCluster{
1566+
ObjectMeta: metav1.ObjectMeta{
1567+
Name: "kafka",
1568+
Namespace: "kafka",
1569+
},
1570+
Spec: v1beta1.KafkaClusterSpec{
1571+
Brokers: []v1beta1.Broker{
1572+
{Id: 101, ReadOnlyConfig: "broker.rack=az1"},
1573+
{Id: 201, ReadOnlyConfig: "broker.rack=az2"},
1574+
{Id: 301, ReadOnlyConfig: "broker.rack=az3"},
1575+
},
1576+
RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{
1577+
FailureThreshold: 2,
1578+
ConcurrentBrokerRestartsAllowed: 2,
1579+
},
1580+
},
1581+
Status: v1beta1.KafkaClusterStatus{State: v1beta1.KafkaClusterRollingUpgrading},
1582+
},
1583+
desiredPod: &corev1.Pod{},
1584+
currentPod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", Labels: map[string]string{"brokerId": "201"}}},
1585+
pods: []corev1.Pod{
1586+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-101", Labels: map[string]string{"brokerId": "101"}}},
1587+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", Labels: map[string]string{"brokerId": "201"}}},
1588+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}},
1589+
},
1590+
outOfSyncReplicas: []int32{101},
1591+
errorExpected: true,
1592+
},
1593+
{
1594+
testName: "Pod is not deleted if there are offline replicas in another AZ, if brokers per AZ < tolerated failures",
1595+
kafkaCluster: v1beta1.KafkaCluster{
1596+
ObjectMeta: metav1.ObjectMeta{
1597+
Name: "kafka",
1598+
Namespace: "kafka",
1599+
},
1600+
Spec: v1beta1.KafkaClusterSpec{
1601+
Brokers: []v1beta1.Broker{
1602+
{Id: 101, ReadOnlyConfig: "broker.rack=az1"},
1603+
{Id: 201, ReadOnlyConfig: "broker.rack=az2"},
1604+
{Id: 301, ReadOnlyConfig: "broker.rack=az3"},
1605+
},
1606+
RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{
1607+
FailureThreshold: 2,
1608+
ConcurrentBrokerRestartsAllowed: 2,
1609+
},
1610+
},
1611+
Status: v1beta1.KafkaClusterStatus{State: v1beta1.KafkaClusterRollingUpgrading},
1612+
},
1613+
desiredPod: &corev1.Pod{},
1614+
currentPod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", Labels: map[string]string{"brokerId": "201"}}},
1615+
pods: []corev1.Pod{
1616+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-101", Labels: map[string]string{"brokerId": "101"}}},
1617+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", Labels: map[string]string{"brokerId": "201"}}},
1618+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}},
1619+
},
1620+
allOfflineReplicas: []int32{101},
1621+
errorExpected: true,
1622+
},
1623+
{
1624+
testName: "Pod is not deleted if pod is restarting in another AZ, if broker rack value contains dashes",
1625+
kafkaCluster: v1beta1.KafkaCluster{
1626+
ObjectMeta: metav1.ObjectMeta{
1627+
Name: "kafka",
1628+
Namespace: "kafka",
1629+
},
1630+
Spec: v1beta1.KafkaClusterSpec{
1631+
Brokers: []v1beta1.Broker{
1632+
{Id: 101, ReadOnlyConfig: "broker.rack=az-1"},
1633+
{Id: 201, ReadOnlyConfig: "broker.rack=az-2"},
1634+
{Id: 301, ReadOnlyConfig: "broker.rack=az-3"},
1635+
},
1636+
RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{
1637+
FailureThreshold: 2,
1638+
ConcurrentBrokerRestartsAllowed: 2,
1639+
},
1640+
},
1641+
Status: v1beta1.KafkaClusterStatus{State: v1beta1.KafkaClusterRollingUpgrading},
1642+
},
1643+
desiredPod: &corev1.Pod{},
1644+
currentPod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", Labels: map[string]string{"brokerId": "201"}}},
1645+
pods: []corev1.Pod{
1646+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-101", Labels: map[string]string{"brokerId": "101"}, DeletionTimestamp: &metav1.Time{Time: time.Now()}}},
1647+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", Labels: map[string]string{"brokerId": "201"}}},
1648+
{ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}},
1649+
},
1650+
errorExpected: true,
1651+
},
15341652
}
15351653

15361654
mockCtrl := gomock.NewController(t)
@@ -1557,7 +1675,7 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) {
15571675

15581676
// Mock kafka client
15591677
mockedKafkaClient := new(mocks.KafkaClient)
1560-
mockedKafkaClient.On("AllOfflineReplicas").Return(test.outOfSyncReplicas, nil)
1678+
mockedKafkaClient.On("AllOfflineReplicas").Return(test.allOfflineReplicas, nil)
15611679
mockedKafkaClient.On("OutOfSyncReplicas").Return(test.outOfSyncReplicas, nil)
15621680
mockKafkaClientProvider.On("NewFromCluster", mockClient, &test.kafkaCluster).Return(mockedKafkaClient, func() {}, nil)
15631681

0 commit comments

Comments
 (0)