Merge branch 'feat-add-staticAllocations-field' into 'main'

Michael Patsula · Michael Patsula · commit 70e069ce6e73 · 2024-07-10T16:59:00.000Z
feat: add staticAllocations field to CRD

See merge request cloudnative/go/cidr-allocator!25
diff --git a/Dockerfile b/Dockerfile
@@ -17,7 +17,7 @@ ARG IMAGE_REPOSITORY=docker.io
 ARG TARGETARCH
 ARG TARGETOS=linux
 
-FROM ${IMAGE_REPOSITORY}/golang:1.22 as builder
+FROM ${IMAGE_REPOSITORY}/golang:1.22.5 as builder
 ARG TARGETOS
 ARG TARGETARCH
 
diff --git a/api/v1alpha1/nodecidrallocation_types.go b/api/v1alpha1/nodecidrallocation_types.go
@@ -40,6 +40,12 @@ type NodeCIDRAllocationSpec struct {
 	//+kubebuilder:validation:MinItems=1
 	AddressPools []string `json:"addressPools,omitempty" protobuf:"bytes,7,opt,name=addressPools" patchStrategy:"merge"`
 
+	// StaticAllocations represents a list of static address pools in the form of a list of
+	// network CIDRs that are reserved from being used by any node.
+	//+optional
+	//+patchStrategy=merge
+	StaticAllocations []string `json:"staticAllocations,omitempty" protobuf:"bytes,7,opt,name=staticAllocations" patchStrategy:"merge"`
+
 	// NodeSelector represents a Kubernetes node selector to filter nodes from
 	// the cluster for which to apply Pod CIDRs onto.
 	// NOTE: Nodes that are selected through the node selector MUST specify a maximum number of pods in order to help identify
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/networking.statcan.gc.ca_nodecidrallocations.yaml b/config/crd/bases/networking.statcan.gc.ca_nodecidrallocations.yaml
@@ -89,6 +89,13 @@ spec:
                         the correct size for the NodeCIDRAllocation Controller to allocate to it. If none is specified a subnet WILL NOT be allocated for the Node.
                 type: object
                 x-kubernetes-map-type: atomic
+              staticAllocations:
+                description: |-
+                  StaticAllocations represents a list of static address pools in the form of a list of
+                  network CIDRs that are reserved from being used by any node.
+                items:
+                  type: string
+                type: array
             type: object
           status:
             description: |-
diff --git a/go.mod b/go.mod
@@ -45,11 +45,11 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/net v0.22.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
 	golang.org/x/oauth2 v0.18.0 // indirect
-	golang.org/x/sys v0.18.0 // indirect
-	golang.org/x/term v0.18.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/term v0.22.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
diff --git a/go.sum b/go.sum
@@ -117,8 +117,10 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
+golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
 golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
 golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -134,16 +136,22 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
 golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
+golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -153,6 +161,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
 golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/install/kubernetes/cidr-allocator/Chart.yaml b/install/kubernetes/cidr-allocator/Chart.yaml
@@ -10,9 +10,9 @@ keywords:
   - Networking
   - Node
 
-version: 2.0.7
+version: 2.1.0
 kubeVersion: ">= 1.16.0-0"
-appVersion: "v1.3.1"
+appVersion: "v1.4.0"
 
 maintainers:
   - name: Ben Sykes
diff --git a/install/kubernetes/cidr-allocator/templates/nodecidrallocation.yaml b/install/kubernetes/cidr-allocator/templates/nodecidrallocation.yaml
@@ -7,4 +7,5 @@ metadata:
 spec:
   nodeSelector: {{ toYaml .nodeSelector | nindent 4 }}
   addressPools: {{ toYaml .addressPools | nindent 4 }}
+  staticAllocations: {{ toYaml .staticAllocations | nindent 4 }}
 {{ end }}
diff --git a/install/kubernetes/cidr-allocator/values.yaml b/install/kubernetes/cidr-allocator/values.yaml
@@ -204,3 +204,4 @@ nodeCIDRAllocations: []
   #     nodeSelector:
   #       kubernetes.io/os: "linux"
   #     addressPools: []
+  #     staticAllocations: []
diff --git a/internal/controller/nodecidrallocation_controller.go b/internal/controller/nodecidrallocation_controller.go
@@ -244,7 +244,7 @@ func (r *NodeCIDRAllocationReconciler) Reconcile(ctx context.Context, req ctrl.R
 			}
 
 			for _, subnet := range subnets {
-				networkAllocated, err := statcan_net.NetworkAllocated(subnet, &allClusterNodes)
+				networkAllocated, err := statcan_net.NetworkAllocated(subnet, &allClusterNodes, nodeCIDRAllocation.Spec.StaticAllocations)
 				if err != nil {
 					rl.Error(
 						err,
diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
@@ -81,6 +81,7 @@ func AvailableHostsPercent() prometheus.Gauge {
 func Update(nodeCIDRAllocations *v1alpha1.NodeCIDRAllocationList, allNodes *corev1.NodeList) {
 	var notAllocated uint64
 	nodeAllocationsCumulative := map[string]struct{}{}
+	overlappingStaticAllocationsCumulative := map[string]struct{}{}
 	addressPoolsCumulative := map[string]struct{}{} // like helper.ObjectContainsLabel(...), we are using a map-like DS to avoid duplicates
 	for _, n := range nodeCIDRAllocations.Items {
 		for _, p := range n.Spec.AddressPools {
@@ -89,6 +90,19 @@ func Update(nodeCIDRAllocations *v1alpha1.NodeCIDRAllocationList, allNodes *core
 	}
 	totalAvailableHosts := accumulatedHosts(helper.Keys(addressPoolsCumulative))
 
+	for _, n := range nodeCIDRAllocations.Items {
+		for _, p := range n.Spec.StaticAllocations {
+			for _, a := range n.Spec.AddressPools {
+				networksOverlap, _ := statcan_net.NetworksOverlap(p, a)
+
+				if networksOverlap {
+					overlappingStaticAllocationsCumulative[p] = struct{}{}
+				}
+			}
+		}
+	}
+	totalOverlappingStaticAllocations := accumulatedHosts(helper.Keys(overlappingStaticAllocationsCumulative))
+
 	for _, n := range allNodes.Items {
 		if n.Spec.PodCIDR == "" {
 			notAllocated++
@@ -101,7 +115,7 @@ func Update(nodeCIDRAllocations *v1alpha1.NodeCIDRAllocationList, allNodes *core
 	metricsExpectedAllocations.Set(float64(len(allNodes.Items)))
 	metricsActualAllocations.Set(float64(len(allNodes.Items) - int(notAllocated)))
 
-	remainingCount, remainingPercent := calculateRemainingHosts(totalAvailableHosts, totalAllocatedHosts)
+	remainingCount, remainingPercent := calculateRemainingHosts(totalAvailableHosts, totalAllocatedHosts, totalOverlappingStaticAllocations)
 	metricsAvailableHosts.Set(remainingCount)
 	metricsAvailableHostsPercent.Set(remainingPercent)
 }
@@ -128,8 +142,8 @@ func accumulatedHosts(networkCIDRs []string) uint32 {
 
 // calculateRemainingHosts is a helper function to calculate remaining hosts given total available hosts and the number of hosts that are already allocated.
 // this function returns both the total count of available hosts and a ratio (as a percent) of hosts left that are allocable.
-func calculateRemainingHosts(totalAvailableHosts, totalAllocatedHosts uint32) (float64, float64) {
-	return float64(totalAvailableHosts - totalAllocatedHosts), (1.0 - (float64(totalAllocatedHosts) / float64(totalAvailableHosts))) * 100
+func calculateRemainingHosts(totalAvailableHosts, totalAllocatedHosts, totalOverlappingStaticAllocations uint32) (float64, float64) {
+	return float64(totalAvailableHosts - totalOverlappingStaticAllocations - totalAllocatedHosts), (1.0 - (float64(totalAllocatedHosts) / float64(totalAvailableHosts-totalOverlappingStaticAllocations))) * 100
 }
 
 // GetMetricValue is a helper function from the metrics package to
diff --git a/internal/metrics/metrics_priv_test.go b/internal/metrics/metrics_priv_test.go
@@ -52,9 +52,9 @@ func TestAccumulatedHosts(t *testing.T) {
 
 func TestCalculateRemainingHosts(t *testing.T) {
 	// Case 1: 64 available hosts and 16 allocated hosts
-	// expected: should return (48, 75) where 48 is the number of remaining addresses and 75 is the percentage representation of remaining addresses
-	got, gotP := calculateRemainingHosts(64, 16)
-	var want, wantP float64 = 48, 75
+	// expected: should return (48, 75) where 48 is the number of remaining addresses and 50 is the percentage representation of remaining addresses
+	got, gotP := calculateRemainingHosts(64, 24, 16)
+	var want, wantP float64 = 24, 50
 
 	if got != want || gotP != wantP {
 		t.Errorf("got (%.0f, %.0f), wanted (%.0f, %.0f)", got, gotP, want, wantP)
diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go
@@ -44,8 +44,9 @@ func TestUpdate(t *testing.T) {
 				TypeMeta:   metav1.TypeMeta{},
 				ObjectMeta: metav1.ObjectMeta{},
 				Spec: v1alpha1.NodeCIDRAllocationSpec{
-					AddressPools: []string{"10.0.0.0/24"},
-					NodeSelector: map[string]string{},
+					AddressPools:      []string{"10.0.0.0/24"},
+					StaticAllocations: []string{},
+					NodeSelector:      map[string]string{},
 				},
 				Status: v1alpha1.NodeCIDRAllocationStatus{},
 			},
@@ -124,9 +125,10 @@ func TestUpdate(t *testing.T) {
 		t.Errorf("got (%.0f, %.0f%%), wanted (%.0f, %.0f%%)", actualVal, actualValPercent, expectedVal, expectedVal)
 	}
 
-	// Case 2: Address pools of 10.0.0.0/24 for 4 nodes where 1 does not have a PodCIDR set
+	// Case 2: Address pools of 10.0.0.0/24 for 4 nodes where 1 does not have a PodCIDR set & there is staticAllocations
 	// expected: should result in expected allocations not equal to actual allocations
 	// and the difference being 1. Additionally, the available hosts should be 64 and as a percent should be 25.
+	allocations.Items[0].Spec.StaticAllocations = []string{"10.0.0.252/30", "10.0.0.248/30", "10.0.0.244/30", "10.0.0.240/30", "10.0.1.238/30"}
 	nodes.Items[0].Spec.PodCIDR = ""
 	metrics.Update(allocations, nodes)
 
@@ -143,8 +145,8 @@ func TestUpdate(t *testing.T) {
 
 	actualVal = metrics.GetMetricValue(metrics.AvailableHosts())
 	actualValPercent = metrics.GetMetricValue(metrics.AvailableHostsPercent())
-	expectedVal = 64
-	expectedValPercent := float64(25)
+	expectedVal = 48
+	expectedValPercent := float64(19.999999999999996)
 
 	if expectedVal != actualVal {
 		t.Errorf("got %.0f, wanted %.0f", actualVal, expectedVal)
diff --git a/internal/networking/networking.go b/internal/networking/networking.go
@@ -98,7 +98,7 @@ func NetworksOverlap(a, b string) (bool, error) {
 // NetworkAllocated uses a variety of conditions to ensure that there is no
 // conflicting allocation that would present problems for subnet.
 // returns (true,nil) when the subnet provided is not allocated by or overlapping with any nodes.
-func NetworkAllocated(subnet string, nodes *corev1.NodeList) (bool, error) {
+func NetworkAllocated(subnet string, nodes *corev1.NodeList, staticAllocations []string) (bool, error) {
 	for _, n := range nodes.Items {
 		if n.Spec.PodCIDR == "" {
 			continue
@@ -114,5 +114,16 @@ func NetworkAllocated(subnet string, nodes *corev1.NodeList) (bool, error) {
 		}
 	}
 
+	for _, sa := range staticAllocations {
+		networksOverlap, err := NetworksOverlap(subnet, sa)
+		if err != nil {
+			return false, err
+		}
+
+		if networksOverlap {
+			return true, nil
+		}
+	}
+
 	return false, nil
 }
diff --git a/internal/networking/networking_test.go b/internal/networking/networking_test.go
@@ -187,6 +187,7 @@ func TestNetworksOverlap(t *testing.T) {
 
 func TestNetworkAllocated(t *testing.T) {
 	subnets := []string{"10.0.0.0/26", "10.0.0.64/26", "10.0.0.128/26", "10.0.0.192/26"}
+
 	nodeList := corev1.NodeList{
 		TypeMeta: metav1.TypeMeta{},
 		ListMeta: metav1.ListMeta{},
@@ -204,7 +205,7 @@ func TestNetworkAllocated(t *testing.T) {
 
 	// Case 1: All nodes specified do not have a PodCIDR in their spec
 	// expected: false
-	got, err := networking.NetworkAllocated(subnets[0], &nodeList)
+	got, err := networking.NetworkAllocated(subnets[0], &nodeList, []string{})
 	want := false
 	if err != nil {
 		t.Errorf("function was not expected to error. got %e", err)
@@ -218,7 +219,7 @@ func TestNetworkAllocated(t *testing.T) {
 		TypeMeta: metav1.TypeMeta{},
 		ListMeta: metav1.ListMeta{},
 		Items:    []corev1.Node{},
-	})
+	}, []string{})
 	want = false
 	if err != nil {
 		t.Errorf("function was not expected to error. got %e", err)
@@ -244,7 +245,7 @@ func TestNetworkAllocated(t *testing.T) {
 				Status: corev1.NodeStatus{},
 			},
 		},
-	})
+	}, []string{})
 	if err == nil {
 		t.Error("function was expected to return with an error")
 	}
@@ -264,7 +265,7 @@ func TestNetworkAllocated(t *testing.T) {
 				Status: corev1.NodeStatus{},
 			},
 		},
-	})
+	}, []string{})
 	want = true
 	if err != nil {
 		t.Errorf("function was not expected to error. got %e", err)
@@ -287,7 +288,53 @@ func TestNetworkAllocated(t *testing.T) {
 				Status: corev1.NodeStatus{},
 			},
 		},
-	})
+	}, []string{})
+	want = false
+	if err != nil {
+		t.Errorf("function was not expected to error. got %e", err)
+	} else if got != want {
+		t.Errorf("got %t, wanted %t", got, want)
+	}
+
+	// Case 6: Provided subnet does not intersect with any Node, but intersects with an static allocation
+	// expected: true
+	got, err = networking.NetworkAllocated(subnets[0], &corev1.NodeList{
+		TypeMeta: metav1.TypeMeta{},
+		ListMeta: metav1.ListMeta{},
+		Items: []corev1.Node{
+			{
+				TypeMeta:   metav1.TypeMeta{},
+				ObjectMeta: metav1.ObjectMeta{},
+				Spec: corev1.NodeSpec{
+					PodCIDR: "10.0.0.64/28",
+				},
+				Status: corev1.NodeStatus{},
+			},
+		},
+	}, []string{"10.0.1.0/26", "10.0.0.8/30"})
+	want = true
+	if err != nil {
+		t.Errorf("function was not expected to error. got %e", err)
+	} else if got != want {
+		t.Errorf("got %t, wanted %t", got, want)
+	}
+
+	// Case 7: Provided subnet does not intersect with any Node and does not intersects with an static allocation
+	// expected: false
+	got, err = networking.NetworkAllocated(subnets[0], &corev1.NodeList{
+		TypeMeta: metav1.TypeMeta{},
+		ListMeta: metav1.ListMeta{},
+		Items: []corev1.Node{
+			{
+				TypeMeta:   metav1.TypeMeta{},
+				ObjectMeta: metav1.ObjectMeta{},
+				Spec: corev1.NodeSpec{
+					PodCIDR: "10.0.0.64/28",
+				},
+				Status: corev1.NodeStatus{},
+			},
+		},
+	}, []string{"10.0.1.0/26", "10.0.1.64/26"})
 	want = false
 	if err != nil {
 		t.Errorf("function was not expected to error. got %e", err)

Original file line number	Diff line number	Diff line change
`@@ -244,7 +244,7 @@ func (r *NodeCIDRAllocationReconciler) Reconcile(ctx context.Context, req ctrl.R`
`244`	`244`	`}`
`245`	`245`
`246`	`246`	`for _, subnet := range subnets {`
`247`		`- networkAllocated, err := statcan_net.NetworkAllocated(subnet, &allClusterNodes)`
	`247`	`+ networkAllocated, err := statcan_net.NetworkAllocated(subnet, &allClusterNodes, nodeCIDRAllocation.Spec.StaticAllocations)`
`248`	`248`	`if err != nil {`
`249`	`249`	`rl.Error(`
`250`	`250`	`err,`