Merge branch 'master' into community

Author: khaf

.gitignore

@@ -8,3 +8,13 @@ tmp/
 .DS_Store
 TODO
 PLAN.todo
+.go_metalinter
+.go_style
+.golangci.yml
+.travis/read-write-udf.ldif
+.travis/people.ldif
+.travis/modify.ldif
+.travis/badwan.ldif
+.travis/access.ldif
+tools/
+cmd/

CHANGELOG.md

@@ -1,5 +1,33 @@
 # Change History
 
+## February 12 2021: v4.2.0
+
+  Major feature and improvements release.
+
+  * **New Features**:
+
+    - [CLIENT-1192] Adds Support for partition scans. Queries which lack a `Statement.Filter` will be automatically converted to partition scans. If the cluster supports partition scans, all Scans and Queries will use the new protocol to allow retrying in case of some errors.
+    - [CLIENT-1237] Adds Support for `MultiPolicy.MaxRecords` in scans and queries without `Statement.Filter`.
+    - Adds `NewHosts` convenience function. (Github #320) thanks to [Yegor Myskin](https://github.com/un000)
+
+  * **Improvements**
+
+    - Adds a few missing error checks.
+    - Moves examples files to dedicated folders to avoid `multiple main function` errors for new users.
+    - Some documentation clean up. (Github #314) thanks to [Shin Uozumi](https://github.com/sinozu)
+    - Fix typo in example `NewKey()`. (Github #331) thanks to [Patrick Kuca](https://github.com/pkuca)
+    - Adds an example to list operations (using operate and list policy).
+    - Runs the XDR tests only when XDR is configured on the server.
+    - Add TLS config to test params.
+    - Mark `NewPredExpXXX` return value as the PredExp interface instead of concrete type. It will now group them under the `PredExp` interface in the docs.
+
+  * **Changes**
+    - Only use `Policy.Priority` and `MultiPolicy.FailOnClusterChange` on server versions < 4.9. `Priority` is now deprecated and replaced with `MultiPolicy.RecordPerSecond`.
+    - `Statement.TaskID` is deprecated and will be removed in the next major version.
+    - `ScanPolicy.ConcurrentNodes` is deprecated and will be removed in the next major version.
+
+… versions < 4.9
+
 ## January 25 2021: v4.1.0
 
   Major feature release.
@@ -10,6 +38,10 @@
     - [CLIENT-1410] Adds `Client.SetXDRFilter()`.
     - [CLIENT-1433] Adds `ExpMemorySize()` to expression filters.
 
+  * **Fixes**
+
+    - Fixes an issue where remainder miscalculation would cause the connection pool to be smaller than it should have been. (Github #332) thanks to [ShawnZhang](https://github.com/xqzhang2015)
+
   * **Improvements**
 
     - [CLIENT-1434] Reset peers, partition and rebalance generations on node tend errors.

README.md

@@ -39,7 +39,7 @@ Please refer to the [`CHANGELOG.md`](CHANGELOG.md) for details.
   - [Tools](#Tools)
 
 
-## Usage:
+## Usage
 
 The following is a very simple example of CRUD operations in an Aerospike database.
 
@@ -114,7 +114,7 @@ Supported operating systems:
 - Windows (untested)
 
 <a name="Installation"></a>
-## Installation:
+## Installation
 
 1. Install Go 1.9+ and setup your environment as [Documented](http://golang.org/doc/code.html#GOPATH) here.
 2. Get the client in your ```GOPATH``` : ```go get github.com/aerospike/aerospike-client-go```

admin_command.go

@@ -1,4 +1,4 @@
-// Copyright 2013-2020 Aerospike, Inc.
+// Copyright 2014-2021 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use acmd file except in compliance with the License.

admin_policy.go

@@ -1,4 +1,4 @@
-// Copyright 2013-2020 Aerospike, Inc.
+// Copyright 2014-2021 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

aerospike.go

@@ -1,16 +1,3 @@
-// Copyright 2013-2020 Aerospike, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
 // Package aerospike provides a client to connect and interact with an Aerospike cluster.
+// This is the official Go implementation of the Aerospike Client.
 package aerospike

aerospike_bench_reflect_test.go

@@ -1,6 +1,6 @@
 // +build !as_performance
 
-// Copyright 2013-2020 Aerospike, Inc.
+// Copyright 2014-2021 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

aerospike_bench_test.go

@@ -1,4 +1,4 @@
-// Copyright 2013-2020 Aerospike, Inc.
+// Copyright 2014-2021 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

aerospike_suite_test.go

@@ -1,4 +1,4 @@
-// Copyright 2013-2020 Aerospike, Inc.
+// Copyright 2014-2021 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,7 +16,12 @@ package aerospike_test
 
 import (
 	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
 	"flag"
+	"fmt"
+	"io/ioutil"
 	"log"
 	"math/rand"
 	"os"
@@ -29,6 +34,7 @@ import (
 
 	as "github.com/aerospike/aerospike-client-go"
 	asl "github.com/aerospike/aerospike-client-go/logger"
+	ast "github.com/aerospike/aerospike-client-go/types"
 )
 
 var (
@@ -41,6 +47,12 @@ var (
 	debug       = flag.Bool("debug", false, "Will set the logging level to DEBUG.")
 	namespace   = flag.String("n", "test", "Namespace")
 
+	certFile          = flag.String("cert_file", "", "Certificate file name.")
+	keyFile           = flag.String("key_file", "", "Key file name.")
+	keyFilePassphrase = flag.String("key_file_passphrase", "", "Key file's pass phrase.")
+	nodeTLSName       = flag.String("node_tls_name", "", "Node's TLS name.")
+	rootCA            = flag.String("root_ca", "", "Root certificate.")
+
 	clientPolicy *as.ClientPolicy
 	client       *as.Client
 )
@@ -70,7 +82,13 @@ func initTestVars() {
 		clientPolicy.AuthMode = as.AuthModeExternal
 	}
 
-	client, err = as.NewClientWithPolicy(clientPolicy, *host, *port)
+	// setup TLS
+	clientPolicy.TlsConfig = initTLS()
+
+	dbHost := as.NewHost(*host, *port)
+	dbHost.TLSName = *nodeTLSName
+
+	client, err = as.NewClientWithPolicyAndHost(clientPolicy, dbHost)
 	if err != nil {
 		log.Fatal(err.Error())
 	}
@@ -121,6 +139,11 @@ func securityEnabled() bool {
 	return err == nil
 }
 
+func xdrEnabled() bool {
+	res := info(client, "get-config:context=xdr")
+	return len(res) > 0 && !strings.HasPrefix(res, "ERROR")
+}
+
 func nsInfo(ns string, feature string) string {
 	node := client.GetNodes()[0]
 	infoMap, err := node.RequestInfo(as.NewInfoPolicy(), "namespace/"+ns)
@@ -144,8 +167,110 @@ func info(client *as.Client, feature string) string {
 	node := client.GetNodes()[0]
 	infoMap, err := node.RequestInfo(as.NewInfoPolicy(), feature)
 	if err != nil {
-		log.Fatal("Failed to connect to aerospike: err:", err)
+		if ae, ok := err.(ast.AerospikeError); ok {
+			return ae.Error()
+		} else {
+			log.Fatal("Failed to connect to aerospike: err:", err)
+		}
 	}
 
 	return infoMap[feature]
 }
+
+func initTLS() *tls.Config {
+	if len(*rootCA) == 0 && len(*certFile) == 0 && len(*keyFile) == 0 {
+		return nil
+	}
+
+	// Try to load system CA certs, otherwise just make an empty pool
+	serverPool, err := x509.SystemCertPool()
+	if serverPool == nil || err != nil {
+		log.Printf("Adding system certificates to the cert pool failed: %s", err)
+		serverPool = x509.NewCertPool()
+	}
+
+	if len(*rootCA) > 0 {
+		// Try to load system CA certs and add them to the system cert pool
+		caCert, err := readFromFile(*rootCA)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		log.Printf("Adding CA certificate to the pool...")
+		serverPool.AppendCertsFromPEM(caCert)
+	}
+
+	var clientPool []tls.Certificate
+	if len(*certFile) > 0 || len(*keyFile) > 0 {
+
+		// Read cert file
+		certFileBytes, err := readFromFile(*certFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// Read key file
+		keyFileBytes, err := readFromFile(*keyFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// Decode PEM data
+		keyBlock, _ := pem.Decode(keyFileBytes)
+		certBlock, _ := pem.Decode(certFileBytes)
+
+		if keyBlock == nil || certBlock == nil {
+			log.Fatalf("Failed to decode PEM data for key or certificate")
+		}
+
+		// Check and Decrypt the the Key Block using passphrase
+		if x509.IsEncryptedPEMBlock(keyBlock) {
+			decryptedDERBytes, err := x509.DecryptPEMBlock(keyBlock, []byte(*keyFilePassphrase))
+			if err != nil {
+				log.Fatalf("Failed to decrypt PEM Block: `%s`", err)
+			}
+
+			keyBlock.Bytes = decryptedDERBytes
+			keyBlock.Headers = nil
+		}
+
+		// Encode PEM data
+		keyPEM := pem.EncodeToMemory(keyBlock)
+		certPEM := pem.EncodeToMemory(certBlock)
+
+		if keyPEM == nil || certPEM == nil {
+			log.Fatalf("Failed to encode PEM data for key or certificate")
+		}
+
+		cert, err := tls.X509KeyPair(certPEM, keyPEM)
+
+		if err != nil {
+			log.Fatalf("Failed to add client certificate and key to the pool: `%s`", err)
+		}
+
+		log.Printf("Adding client certificate and key to the pool...")
+		clientPool = append(clientPool, cert)
+	}
+
+	tlsConfig := &tls.Config{
+		Certificates:             clientPool,
+		RootCAs:                  serverPool,
+		InsecureSkipVerify:       false,
+		PreferServerCipherSuites: true,
+	}
+	tlsConfig.BuildNameToCertificate()
+
+	return tlsConfig
+}
+
+// Read content from file
+func readFromFile(filePath string) ([]byte, error) {
+	dataBytes, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to read from file `%s`: `%v`", filePath, err)
+	}
+
+	data := bytes.TrimSuffix(dataBytes, []byte("\n"))
+
+	return data, nil
+}

anonymous_fields_test.go

@@ -1,6 +1,6 @@
 // +build !as_performance
 
-// Copyright 2013-2019 Aerospike, Inc.
+// Copyright 2014-2019 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

auth_mode.go

@@ -1,4 +1,4 @@
-// Copyright 2013-2020 Aerospike, Inc.
+// Copyright 2014-2021 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

batch_command.go

@@ -1,4 +1,4 @@
-// Copyright 2013-2020 Aerospike, Inc.
+// Copyright 2014-2021 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

batch_command_exists.go

@@ -1,4 +1,4 @@
-// Copyright 2013-2020 Aerospike, Inc.
+// Copyright 2014-2021 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -45,7 +45,6 @@ func newBatchCommandExists(
 		keys:        keys,
 		existsArray: existsArray,
 	}
-	res.oneShot = false
 	return res
 }
 

batch_command_get.go

@@ -1,4 +1,4 @@
-// Copyright 2013-2020 Aerospike, Inc.
+// Copyright 2014-2021 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -68,7 +68,6 @@ func newBatchCommandGet(
 		records:  records,
 		readAttr: readAttr,
 	}
-	res.oneShot = false
 	return res
 }
 

batch_command_get_reflect.go

@@ -1,6 +1,6 @@
 // +build !as_performance
 
-// Copyright 2013-2020 Aerospike, Inc.
+// Copyright 2014-2021 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

batch_command_reflect.go

@@ -1,6 +1,6 @@
 // +build !as_performance
 
-// Copyright 2013-2020 Aerospike, Inc.
+// Copyright 2014-2021 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

batch_index_command_get.go

@@ -1,4 +1,4 @@
-// Copyright 2013-2020 Aerospike, Inc.
+// Copyright 2014-2021 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -39,7 +39,6 @@ func newBatchIndexCommandGet(
 			indexRecords: records,
 		},
 	}
-	res.oneShot = false
 	return res
 }
 

batch_node.go

@@ -1,4 +1,4 @@
-// Copyright 2013-2020 Aerospike, Inc.
+// Copyright 2014-2021 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

batch_policy.go

@@ -1,4 +1,4 @@
-// Copyright 2013-2020 Aerospike, Inc.
+// Copyright 2014-2021 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

batch_read.go

@@ -1,4 +1,4 @@
-// Copyright 2013-2020 Aerospike, Inc.
+// Copyright 2014-2021 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.