CCBC-1350: an option for cbc-n1ql to control prepared/adhoc switch

Change-Id: Iff3319173ad364b26c2fe7d0a00c462bbdc72f0a
Reviewed-on: http://review.couchbase.org/c/libcouchbase/+/142170
Tested-by: Build Bot <build@couchbase.com>
Reviewed-by: Sergey Avseyev <sergey.avseyev@gmail.com>

Author: avsej

doc/cbc-n1qlback.markdown

@@ -21,6 +21,15 @@ For simple queries, only the `statement` field needs to be set:
 For more complex queries (for example, placeholders, custom options), you may
 refer to the N1QL REST API reference.
 
+For example, the following line shows how to use named parameter
+
+    {"statement":"SELECT RAW meta().id FROM `travel-sample` WHERE type=$type LIMIT 1", "$type":"airline"}
+
+There is also special query option, `"n1qlback"`, which is stripped before sending the payload.
+Currently it allows to tell if the particular query must be prepared before execution:
+
+    {"statement":"SELECT * FROM `travel-sample` WHERE type=$type LIMIT 10", "$type":"airline", "n1qlback": {"prepare": true}}
+
 `n1qlback` requires that any resources (data items, indexes) are already
 defined.
 
@@ -64,16 +73,19 @@ command-line
 The following will create a file with 3 queries and 5 threads alternating
 between them. It also creates indexes on the `travel-sample` bucket
 
-    cbc n1ql -U couchbase://192.168.72.101/a_bucket 'CREATE INDEX ix_name ON `travel-sample`(name)'
-    cbc n1ql -U couchbase://192.168.72.101/a_bucket 'CREATE INDEX ix_country ON `travel-sample`(country)'
+    cbc n1ql 'CREATE INDEX ix_name ON `travel-sample`(name)'
+    cbc n1ql 'CREATE INDEX ix_country ON `travel-sample`(country)'
+
+Crete text file `queries.txt` with the following content (note that fourth query has parameter)
 
-    cat queries.txt <<EOF
     {"statement":"SELECT country FROM `travel-sample` WHERE `travel-sample`.country = \"United States\""}
     {"statement":"SELECT name FROM `travel-sample` LIMIT 10"}
     {"statement":"SELECT country, COUNT(country) FROM `travel-sample` GROUP BY country"}
-    EOF
+    {"statement":"SELECT RAW meta().id FROM `travel-sample` WHERE type=$type LIMIT 1", "$type":"airline", "n1qlback": {"prepare": true}}
+
+Run the test load
 
-    cbc-n1qlback -U couchbase://192.168.72.101/a_bucket -t 5 -f queries.txt
+    cbc-n1qlback --num-threads 5 --queryfile queries.txt
 
 ## BUGS
 

doc/cbc-pillowfight.markdown

@@ -125,7 +125,7 @@ The following options control workload generation:
 
 * `d`, `--durability`=_LEVEL_:
   Specify durability level for mutation operations. Known values are: "none",
-  "majority", "majority\_and\_persist\_on\_master", "persist\_to\_majority".
+  "majority", "majority\_and\_persist\_to\_active", "persist\_to\_majority".
 
 * `p`, `--persist-to`=_NUMNODES_:
   Wait until the item has been persisted to at least `NUMNODES` nodes' disk. If

doc/cbc.markdown

@@ -124,7 +124,7 @@ In addition to the options in the [OPTIONS](#OPTIONS) section, the following opt
 
 * `d`, `--durability`=_LEVEL_:
   Specify durability level for mutation operations. Known values are: "none",
-  "majority", "majority\_and\_persist\_on\_master", "persist\_to\_majority".
+  "majority", "majority\_and\_persist\_to\_active", "persist\_to\_majority".
 
 * `p`, `--persist-to`=_NUMNODES_:
   Wait until the item has been persisted to at least `NUMNODES` nodes' disk. If

doc/man/cbc-n1qlback.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "CBC\-N1QLBACK" "1" "April 2019" "" ""
+.TH "CBC\-N1QLBACK" "1" "December 2020" "" ""
 .
 .SH "NAME"
 \fBcbc\-n1qlback\fR \- Stress Test for Couchbase Query (N1QL)
@@ -30,6 +30,32 @@
 For more complex queries (for example, placeholders, custom options), you may refer to the N1QL REST API reference\.
 .
 .P
+For example, the following line shows how to use named parameter
+.
+.IP "" 4
+.
+.nf
+
+{"statement":"SELECT RAW meta()\.id FROM `travel\-sample` WHERE type=$type LIMIT 1", "$type":"airline"}
+.
+.fi
+.
+.IP "" 0
+.
+.P
+There is also special query option, \fB"n1qlback"\fR, which is stripped before sending the payload\. Currently it allows to tell if the particular query must be prepared before execution:
+.
+.IP "" 4
+.
+.nf
+
+{"statement":"SELECT * FROM `travel\-sample` WHERE type=$type LIMIT 10", "$type":"airline", "n1qlback": {"prepare": true}}
+.
+.fi
+.
+.IP "" 0
+.
+.P
 \fBn1qlback\fR requires that any resources (data items, indexes) are already defined\.
 .
 .SH "OPTIONS"
@@ -176,16 +202,37 @@ The following will create a file with 3 queries and 5 threads alternating betwee
 .
 .nf
 
-cbc n1ql \-U couchbase://192\.168\.72\.101/a_bucket \'CREATE INDEX ix_name ON `travel\-sample`(name)\'
-cbc n1ql \-U couchbase://192\.168\.72\.101/a_bucket \'CREATE INDEX ix_country ON `travel\-sample`(country)\'
+cbc n1ql \'CREATE INDEX ix_name ON `travel\-sample`(name)\'
+cbc n1ql \'CREATE INDEX ix_country ON `travel\-sample`(country)\'
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Crete text file \fBqueries\.txt\fR with the following content (note that fourth query has parameter)
+.
+.IP "" 4
+.
+.nf
 
-cat queries\.txt <<EOF
 {"statement":"SELECT country FROM `travel\-sample` WHERE `travel\-sample`\.country = \e"United States\e""}
 {"statement":"SELECT name FROM `travel\-sample` LIMIT 10"}
 {"statement":"SELECT country, COUNT(country) FROM `travel\-sample` GROUP BY country"}
-EOF
+{"statement":"SELECT RAW meta()\.id FROM `travel\-sample` WHERE type=$type LIMIT 1", "$type":"airline", "n1qlback": {"prepare": true}}
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Run the test load
+.
+.IP "" 4
+.
+.nf
 
-cbc\-n1qlback \-U couchbase://192\.168\.72\.101/a_bucket \-t 5 \-f queries\.txt
+cbc\-n1qlback \-\-num\-threads 5 \-\-queryfile queries\.txt
 .
 .fi
 .

tools/CMakeLists.txt

@@ -13,7 +13,7 @@ ADD_EXECUTABLE(cbc-pillowfight cbc-pillowfight.cc
 TARGET_LINK_LIBRARIES(cbc-pillowfight couchbase)
 
 ADD_EXECUTABLE(cbc-n1qlback cbc-n1qlback.cc
-    $<TARGET_OBJECTS:lcbtools> $<TARGET_OBJECTS:cliopts>)
+    $<TARGET_OBJECTS:lcbtools> $<TARGET_OBJECTS:cliopts> $<TARGET_OBJECTS:lcb_jsoncpp>)
 TARGET_LINK_LIBRARIES(cbc-n1qlback couchbase)
 
 INSTALL(TARGETS cbc cbc-pillowfight cbc-n1qlback

tools/cbc-n1qlback.cc

@@ -35,6 +35,7 @@
 #endif
 #include "common/options.h"
 #include "common/histogram.h"
+#include "contrib/lcb-jsoncpp/lcb-jsoncpp.h"
 
 using namespace cbc;
 using namespace cliopts;
@@ -56,6 +57,11 @@ static void do_or_die(lcb_STATUS rc)
     }
 }
 
+struct Query {
+    std::string payload{};
+    bool prepare{false};
+};
+
 class Metrics
 {
   public:
@@ -237,7 +243,22 @@ class Configuration
         while (ifs.good()) {
             std::getline(ifs, curline);
             if (!curline.empty()) {
-                m_queries.push_back(curline);
+                Json::Value json;
+                if (!Json::Reader().parse(curline, json)) {
+                    std::cerr << "Failed to parse query \"" << curline << "\" as JSON, skipping" << std::endl;
+                    continue;
+                }
+                Query query{};
+                Json::Value options = json.get("n1qlback", Json::nullValue);
+                if (options.isObject()) {
+                    Json::Value should_prepare = options.get("prepare", Json::nullValue);
+                    if (should_prepare.isBool()) {
+                        query.prepare = should_prepare.asBool();
+                    }
+                }
+                json.removeMember("n1qlback");
+                query.payload = Json::FastWriter().write(json);
+                m_queries.emplace_back(query);
             }
         }
         std::cerr << "Loaded " << m_queries.size() << " queries "
@@ -255,14 +276,15 @@ class Configuration
                 errstr += strerror(ec_save);
                 throw std::runtime_error(errstr);
             }
+            std::cerr << "Errors will be logged in \"" << o_errlog.const_result() << "\"" << std::endl;
         }
     }
 
-    void set_cropts(lcb_CREATEOPTS *opts)
+    void set_cropts(lcb_CREATEOPTS *&opts)
     {
         m_params.fillCropts(opts);
     }
-    const vector<string> &queries() const
+    const vector<Query> &queries() const
     {
         return m_queries;
     }
@@ -281,7 +303,7 @@ class Configuration
     }
 
   private:
-    vector<string> m_queries;
+    vector<Query> m_queries;
     StringOption o_file;
     UIntOption o_threads;
     ConnParams m_params;
@@ -312,9 +334,8 @@ class ThreadContext
     void run()
     {
         while (!m_cancelled) {
-            vector<string>::const_iterator ii = m_queries.begin();
-            for (; ii != m_queries.end(); ++ii) {
-                run_one_query(*ii);
+            for (const auto &query : m_queries) {
+                run_one_query(query);
             }
         }
     }
@@ -388,8 +409,7 @@ class ThreadContext
         }
     }
 
-    ThreadContext(Metrics &metrics, lcb_INSTANCE *instance, const vector<string> &initial_queries,
-                  std::ofstream *errlog)
+    ThreadContext(Metrics &metrics, lcb_INSTANCE *instance, const vector<Query> &initial_queries, std::ofstream *errlog)
         : m_instance(instance), last_nrow(0), m_cmd(nullptr), m_metrics(metrics), m_cancelled(false), m_thr(nullptr),
           m_errlog(errlog)
     {
@@ -424,19 +444,20 @@ class ThreadContext
         }
     }
 
-    void run_one_query(const string &txt)
+    void run_one_query(const Query &query)
     {
         // Reset counters
         last_nrow = 0;
 
-        lcb_cmdquery_payload(m_cmd, txt.c_str(), txt.size());
+        lcb_cmdquery_payload(m_cmd, query.payload.c_str(), query.payload.size());
+        lcb_cmdquery_adhoc(m_cmd, !query.prepare);
 
         // Set up our context
         QueryContext qctx(this);
 
         lcb_STATUS rc = lcb_query(m_instance, &qctx, m_cmd);
         if (rc != LCB_SUCCESS) {
-            log_error(rc, txt.c_str(), txt.size());
+            log_error(rc, query.payload.c_str(), query.payload.size());
         } else {
             lcb_wait(m_instance, LCB_WAIT_DEFAULT);
             m_metrics.lock();
@@ -447,7 +468,7 @@ class ThreadContext
     }
 
     lcb_INSTANCE *m_instance;
-    vector<string> m_queries;
+    vector<Query> m_queries;
     size_t last_nrow;
     lcb_CMDQUERY *m_cmd;
     Metrics &m_metrics;

tools/common/options.cc

@@ -303,9 +303,9 @@ void ConnParams::fillCropts(lcb_CREATEOPTS *&cropts)
         string host = o_host.result();
         string bucket = o_bucket.result();
 
-        for (size_t ii = 0; ii < host.size(); ++ii) {
-            if (host[ii] == ';') {
-                host[ii] = ',';
+        for (char &ii : host) {
+            if (ii == ';') {
+                ii = ',';
             }
         }
 
@@ -374,9 +374,9 @@ void ConnParams::fillCropts(lcb_CREATEOPTS *&cropts)
         connstr += '&';
     }
 
-    const std::vector< std::string > &extras = o_cparams.const_result();
-    for (size_t ii = 0; ii < extras.size(); ii++) {
-        connstr += extras[ii];
+    const std::vector<std::string> &extras = o_cparams.const_result();
+    for (const auto &extra : extras) {
+        connstr += extra;
         connstr += '&';
     }
 
@@ -396,7 +396,8 @@ void ConnParams::fillCropts(lcb_CREATEOPTS *&cropts)
     lcb_createopts_credentials(cropts, NULL, 0, passwd.c_str(), passwd.size());
 }
 
-template < typename T > void doPctl(lcb_INSTANCE *instance, int cmd, T arg)
+template <typename T>
+void doPctl(lcb_INSTANCE *instance, int cmd, T arg)
 {
     lcb_STATUS err;
     err = lcb_cntl(instance, LCB_CNTL_SET, cmd, (void *)arg);
@@ -405,9 +406,10 @@ template < typename T > void doPctl(lcb_INSTANCE *instance, int cmd, T arg)
     }
 }
 
-template < typename T > void doSctl(lcb_INSTANCE *instance, int cmd, T arg)
+template <typename T>
+void doSctl(lcb_INSTANCE *instance, int cmd, T arg)
 {
-    doPctl< T * >(instance, cmd, &arg);
+    doPctl<T *>(instance, cmd, &arg);
 }
 
 void doStringCtl(lcb_INSTANCE *instance, const char *s, const char *val)
@@ -423,11 +425,11 @@ lcb_STATUS ConnParams::doCtls(lcb_INSTANCE *instance)
 {
     try {
         if (o_saslmech.passed()) {
-            doPctl< const char * >(instance, LCB_CNTL_FORCE_SASL_MECH, o_saslmech.result().c_str());
+            doPctl<const char *>(instance, LCB_CNTL_FORCE_SASL_MECH, o_saslmech.result().c_str());
         }
 
         // Set the detailed error codes option
-        doSctl< int >(instance, LCB_CNTL_DETAILED_ERRCODES, 1);
+        doSctl<int>(instance, LCB_CNTL_DETAILED_ERRCODES, 1);
 
         if (!o_connstr.passed() || o_connstr.result().find("compression=") == std::string::npos) {
             int opts = LCB_COMPRESS_IN;