Merge pull request #4378 from esl/cockroachdb-preset

Adding cockroachdb_cets preset

Author: chrzaszcz

Diff for: .circleci/template.yml

@@ -54,6 +54,26 @@ containers:
       - ENV_FILE_KEY_PATH: "/tmp/sql/fake_key.pem"
       - ENV_FILE_KEY_DATA: "__MIM_KEY__"
     entrypoint: *ENTRYPOINT
+  - &cockroachdb_container
+    image: cockroachdb/cockroach:__COCKROACHDB_VERSION__
+    environment:
+      - COCKROACH_DATABASE: mongooseim
+      - OLD_ENTRYPOINT: "chmod 777 /start.sh && /start.sh"
+      - ENV_FILE_SETUP_PATH: "/start.sh"
+      - ENV_FILE_SETUP_DATA: "__COCKROACHDB_SETUP__"
+      - ENV_FILE_SQL_PATH: "/docker-entrypoint-initdb.d/init.sql"
+      - ENV_FILE_SQL_DATA: "__COCKROACHDB_SQL__"
+      - ENV_FILE_USER_PATH: "/docker-entrypoint-initdb.d/create_user.sql"
+      - ENV_FILE_USER_DATA: "__COCKROACHDB_USER_SQL__"
+      - ENV_FILE_CACERT_PATH: "/tmp/ca.key"
+      - ENV_FILE_CACERT_DATA: "__DB_CAKEY__"
+      - ENV_FILE_CAKEY_PATH: "/tmp/ca.crt"
+      - ENV_FILE_CAKEY_DATA: "__DB_CACERT__"
+      - ENV_FILE_CERT_PATH: "/tmp/client.mongooseim.crt"
+      - ENV_FILE_CERT_DATA: "__MIM_CERT__"
+      - ENV_FILE_KEY_PATH: "/tmp/client.mongooseim.key"
+      - ENV_FILE_KEY_DATA: "__MIM_KEY__"
+    entrypoint: *ENTRYPOINT
   - &ldap_container
     image: osixia/openldap:__LDAP_VERSION__
     environment:
@@ -185,6 +205,11 @@ executors:
       - image: *OTP27
       - *redis_container
       - *pgsql_container
+  otp_27_cockroachdb_redis:
+    docker:
+      - image: *OTP27
+      - *redis_container
+      - *cockroachdb_container
   otp_26_ldap_redis:
     docker:
       - image: *OTP26
@@ -552,7 +577,8 @@ jobs:
       preset:
         type: enum
         enum: [internal_mnesia, mysql_redis, odbc_mssql_mnesia, ldap_mnesia,
-               elasticsearch_and_cassandra_mnesia, pgsql_mnesia, pgsql_cets]
+               elasticsearch_and_cassandra_mnesia, pgsql_mnesia, pgsql_cets,
+               cockroachdb_cets]
         description: Preset to run
         default: internal_mnesia
       db:
@@ -843,6 +869,15 @@ workflows:
           requires:
             - otp_27_docker
           filters: *all_tags
+      - big_tests_in_docker:
+          name: cockroachdb_cets_27
+          executor: otp_27_cockroachdb_redis
+          context: mongooseim-org
+          preset: cockroachdb_cets
+          db: "mnesia cockroachdb cets"
+          requires:
+            - otp_27_docker
+          filters: *all_tags
       - big_tests_in_docker:
           name: pgsql_cets_27
           executor: otp_27_pgsql_redis
@@ -966,6 +1001,7 @@ workflows:
             - small_tests_27
             - internal_mnesia_27
             - pgsql_cets_27
+            - cockroachdb_cets_27
             - pgsql_mnesia_27
             - mysql_redis_27
             - mssql_mnesia_27

Diff for: Makefile

@@ -59,7 +59,7 @@ $(DEVNODES): certs configure.out rel/vars-toml.config
 
 maybe_clean_certs:
 	if [ "$$SKIP_CERT_BUILD" != 1 ]; then \
-		if ! openssl x509 -checkend 3600 -noout -in tools/ssl/ca/cacert.pem ; then \
+		if ! openssl x509 -checkend 36000 -noout -in tools/ssl/ca/cacert.pem ; then \
 			cd tools/ssl && make clean_certs; \
 		fi \
 	fi

Diff for: big_tests/test.config

@@ -265,6 +265,36 @@
   connection.password = \"mongooseim_secret\"
   connection.tls.required = true
   connection.tls.cacertfile = \"priv/ssl/cacert.pem\"
+  connection.tls.server_name_indication.enabled = false"},
+      {service_domain_db, ""},
+      {mod_vcard, "  backend = \"rdbms\"
+  host = \"vjud.@HOST@\"\n"},
+      {mod_roster, "  backend = \"rdbms\"\n"}]},
+    {cockroachdb_cets,
+     [{dbs, [redis, cockroachdb]},
+      {sm_backend, "\"cets\""},
+      {bosh_backend, "\"cets\""},
+      {component_backend, "\"cets\""},
+      {s2s_backend, "\"cets\""},
+      {stream_management_backend, cets},
+      {auth_anonymous_backend, cets},
+      {auth_method, "rdbms"},
+      {internal_databases, "[internal_databases.cets]
+  cluster_name = \"{{cluster_name}}\""},
+      {outgoing_pools, "[outgoing_pools.redis.global_distrib]
+  scope = \"global\"
+  workers = 10
+[outgoing_pools.rdbms.default]
+  scope = \"global\"
+  workers = 5
+  connection.driver = \"cockroachdb\"
+  connection.host = \"localhost\"
+  connection.port = 26257
+  connection.database = \"mongooseim\"
+  connection.username = \"mongooseim\"
+  connection.password = \"mongooseim_secret\"
+  connection.tls.required = true
+  connection.tls.cacertfile = \"priv/ssl/cacert.pem\"
   connection.tls.server_name_indication.enabled = false"},
       {service_domain_db, ""},
       {mod_vcard, "  backend = \"rdbms\"

Diff for: big_tests/tests/muc_SUITE.erl

@@ -386,13 +386,13 @@ init_per_group(disco_rsm_with_offline, Config) ->
 init_per_group(G, Config) when G =:= http_auth_no_server;
                                G =:= http_auth ->
     PoolOpts = #{strategy => available_worker, workers => 5},
-    ConnOpts = #{host => "http://localhost:8080", path_prefix => <<"/muc/auth/">>,
+    ConnOpts = #{host => "http://localhost:8081", path_prefix => <<"/muc/auth/">>,
                  request_timeout => 2000},
     Pool = config([outgoing_pools, http, muc_http_auth_test],
                   #{opts => PoolOpts, conn_opts => ConnOpts}),
     [{ok, _Pid}] = rpc(mim(), mongoose_wpool, start_configured_pools, [[Pool]]),
     case G of
-        http_auth -> http_helper:start(8080, "/muc/auth/check_password", fun handle_http_auth/1);
+        http_auth -> http_helper:start(8081, "/muc/auth/check_password", fun handle_http_auth/1);
         _ -> ok
     end,
     ConfigWithModules = dynamic_modules:save_modules(host_type(), Config),

Diff for: big_tests/tests/rdbms_SUITE.erl

@@ -226,7 +226,7 @@ binary_values() ->
     binary:copy(<<0>>, 100000),
     null
     ] ++
-    case is_odbc() orelse is_pgsql() of
+    case is_odbc() orelse is_pgsql() orelse is_cockroachdb() of
         true ->
             [];
         false ->
@@ -1277,6 +1277,9 @@ is_pgsql() ->
 is_mysql() ->
     db_engine() == mysql.
 
+is_cockroachdb() ->
+    db_engine() == cockroachdb.
+
 stop_global_default_pool() ->
     Pools = rpc(mim(), mongoose_config, get_opt, [outgoing_pools]),
     [GlobalRdbmsPool] = [Pool || Pool = #{type := rdbms, scope := global, tag := default} <- Pools],

Diff for: doc/configuration/database-backends-configuration.md

@@ -43,7 +43,7 @@ Persistent Data:
 * RDBMS - MongooseIM has a strong backend support for relational databases.
  Reliable and battle proven, they are a great choice for regular MongooseIM use cases and features like `privacy lists`, `vcards`, `roster`, `private storage`, `last activity` and `message archive`.
  Never loose your data.
- Use MySQL, MariaDB, PostgreSQL, or MS SQL Server.
+ Use MySQL, MariaDB, PostgreSQL, CockroachDB, or MS SQL Server.
 
 * Cassandra - Only for MAM (Message Archive Management).
 
@@ -125,6 +125,35 @@ Please refer to the [RDBMS options](outgoing-connections.md#rdbms-options)
 and [general database options](general.md#database-settings)
 for more information.
 
+### CockroachDB
+
+**Can be used for:**
+
+* users (credentials)
+* vcards
+* roster
+* private storage
+* privacy/block lists
+* last activity
+* mam (message archive management)
+* muc_light rooms
+
+**Setup**
+
+The schema files can be found in the `priv` directory.
+The default schema is defined in the `cockroachdb.sql` file.
+
+You can use the following command to apply it on localhost:
+
+```bash
+psql -h localhost -U user -p 26257 -c "CREATE DATABASE mongooseim;"
+psql -h localhost -U user -p 26257 -q -d mongooseim -f cockroachdb.sql
+```
+You should also configure the CockroachDB database in the `mongooseim.toml` file.
+Please refer to the [RDBMS options](outgoing-connections.md#rdbms-options)
+and [general database options](general.md#database-settings)
+for more information.
+
 ### Microsoft SQL Server
 
 Microsoft SQL Server, sometimes called MSSQL, or Azure SQL Database.

Diff for: doc/configuration/outgoing-connections.md

@@ -74,7 +74,7 @@ For example:
 ### RDBMS options
 
 #### `outgoing_pools.rdbms.*.connection.driver`
-* **Syntax:** string, one of `"pgsql"`, `"mysql"` or `"odbc"` (a supported driver)
+* **Syntax:** string, one of `"pgsql"`, `"mysql"`, `"cockroachdb"` or `"odbc"` (a supported driver)
 * **Default:** none - this option is mandatory
 * **Example:** `driver = "psgql"`
 
@@ -101,34 +101,34 @@ How long MongooseIM will wait for the database to answer for a query.
 
 When MongooseIM fails to connect to the DB, it retries with an exponential backoff. This option limits the backoff time for faster reconnection when the DB becomes reachable again.
 
-### Options for `pgsql` and `mysql`
+### Options for `pgsql`, `cockroachdb` and `mysql`
 
 #### `outgoing_pools.rdbms.*.connection.host`
 * **Syntax:** string
-* **Default:** no default; required for `pgsql` and `mysql`
+* **Default:** no default; required for `pgsql`, `cockroachdb` and `mysql`
 * **Example:** `host = "localhost"`
 
 #### `outgoing_pools.rdbms.*.connection.port`
 * **Syntax:** integer, between 0 and 65535
-* **Default:** `5432` for `pgsql`; `3306` for `mysql`
+* **Default:** `5432` for `pgsql`; `26257` for `cockroachdb`; `3306` for `mysql`
 * **Example:** `port = 5343`
 
 #### `outgoing_pools.rdbms.*.connection.database`
 * **Syntax:** string
-* **Default:** no default; required for `pgsql` and `mysql`
+* **Default:** no default; required for `pgsql`, `cockroachdb` and `mysql`
 * **Example:** `database = "mim-db"`
 
 #### `outgoing_pools.rdbms.*.connection.username`
 * **Syntax:** string
-* **Default:** no default; required for `pgsql` and `mysql`
+* **Default:** no default; required for `pgsql`, `cockroachdb` and `mysql`
 * **Example:** `username = "mim-user"`
 
 #### `outgoing_pools.rdbms.*.connection.password`
 * **Syntax:** string
-* **Default:** no default; required for `pgsql` and `mysql`
+* **Default:** no default; required for `pgsql`, `cockroachdb` and `mysql`
 * **Example:** `password = "mim-password"`
 
-To enable TLS, you need to include the [TLS section](#tls-options) in the connection options. There is one additonal option for PostgreSQL:
+To enable TLS, you need to include the [TLS section](#tls-options) in the connection options. There is one additonal option for PostgreSQL and CockroachDB:
 
 #### `outgoing_pools.rdbms.*.connection.tls.required`
 * **Syntax:** boolean

Diff for: doc/migrations/6.2.1_x.x.x.md

@@ -107,4 +107,23 @@ Stop the cluster, or individual nodes, if performing a rolling upgrade, and exec
     3. Add Prometheus to the [instrumentation section](../configuration/instrumentation.md).
     4. Configure a [listener](../listeners/listen-http.md#handler-types-prometheus-mongoose_prometheus_handler) for Prometheus.
 
-Restart the node or cluster.
+Restart the node or cluster.
+
+## Support for the new RDBMS database - CockroachDB
+
+We've introduced support for a new RDBMS database, CockroachDB.
+
+To configure CockroachDB, update the connection settings in your configuration file as follows:
+```toml
+[outgoing_pools.rdbms.default.connection]
+  driver = "cockroachdb"
+  host = "localhost"
+  port = 26257
+  database = "mongooseim"
+  username = "mongooseim"
+  password = "mongooseim_secret"
+```
+
+Refer to the [Outgoing connections configuration doc](../configuration/outgoing-connections.md) for more information.
+
+The migration script for CockroachDB , `cockroachdb.sql` is located in the [`priv/migrations`](https://github.com/esl/MongooseIM/tree/master/priv/migrations) directory.

Diff for: doc/modules/mod_mam.md

@@ -9,7 +9,7 @@ MongooseIM is compatible with MAM 0.4-1.1.0.
 
 Configure MAM with different storage backends:
 
-* RDBMS (databases like MySQL, PostgreSQL, MS SQL Server)
+* RDBMS (databases like MySQL, PostgreSQL, CockroachDB, MS SQL Server)
 * Cassandra (NoSQL)
 * ElasticSearch (NoSQL)