diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ebe96bac..d242aa05 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,6 +25,9 @@ jobs: - test-mariadb-11.4 - test-mariadb-11.7.2 - test-mariadb-11.8 + - test-mysql-8.0.44 + - test-mysql-8.4 + - test-mysql-9.5.0 steps: - name: Checkout repository diff --git a/Makefile b/Makefile index a04bb6a4..6daa4b95 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ resources/public/admin: # All other phony targets run lrsql instances that can be used and tested # during development. All start up with fixed DB properties and seed creds. -.phony: clean-dev, ci, ephemeral, ephemeral-prod, sqlite, postgres, bench, bench-async, keycloak-demo, ephemeral-oidc, superset-demo, clamav-demo, test-sqlite, test-postgres, test-postgres-11, test-postgres-12, test-postgres-13, test-postgres-14, test-postgres-15, test-mariadb, test-mariadb-10.6, test-mariadb-10.11, test-mariadb-11.4, test-mariadb-11.7.2, test-mariadb-11.8, mariadb +.phony: clean-dev, ci, ephemeral, ephemeral-prod, sqlite, postgres, mariadb, mysql, bench, bench-async, keycloak-demo, ephemeral-oidc, superset-demo, clamav-demo, test-sqlite, test-postgres, test-postgres-11, test-postgres-12, test-postgres-13, test-postgres-14, test-postgres-15, test-mariadb, test-mariadb-10.6, test-mariadb-10.11, test-mariadb-11.4, test-mariadb-11.7.2, test-mariadb-11.8, test-mysql, test-mysql-8.0.44, test-mysql-8.4, test-mysql-9.5.0 clean-dev: rm -rf *.db *.log resources/public tmp @@ -33,10 +33,15 @@ clean-dev: test-sqlite: clojure -M:test -m lrsql.test-runner --database sqlite $(if $(ns),--ns $(ns)) +# Without LRSQL_TEST_DB_VERSION, defaults to version 11 TEST_PG_COMMAND ?= clojure -M:test -m lrsql.test-runner --database postgres $(if $(ns),--ns $(ns)) + +# Without LRSQL_TEST_DB_VERSION, defaults to version 11.7.2 TEST_MARIADB_COMMAND ?= clojure -M:test -m lrsql.test-runner --database mariadb $(if $(ns),--ns $(ns)) -# Without LRSQL_TEST_DB_VERSION, defaults to version 11 +# Without LRSQL_TEST_DB_VERSION, defaults to version 8.0.44 +TEST_MYSQL_COMMAND ?= clojure -M:test -m lrsql.test-runner --database mysql $(if $(ns),--ns $(ns)) + test-postgres: $(TEST_PG_COMMAND) @@ -73,10 +78,17 @@ test-mariadb-11.7.2: test-mariadb-11.8: LRSQL_TEST_DB_VERSION=11.8 $(TEST_MARIADB_COMMAND) +test-mysql: + $(TEST_MYSQL_COMMAND) -ci: test-sqlite test-postgres test-mariadb - +test-mysql-8.0.44: + LRSQL_TEST_DB_VERSION=8.0.44 $(TEST_MYSQL_COMMAND) +test-mysql-8.4: + LRSQL_TEST_DB_VERSION=8.4 $(TEST_MYSQL_COMMAND) +test-mysql-9.5.0: + LRSQL_TEST_DB_VERSION=9.5.0 $(TEST_MYSQL_COMMAND) +ci: test-sqlite test-postgres test-mariadb test-mysql # Dev @@ -220,6 +232,10 @@ target/bundle/lrsql_mariadb.exe: exe/lrsql_mariadb.exe mkdir -p target/bundle cp exe/lrsql_mariadb.exe target/bundle/lrsql_mariadb.exe +target/bundle/lrsql_mysql.exe: exe/lrsql_mysql.exe + mkdir -p target/bundle + cp exe/lrsql_mysql.exe target/bundle/lrsql_mysql.exe + # Copy Admin UI target/bundle/admin: resources/public/admin @@ -231,9 +247,9 @@ target/bundle/admin: resources/public/admin BUNDLE_RUNTIMES ?= true ifeq ($(BUNDLE_RUNTIMES),true) -target/bundle: target/bundle/config target/bundle/doc target/bundle/bin target/bundle/lrsql.jar target/bundle/admin target/bundle/lrsql.exe target/bundle/lrsql_pg.exe target/bundle/lrsql_mariadb.exe target/bundle/LICENSE target/bundle/NOTICE target/bundle/customization target/bundle/bench.jar target/bundle/bench target/bundle/runtimes +target/bundle: target/bundle/config target/bundle/doc target/bundle/bin target/bundle/lrsql.jar target/bundle/admin target/bundle/lrsql.exe target/bundle/lrsql_pg.exe target/bundle/lrsql_mariadb.exe target/bundle/lrsql_mysql.exe target/bundle/LICENSE target/bundle/NOTICE target/bundle/customization target/bundle/bench.jar target/bundle/bench target/bundle/runtimes else -target/bundle: target/bundle/config target/bundle/doc target/bundle/bin target/bundle/lrsql.jar target/bundle/admin target/bundle/lrsql.exe target/bundle/lrsql_pg.exe target/bundle/lrsql_mariadb.exe target/bundle/LICENSE target/bundle/NOTICE target/bundle/customization target/bundle/bench.jar target/bundle/bench +target/bundle: target/bundle/config target/bundle/doc target/bundle/bin target/bundle/lrsql.jar target/bundle/admin target/bundle/lrsql.exe target/bundle/lrsql_pg.exe target/bundle/lrsql_mariadb.exe target/bundle/lrsql_mysql.exe target/bundle/LICENSE target/bundle/NOTICE target/bundle/customization target/bundle/bench.jar target/bundle/bench endif bundle: target/bundle @@ -283,15 +299,21 @@ else launch4j exe/config_mariadb.xml endif +exe/lrsql_mysql.exe: +ifeq (,$(shell which launch4j)) + $(error "ERROR: launch4j is not installed!") +else + launch4j exe/config_mysql.xml +endif -exe: exe/lrsql.exe exe/lrsql_pg.exe exe/lrsql_mariadb.exe +exe: exe/lrsql.exe exe/lrsql_pg.exe exe/lrsql_mariadb.exe exe/lrsql_mysql.exe # *** Run build *** # These targets create a bundle containing a lrsql JAR and then runs # the JAR to create the specific lrsql instance. -.phony: run-jar-sqlite, run-jar-sqlite-ephemeral, run-jar-postgres, run-jar-mariadb +.phony: run-jar-sqlite, run-jar-sqlite-ephemeral, run-jar-postgres, run-jar-mariadb, run-jar-mysql run-jar-sqlite-ephemeral: target/bundle cd target/bundle; \ @@ -326,6 +348,14 @@ run-jar-mariadb: target/bundle LRSQL_API_SECRET_DEFAULT=password \ bin/run_mariadb.sh +run-jar-mysql: target/bundle + cd target/bundle; \ + LRSQL_ADMIN_USER_DEFAULT=username \ + LRSQL_ADMIN_PASS_DEFAULT=password \ + LRSQL_API_KEY_DEFAULT=username \ + LRSQL_API_SECRET_DEFAULT=password \ + bin/run_mysql.sh + # *** Report Dependency Graph to GitHub *** ## This pom.xml file is generated solely in an action to populate the GitHub diff --git a/bin/run_mysql.sh b/bin/run_mysql.sh new file mode 100755 index 00000000..c7f85f0a --- /dev/null +++ b/bin/run_mysql.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +MACHINE=`bin/machine.sh` + +runtimes/$MACHINE/bin/java -Dfile.encoding=UTF-8 -server -cp lrsql.jar lrsql.mariadb.main $@ +#COMMENT: SQL-LRS uses the same code to interface with MySQL as it uses for MariaDB; the above is intentional. diff --git a/deps.edn b/deps.edn index 8fdfb203..162b579c 100644 --- a/deps.edn +++ b/deps.edn @@ -125,7 +125,10 @@ {:mvn/version "2.6.1"} org.mariadb.jdbc/mariadb-java-client {:mvn/version "3.5.3"} + com.mysql/mysql-connector-j + {:mvn/version "9.4.0"} org.testcontainers/mariadb {:mvn/version "1.21.2"} + org.testcontainers/mysql {:mvn/version "1.21.3"} ;; Other test deps org.clojure/test.check {:mvn/version "1.1.1"} diff --git a/dev-resources/mariadb/docker-compose.yml b/dev-resources/mariadb/docker-compose.yml index 20d43485..81a245da 100644 --- a/dev-resources/mariadb/docker-compose.yml +++ b/dev-resources/mariadb/docker-compose.yml @@ -19,8 +19,8 @@ services: clamav: image: clamav/clamav:1.2.1 lrs: - build: ../../. # switch to this for active dev - #image: yetanalytics/lrsql:latest + #build: ../../. # switch to this for active dev + image: yetanalytics/lrsql:latest command: - /lrsql/bin/run_mariadb.sh ports: diff --git a/dev-resources/mysql/docker-compose.yml b/dev-resources/mysql/docker-compose.yml index 2f98b5ec..a48def2c 100644 --- a/dev-resources/mysql/docker-compose.yml +++ b/dev-resources/mysql/docker-compose.yml @@ -1,11 +1,11 @@ -# Runs SQL LRS with MariaDB - Provided for demonstration purposes only! +# Runs SQL LRS with MySQL - Provided for demonstration purposes only! # To run: docker compose up # See the Docker Compose docs for more info: https://docs.docker.com/compose/ volumes: db_data: services: db: - image: mysql:8.0 + image: mysql:9.5.0 volumes: - db_data:/var/lib/mysql/ environment: @@ -19,16 +19,15 @@ services: clamav: image: clamav/clamav:1.2.1 lrs: - build: ../../. # switch to this for active dev - #image: yetanalytics/lrsql:latest + #build: ../../. # switch to this for active dev + image: yetanalytics/lrsql:latest command: - - /lrsql/bin/run_mariadb.sh + - /lrsql/bin/run_mysql.sh ports: - "8080:8080" depends_on: - db environment: - LRSQL_DB_TYPE: mysql LRSQL_API_KEY_DEFAULT: my_key LRSQL_API_SECRET_DEFAULT: my_secret LRSQL_ADMIN_USER_DEFAULT: my_username @@ -37,7 +36,7 @@ services: LRSQL_DB_NAME: lrsql_db LRSQL_DB_USER: lrsql_user LRSQL_DB_PASSWORD: lrsql_password - # If MariaDB is too slow to start, increase this + # If MySQL is too slow to start, increase this LRSQL_POOL_INITIALIZATION_FAIL_TIMEOUT: 10000 # Set to true if using dev UI, domain name, proxy server, etc. LRSQL_ALLOW_ALL_ORIGINS: false diff --git a/doc/dev.md b/doc/dev.md index fc747d1d..6869f630 100644 --- a/doc/dev.md +++ b/doc/dev.md @@ -23,7 +23,16 @@ The SQL LRS can be built or run with the following Makefile targets. They can be | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | `ci` | Called when running continuous integration; runs all test cases in all SQL flavors. | | `test-sqlite` | Run all tests with SQLite database. | -| `test-mariadb` | Run all tests with Mariadb database version 11.7.2. Set the `LRSQL_TEST_DB_VERSION` env var to a valid Postgres docker tag to use another version. | +| `test-mariadb` | Run all tests with Mariadb database version 11.7.2. Set the `LRSQL_TEST_DB_VERSION` env var to a valid MariaDB docker tag to use another version. | +| `test-mariadb-10.6` | Run all tests with Mariadb database version 10.6. | +| `test-mariadb-10.11` | Run all tests with Mariadb database version 10.11. | +| `test-mariadb-11.4` | Run all tests with Mariadb database version 11.4. | +| `test-mariadb-11.7.2` | Run all tests with Mariadb database version 11.7.2. | +| `test-mariadb-11.8` | Run all tests with Mariadb database version 11.8. | +| `test-mysql` | Run all tests with MySQL database version 8.0.44. Set the `LRSQL_TEST_DB_VERSION` env var to a valid MySQL docker tag to use another version. | +| `test-mysql-8.0.44` | Run all tests with MySQL database version 8.0.44 | +| `test-mysql-8.4` | Run all tests with MySQL database version 8.4 | +| `test-mysql-9.5.0` | Run all tests with MySQL database version 9.5.0 | | `test-postgres` | Run all tests with Postgres database version 11. Set the `LRSQL_TEST_DB_VERSION` env var to a valid Postgres docker tag to use another version. | | `test-postgres-11` | Run all tests with Postgres database version 11. | | `test-postgres-12` | Run all tests with Postgres database version 12. | @@ -43,10 +52,13 @@ The SQL LRS can be built or run with the following Makefile targets. They can be | `sqlite` | Start a SQLite-based SQL LRS. | | `postgres` | Start a Postgres SQL LRS. Requires a running Postgres instance. | | `mariadb` | Start a Mariadb SQL LRS. Requires a running MariaDB instance. | +| `mysql` | Start a MySQL SQL LRS. Requires a running MySQL instance. | | `run-jar-sqlite` | Similar to `sqlite` but it runs the finished Jar instead of directly from Clojure. Runs with a predefined default set of env variables. | | `run-jar-sqlite-ephemeral` | Similar to `ephemeral` but it runs the finished Jar instead of directly from Clojure. Runs with a predefined default set of env variables. | | `run-jar-postgres` | Similar to `postgres` but it runs the finished Jar instead of directly from Clojure. Runs with a predefined default set of env variables. | | `run-jar-mariadb` | Similar to `mariadb` but it runs the finished Jar instead of directly from Clojure. Runs with a predefined default set of env variables. | +| `run-jar-mysql` | Similar to `mysql` but it runs the finished Jar instead of directly from Clojure. Runs with a predefined default set of env variables. | + #### Cleanup Targets diff --git a/doc/index.md b/doc/index.md index b552e56d..37c36c74 100644 --- a/doc/index.md +++ b/doc/index.md @@ -22,6 +22,7 @@ - [Postgres](postgres.md) - [MariaDB](mariadb.md) +- [MySQL](mysql.md) - [SQLite](sqlite.md) ### Reference diff --git a/doc/mysql.md b/doc/mysql.md new file mode 100644 index 00000000..08a2d405 --- /dev/null +++ b/doc/mysql.md @@ -0,0 +1,41 @@ +[<- Back to Index](index.md) + +# MySQL + +SQL-LRS supports MySQL. + +To run SQL-LRS with MySQL, you need a properly configured MySQL instance. "Properly configured" really just means setting up a database, user, and password, and configuring SQL-LRS to connect to/with those. + +## Environment Variables + +You can configure MySQLDB by setting the relevant environment variables. Here is the `environment` entry from our [MySQL docker-compose demo](https://github.com/yetanalytics/lrsql/blob/main/dev-resources/mysql/docker-compose.yml) + +```yml + environment: + MYSQL_ROOT_PASSWORD: lrsql_root_password + MYSQL_DATABASE: lrsql_db + MYSQL_USER: lrsql_user + MYSQL_PASSWORD: lrsql_password +``` + +(`MYSQL_ROOT_PASSWORD` can be anything, as SQL-LRS doesn't use the root account. It is only included here because the MySQL Docker container requires a root password setting. See the [MySQL docs](https://dev.mysql.com/doc/refman/8.4/en/docker-mysql-more-topics.html] for details) + +The corresponding `lrsql.json` would look like + +```json +{ + ... + "database": { + "dbHost": "0.0.0.0", + "dbPort": 3306, + "dbName": "lrsql_db", + "dbUser": "lrsql_user", + "dbPassword": "lrsql_password", + } +} + +``` + +## Precision Limitation + +SQL-LRS stores statements in its implementation databases as JSON. Due to the way MySQL interprets numbers in JSON, we **cannot guarantee more than 15 significant digits of precision** when using SQL-LRS alongside MySQL. If you need that much precision, consider using SQL-LRS alongside [Postgres](postgres.md) or [MariaDB](mariadb.md) instead. \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 0eca9c59..68249178 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: db: image: postgres volumes: - - db_data:/var/lib/postgresql/data + - db_data:/var/lib/postgresql environment: POSTGRES_USER: lrsql_user POSTGRES_PASSWORD: lrsql_password diff --git a/exe/config_mysql.xml b/exe/config_mysql.xml new file mode 100644 index 00000000..3e25d5e5 --- /dev/null +++ b/exe/config_mysql.xml @@ -0,0 +1,29 @@ + + + true + console + + lrsql.mariadb.main + lrsql.jar + + lrsql_mysql.exe + You must have Java 11+ installed to run SQL LRS + . + normal + http://java.com/download + https://yet.zendesk.com/hc/en-us + false + false + + lrsql.ico + + runtimes/windows + false + false + 11 + + preferJre + 64/32 + -Dfile.encoding=UTF-8 + + diff --git a/exe/lrsql_mysql.exe b/exe/lrsql_mysql.exe new file mode 100755 index 00000000..9a16fa8a Binary files /dev/null and b/exe/lrsql_mysql.exe differ diff --git a/resources/lrsql/config/config.edn b/resources/lrsql/config/config.edn index 09c851d8..4ec476ea 100644 --- a/resources/lrsql/config/config.edn +++ b/resources/lrsql/config/config.edn @@ -29,6 +29,18 @@ ;; Testing Only! Specify the version used with testcontainers :test-db-version #or [#env LRSQL_TEST_DB_VERSION "11.7.2"]} + + :test-mysql {:db-type "mysql" + :db-name "lrsql_db" + :db-host "0.0.0.0" + :db-port 3306 + :db-user "lrsql_user" + :db-password "lrsql_password" + :db-properties "allowMultiQueries=true" + + ;; Testing Only! Specify the version used with testcontainers + :test-db-version #or [#env LRSQL_TEST_DB_VERSION "8.0.44"]} + :test-oidc {:db-type "sqlite" :db-name ":memory:"} @@ -45,12 +57,14 @@ :test-sqlite-mem #include "test/sqlite_mem/connection.edn" :test-postgres #include "test/postgres/connection.edn" :test-mariadb #include "test/mariadb/connection.edn" + :test-mysql #include "test/mysql/connection.edn" :test-oidc #include "test/default/connection.edn" ;; Production :prod-sqlite #include "prod/default/connection.edn" :prod-sqlite-mem #include "prod/sqlite_mem/connection.edn" :prod-postgres #include "prod/postgres/connection.edn" :prod-mariadb #include "prod/mariadb/connection.edn" + :prod-mysql #include "prod/mysql/connection.edn" } :tuning #profile @@ -59,12 +73,14 @@ :test-sqlite-mem #include "test/default/tuning.edn" :test-postgres #include "test/postgres/tuning.edn" :test-mariadb #include "test/mariadb/tuning.edn" + :test-mysql #include "test/mysql/tuning.edn" :test-oidc #include "test/default/tuning.edn" ;; Production :prod-sqlite #include "prod/default/tuning.edn" :prod-sqlite-mem #include "prod/default/tuning.edn" :prod-postgres #include "prod/postgres/tuning.edn" :prod-mariadb #include "prod/mariadb/tuning.edn" + :prod-mysql #include "prod/mysql/tuning.edn" } :lrs #profile @@ -73,12 +89,14 @@ :test-sqlite-mem #include "test/default/lrs.edn" :test-postgres #include "test/default/lrs.edn" :test-mariadb #include "test/default/lrs.edn" + :test-mysql #include "test/default/lrs.edn" :test-oidc #include "test/oidc/lrs.edn" ;; Production :prod-sqlite #include "prod/default/lrs.edn" :prod-sqlite-mem #include "prod/default/lrs.edn" :prod-postgres #include "prod/default/lrs.edn" :prod-mariadb #include "prod/default/lrs.edn" + :prod-mysql #include "prod/default/lrs.edn" } :webserver #profile @@ -86,13 +104,15 @@ :test-sqlite #include "test/default/webserver.edn" :test-sqlite-mem #include "test/default/webserver.edn" :test-postgres #include "test/default/webserver.edn" - :test-mariadb #include "test/default/webserver.edn" + :test-mariadb #include "test/default/webserver.edn" + :test-mysql #include "test/default/webserver.edn" :test-oidc #include "test/oidc/webserver.edn" ;; Production :prod-sqlite #include "prod/default/webserver.edn" :prod-sqlite-mem #include "prod/default/webserver.edn" :prod-postgres #include "prod/default/webserver.edn" - :prod-mariadb #include "prod/default/webserver.edn" + :prod-mariadb #include "prod/default/webserver.edn" + :prod-mysql #include "prod/default/webserver.edn" } ;; A user-provided JSON file for merge-with merge into this map :user-config-json #or [#env LRSQL_USER_CONFIG_JSON "config/lrsql.json"] diff --git a/resources/lrsql/config/prod/mysql/connection.edn b/resources/lrsql/config/prod/mysql/connection.edn new file mode 100644 index 00000000..741831a8 --- /dev/null +++ b/resources/lrsql/config/prod/mysql/connection.edn @@ -0,0 +1,4 @@ +#merge +[#include "prod/default/connection.edn" + {:pool-minimum-idle #long #or [#env LRSQL_POOL_MINIMUM_IDLE 10] + :pool-maximum-size #long #or [#env LRSQL_POOL_MAXIMUM_SIZE 10]}] diff --git a/resources/lrsql/config/prod/mysql/database.edn b/resources/lrsql/config/prod/mysql/database.edn new file mode 100644 index 00000000..b1256a1c --- /dev/null +++ b/resources/lrsql/config/prod/mysql/database.edn @@ -0,0 +1,10 @@ +{:db-type #or [#env LRSQL_DB_TYPE "mysql"] + :db-name #or [#env LRSQL_DB_NAME "lrsql_db"] + :db-host #or [#env LRSQL_DB_HOST "0.0.0.0"] + :db-port #long #or [#env LRSQL_DB_PORT 3306] + :db-properties #or [#env LRSQL_DB_PROPERTIES "allowMultiQueries=true"] + :db-jdbc-url #or [#env LRSQL_DB_JDBC_URL nil] + :db-user #or [#env LRSQL_DB_USER nil] + :db-password #or [#env LRSQL_DB_PASSWORD nil] + :db-schema #or [#env LRSQL_DB_SCHEMA nil] + :db-catalog #or [#env LRSQL_DB_CATALOG nil]} diff --git a/resources/lrsql/config/prod/mysql/tuning.edn b/resources/lrsql/config/prod/mysql/tuning.edn new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/resources/lrsql/config/prod/mysql/tuning.edn @@ -0,0 +1 @@ +{} diff --git a/resources/lrsql/config/test/mysql/connection.edn b/resources/lrsql/config/test/mysql/connection.edn new file mode 100644 index 00000000..5af29160 --- /dev/null +++ b/resources/lrsql/config/test/mysql/connection.edn @@ -0,0 +1,5 @@ +#merge + [#include "test/default/connection.edn" + {:pool-minimum-idle 10 + :pool-maximum-size 10 + :pool-initialization-fail-timeout 0}] diff --git a/resources/lrsql/config/test/mysql/database.edn b/resources/lrsql/config/test/mysql/database.edn new file mode 100644 index 00000000..4747ccd4 --- /dev/null +++ b/resources/lrsql/config/test/mysql/database.edn @@ -0,0 +1,4 @@ +#merge +[#include "test/default/database.edn" + {:db-properties #or [#env LRSQL_DB_PROPERTIES "allowMultiQueries=true"]}] + diff --git a/resources/lrsql/config/test/mysql/tuning.edn b/resources/lrsql/config/test/mysql/tuning.edn new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/resources/lrsql/config/test/mysql/tuning.edn @@ -0,0 +1 @@ +{} diff --git a/src/db/mariadb/lrsql/mariadb/data.clj b/src/db/mariadb/lrsql/mariadb/data.clj index a2f19eb4..d1a6987c 100644 --- a/src/db/mariadb/lrsql/mariadb/data.clj +++ b/src/db/mariadb/lrsql/mariadb/data.clj @@ -79,8 +79,8 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (def type->mdb-type - {:bool "BOOLEAN" - :int "INTEGER" + {:bool "UNSIGNED" + :int "SIGNED" :dec "DECIMAL" :string "CHAR" :json "JSON"}) diff --git a/src/db/mariadb/lrsql/mariadb/record.clj b/src/db/mariadb/lrsql/mariadb/record.clj index b81554f0..11a115c5 100644 --- a/src/db/mariadb/lrsql/mariadb/record.clj +++ b/src/db/mariadb/lrsql/mariadb/record.clj @@ -10,11 +10,11 @@ (:import [java.security MessageDigest])) (defn make-path-str [p] - (as-> p path - (map #(format "\"%s\"" %) path) - (into [\$] path) - (clojure.string/join \. path) - (format "'%s'" path))) + (as-> p path + (map #(format "\"%s\"" %) path) + (into [\$] path) + (clojure.string/join \. path) + (format "'%s'" path))) (defn sha256-bytes [^String s] (let [md (MessageDigest/getInstance "SHA-256")] @@ -283,8 +283,13 @@ (-error-reaction! [_ tx params] (error-reaction! tx params)) (-snip-json-extract [_ {:keys [datatype] :as params}] - (snip-json-extract (assoc params :type - datatype))) + (let [params (assoc params + :path-str (make-path-str (:path params)) + :type (md/type->mdb-type datatype))] + (cond (#{:bool :int :dec} datatype) + (snip-json-extract-numeric params) + (#{:string :json} datatype) + (snip-json-extract-string params)))) (-snip-val [_ params] (snip-val params)) (-snip-col [_ params] diff --git a/src/db/mariadb/lrsql/mariadb/sql/query.sql b/src/db/mariadb/lrsql/mariadb/sql/query.sql index f51d94f9..18356f7f 100644 --- a/src/db/mariadb/lrsql/mariadb/sql/query.sql +++ b/src/db/mariadb/lrsql/mariadb/sql/query.sql @@ -366,10 +366,11 @@ ORDER BY stored_time ASC; /* Statement Reactions */ --- :snip snip-json-extract -JSON_EXTRACT(:i:col, ---~ (lrsql.mariadb.record/make-path-str (:path params)) -) +-- :snip snip-json-extract-numeric +CAST(JSON_EXTRACT(:i:col, :sql:path-str)+0 AS :sql:type) + +-- :snip snip-json-extract-string +JSON_UNQUOTE(JSON_EXTRACT(:i:col, :sql:path-str)) -- :snip snip-val :v:val diff --git a/src/main/lrsql/system/lrs.clj b/src/main/lrsql/system/lrs.clj index e1ee633f..e1f5e14b 100644 --- a/src/main/lrsql/system/lrs.clj +++ b/src/main/lrsql/system/lrs.clj @@ -120,45 +120,48 @@ attachments)) retry-test (partial bp/-txn-retry? backend) retry-limit (:stmt-retry-limit config) - retry-budget (:stmt-retry-budget config)] - (with-rerunable-txn [tx conn {:retry-test retry-test - :budget retry-budget - :max-attempt retry-limit}] - (loop [stmt-ins stmt-inputs - stmt-res {:statement-ids []}] - (if-some [stmt-input (first stmt-ins)] - ;; Statement input available to insert - (let [stmt-descs (stmt-q/query-descendants - backend - tx - stmt-input) - stmt-input' (stmt-input/add-insert-descendant-inputs - stmt-input - stmt-descs) - stmt-result (stmt-cmd/insert-statement! - backend - tx - stmt-input')] - (if-some [err (:error stmt-result)] - ;; Statement conflict or some other error - stop and rollback - ;; Return the error, which will either be logged here or - ;; (if it's unexpected) bubble up until the end - (do (when (= ::lrsp/statement-conflict (-> err ex-data :type)) - (log/warn (ex-message err))) - (log/warn "Rolling back transaction...") - (.rollback tx) - stmt-result) - ;; Non-error result - continue - (if-some [stmt-id (:statement-id stmt-result)] - (do - ;; Submit statement for reaction if enabled - (react-init/offer-trigger! reaction-channel stmt-id) - (recur (rest stmt-ins) - (update stmt-res :statement-ids conj stmt-id))) - (recur (rest stmt-ins) - stmt-res)))) - ;; No more statement inputs - return - stmt-res))))) + retry-budget (:stmt-retry-budget config) + result (with-rerunable-txn [tx conn {:retry-test retry-test + :budget retry-budget + :max-attempt retry-limit}] + (loop [stmt-ins stmt-inputs + stmt-res {:statement-ids []}] + (if-some [stmt-input (first stmt-ins)] + ;; Statement input available to insert + (let [stmt-descs (stmt-q/query-descendants + backend + tx + stmt-input) + stmt-input' (stmt-input/add-insert-descendant-inputs + stmt-input + stmt-descs) + stmt-result (stmt-cmd/insert-statement! + backend + tx + stmt-input')] + ;verify statement presence and query success + (if-some [err (:error stmt-result)] + ;; Statement conflict or some other error - stop and rollback + ;; Return the error, which will either be logged here or + ;; (if it's unexpected) bubble up until the end + (do (when (= ::lrsp/statement-conflict (-> err ex-data :type)) + (log/warn (ex-message err))) + (log/warn "Rolling back transaction...") + (.rollback tx) + stmt-result) + ;; Non-error result - continue + (if-some [stmt-id (:statement-id stmt-result)] + (recur (rest stmt-ins) + (update stmt-res :statement-ids conj stmt-id)) + (recur (rest stmt-ins) + stmt-res)))) + ;; No more statement inputs - return + stmt-res)))] + ;; Log stmts for reaction submission if enabled + (when-not (:error result) + (doseq [stmt-id (:statement-ids result)] + (react-init/offer-trigger! reaction-channel stmt-id))) + result)) (-get-statements [lrs ctx auth-identity params ltags] (let [conn (lrs-conn lrs) diff --git a/src/test/lrsql/lrs_test.clj b/src/test/lrsql/lrs_test.clj index d723c933..6c5f10e6 100644 --- a/src/test/lrsql/lrs_test.clj +++ b/src/test/lrsql/lrs_test.clj @@ -1,6 +1,7 @@ (ns lrsql.lrs-test (:require [clojure.test :refer [deftest testing is use-fixtures]] [clojure.string :as cstr] + [clojure.walk :as walk] [com.stuartsierra.component :as component] [com.yetanalytics.lrs.protocol :as lrsp] [lrsql.admin.protocol :as adp] @@ -984,8 +985,24 @@ ;; Taken from lrs and third lib tests ;; We reuse bench resources for tests here. + + +(defn trunc [n d] + (let [pow (Math/pow 10 d)] + (-> n + (* pow) + (Math/floor) + (/ pow)))) + (def test-statements - (support/bench-statements 50)) + (->> (support/bench-statements 50) + (walk/prewalk + (fn [v] + (if (and (number? v) + (not= v (Math/floor v))) + (let [before (count (str (int (Math/floor v)) ))] + (trunc v (- 15 before))) + v))))) (deftest datasim-tests (let [sys (support/test-system) diff --git a/src/test/lrsql/test_runner.clj b/src/test/lrsql/test_runner.clj index 1e1ce194..019822f0 100644 --- a/src/test/lrsql/test_runner.clj +++ b/src/test/lrsql/test_runner.clj @@ -11,19 +11,22 @@ :or {db "sqlite" ns nil}} args] (when (contains? #{"postgres" - "mariadb"} + "mariadb" + "mysql"} db) (log/infof "Starting container for %s..." db)) (with-redefs [support/fresh-db-fixture (case db "sqlite" support/fresh-sqlite-fixture "postgres" support/fresh-postgres-fixture - "mariadb" support/fresh-mariadb-fixture) + "mariadb" support/fresh-mariadb-fixture + "mysql" support/fresh-mysql-fixture) support/*container* (case db "sqlite" nil "postgres" (tc/start! support/postgres-container) - "mariadb" (tc/start! support/mariadb-container))] + "mariadb" (tc/start! support/mariadb-container) + "mysql" (tc/start! support/mysql-container))] (try (when support/*container* (log/infof "Container for %s started!" db)) diff --git a/src/test/lrsql/test_support.clj b/src/test/lrsql/test_support.clj index f9179a50..bb4e3cd1 100644 --- a/src/test/lrsql/test_support.clj +++ b/src/test/lrsql/test_support.clj @@ -93,6 +93,27 @@ :message "ready for connections" :times 1}}))) +(def mysql-container + (let [{{{:keys [db-type + db-port + db-name + db-user + db-password + test-db-version]} + :database} + :connection} (read-config :test-mysql)] + (tc/create + {:image-name (format "%s:%s" db-type test-db-version) + :exposed-ports [db-port] + :env-vars {"MYSQL_DATABASE" db-name + "MYSQL_USER" db-user + "MYSQL_PASSWORD" db-password + "MYSQL_RANDOM_ROOT_PASSWORD" "true"} + :wait-for {:wait-strategy :log + :message "ready for connections" + :times 1}}))) + + (def ^:dynamic *container* nil) (def table-names @@ -160,7 +181,8 @@ (binding [*container* (tc/start! (case dbtype :postgres postgres-container - :mariadb mariadb-container))] + :mariadb mariadb-container + :mysql mysql-container))] (log/infof "%s container started!" (name dbtype)) (try (db-fixture-fn f) @@ -170,7 +192,8 @@ (log/infof "%s container stopped!" (name dbtype)))))) (let [profile-kw (case dbtype :postgres :test-postgres - :mariadb :test-mariadb) + :mariadb :test-mariadb + :mysql :test-mysql) {{{:keys [db-type db-port db-name @@ -199,14 +222,16 @@ test-system (fn [& {:keys [conf-overrides]}] (system/system ((case dbtype :postgres pr/map->PostgresBackend - :mariadb mr/map->MariadbBackend) + :mariadb mr/map->MariadbBackend + :mysql mr/map->MariadbBackend) {}) profile-kw :conf-overrides conf-overrides))] (let [ret (f)] ((case dbtype :postgres truncate-all-postgres! - :mariadb truncate-all-mariadb!) + :mariadb truncate-all-mariadb! + :mysql truncate-all-mariadb!) ds) ret)))))) @@ -216,6 +241,9 @@ (def fresh-mariadb-fixture (fixture-builder :mariadb)) +(def fresh-mysql-fixture + (fixture-builder :mysql)) + (def fresh-db-fixture fresh-sqlite-fixture) (defn set-db-fixture-mode!