Skip to content

Commit a121cba

Browse files
authored
Mysql test (#496)
* test error remaining * run-test * remove myrm * json-extract handling normalized and fixed for mysql and maria both * prittay * prittay * little bits * moves reaction trigger outside of transaction * docs, truncate fix, mysql infra * expanded matrix * minor Makefile cleanup * updated default test version of mysql and comment * updates docs to correctly show default test version of mysql * update docs * removed unnecessary newline * removed unnecessary db-type env var in mysql docker-compose * splitting the atom * spark joy * docs
1 parent 4b3f3cb commit a121cba

File tree

26 files changed

+307
-81
lines changed

26 files changed

+307
-81
lines changed

.github/workflows/test.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ jobs:
2525
- test-mariadb-11.4
2626
- test-mariadb-11.7.2
2727
- test-mariadb-11.8
28+
- test-mysql-8.0.44
29+
- test-mysql-8.4
30+
- test-mysql-9.5.0
2831

2932
steps:
3033
- name: Checkout repository

Makefile

Lines changed: 38 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ resources/public/admin:
2323
# All other phony targets run lrsql instances that can be used and tested
2424
# during development. All start up with fixed DB properties and seed creds.
2525

26-
.phony: clean-dev, ci, ephemeral, ephemeral-prod, sqlite, postgres, bench, bench-async, keycloak-demo, ephemeral-oidc, superset-demo, clamav-demo, test-sqlite, test-postgres, test-postgres-11, test-postgres-12, test-postgres-13, test-postgres-14, test-postgres-15, test-mariadb, test-mariadb-10.6, test-mariadb-10.11, test-mariadb-11.4, test-mariadb-11.7.2, test-mariadb-11.8, mariadb
26+
.phony: clean-dev, ci, ephemeral, ephemeral-prod, sqlite, postgres, mariadb, mysql, bench, bench-async, keycloak-demo, ephemeral-oidc, superset-demo, clamav-demo, test-sqlite, test-postgres, test-postgres-11, test-postgres-12, test-postgres-13, test-postgres-14, test-postgres-15, test-mariadb, test-mariadb-10.6, test-mariadb-10.11, test-mariadb-11.4, test-mariadb-11.7.2, test-mariadb-11.8, test-mysql, test-mysql-8.0.44, test-mysql-8.4, test-mysql-9.5.0
2727

2828
clean-dev:
2929
rm -rf *.db *.log resources/public tmp
@@ -33,10 +33,15 @@ clean-dev:
3333
test-sqlite:
3434
clojure -M:test -m lrsql.test-runner --database sqlite $(if $(ns),--ns $(ns))
3535

36+
# Without LRSQL_TEST_DB_VERSION, defaults to version 11
3637
TEST_PG_COMMAND ?= clojure -M:test -m lrsql.test-runner --database postgres $(if $(ns),--ns $(ns))
38+
39+
# Without LRSQL_TEST_DB_VERSION, defaults to version 11.7.2
3740
TEST_MARIADB_COMMAND ?= clojure -M:test -m lrsql.test-runner --database mariadb $(if $(ns),--ns $(ns))
3841

39-
# Without LRSQL_TEST_DB_VERSION, defaults to version 11
42+
# Without LRSQL_TEST_DB_VERSION, defaults to version 8.0.44
43+
TEST_MYSQL_COMMAND ?= clojure -M:test -m lrsql.test-runner --database mysql $(if $(ns),--ns $(ns))
44+
4045
test-postgres:
4146
$(TEST_PG_COMMAND)
4247

@@ -73,10 +78,17 @@ test-mariadb-11.7.2:
7378
test-mariadb-11.8:
7479
LRSQL_TEST_DB_VERSION=11.8 $(TEST_MARIADB_COMMAND)
7580

81+
test-mysql:
82+
$(TEST_MYSQL_COMMAND)
7683

77-
ci: test-sqlite test-postgres test-mariadb
78-
84+
test-mysql-8.0.44:
85+
LRSQL_TEST_DB_VERSION=8.0.44 $(TEST_MYSQL_COMMAND)
86+
test-mysql-8.4:
87+
LRSQL_TEST_DB_VERSION=8.4 $(TEST_MYSQL_COMMAND)
88+
test-mysql-9.5.0:
89+
LRSQL_TEST_DB_VERSION=9.5.0 $(TEST_MYSQL_COMMAND)
7990

91+
ci: test-sqlite test-postgres test-mariadb test-mysql
8092

8193
# Dev
8294

@@ -220,6 +232,10 @@ target/bundle/lrsql_mariadb.exe: exe/lrsql_mariadb.exe
220232
mkdir -p target/bundle
221233
cp exe/lrsql_mariadb.exe target/bundle/lrsql_mariadb.exe
222234

235+
target/bundle/lrsql_mysql.exe: exe/lrsql_mysql.exe
236+
mkdir -p target/bundle
237+
cp exe/lrsql_mysql.exe target/bundle/lrsql_mysql.exe
238+
223239
# Copy Admin UI
224240

225241
target/bundle/admin: resources/public/admin
@@ -231,9 +247,9 @@ target/bundle/admin: resources/public/admin
231247
BUNDLE_RUNTIMES ?= true
232248

233249
ifeq ($(BUNDLE_RUNTIMES),true)
234-
target/bundle: target/bundle/config target/bundle/doc target/bundle/bin target/bundle/lrsql.jar target/bundle/admin target/bundle/lrsql.exe target/bundle/lrsql_pg.exe target/bundle/lrsql_mariadb.exe target/bundle/LICENSE target/bundle/NOTICE target/bundle/customization target/bundle/bench.jar target/bundle/bench target/bundle/runtimes
250+
target/bundle: target/bundle/config target/bundle/doc target/bundle/bin target/bundle/lrsql.jar target/bundle/admin target/bundle/lrsql.exe target/bundle/lrsql_pg.exe target/bundle/lrsql_mariadb.exe target/bundle/lrsql_mysql.exe target/bundle/LICENSE target/bundle/NOTICE target/bundle/customization target/bundle/bench.jar target/bundle/bench target/bundle/runtimes
235251
else
236-
target/bundle: target/bundle/config target/bundle/doc target/bundle/bin target/bundle/lrsql.jar target/bundle/admin target/bundle/lrsql.exe target/bundle/lrsql_pg.exe target/bundle/lrsql_mariadb.exe target/bundle/LICENSE target/bundle/NOTICE target/bundle/customization target/bundle/bench.jar target/bundle/bench
252+
target/bundle: target/bundle/config target/bundle/doc target/bundle/bin target/bundle/lrsql.jar target/bundle/admin target/bundle/lrsql.exe target/bundle/lrsql_pg.exe target/bundle/lrsql_mariadb.exe target/bundle/lrsql_mysql.exe target/bundle/LICENSE target/bundle/NOTICE target/bundle/customization target/bundle/bench.jar target/bundle/bench
237253
endif
238254

239255
bundle: target/bundle
@@ -283,15 +299,21 @@ else
283299
launch4j exe/config_mariadb.xml
284300
endif
285301

302+
exe/lrsql_mysql.exe:
303+
ifeq (,$(shell which launch4j))
304+
$(error "ERROR: launch4j is not installed!")
305+
else
306+
launch4j exe/config_mysql.xml
307+
endif
286308

287-
exe: exe/lrsql.exe exe/lrsql_pg.exe exe/lrsql_mariadb.exe
309+
exe: exe/lrsql.exe exe/lrsql_pg.exe exe/lrsql_mariadb.exe exe/lrsql_mysql.exe
288310

289311
# *** Run build ***
290312

291313
# These targets create a bundle containing a lrsql JAR and then runs
292314
# the JAR to create the specific lrsql instance.
293315

294-
.phony: run-jar-sqlite, run-jar-sqlite-ephemeral, run-jar-postgres, run-jar-mariadb
316+
.phony: run-jar-sqlite, run-jar-sqlite-ephemeral, run-jar-postgres, run-jar-mariadb, run-jar-mysql
295317

296318
run-jar-sqlite-ephemeral: target/bundle
297319
cd target/bundle; \
@@ -326,6 +348,14 @@ run-jar-mariadb: target/bundle
326348
LRSQL_API_SECRET_DEFAULT=password \
327349
bin/run_mariadb.sh
328350

351+
run-jar-mysql: target/bundle
352+
cd target/bundle; \
353+
LRSQL_ADMIN_USER_DEFAULT=username \
354+
LRSQL_ADMIN_PASS_DEFAULT=password \
355+
LRSQL_API_KEY_DEFAULT=username \
356+
LRSQL_API_SECRET_DEFAULT=password \
357+
bin/run_mysql.sh
358+
329359
# *** Report Dependency Graph to GitHub ***
330360

331361
## This pom.xml file is generated solely in an action to populate the GitHub

bin/run_mysql.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/sh
2+
3+
MACHINE=`bin/machine.sh`
4+
5+
runtimes/$MACHINE/bin/java -Dfile.encoding=UTF-8 -server -cp lrsql.jar lrsql.mariadb.main $@
6+
#COMMENT: SQL-LRS uses the same code to interface with MySQL as it uses for MariaDB; the above is intentional.

deps.edn

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,10 @@
125125
{:mvn/version "2.6.1"}
126126
org.mariadb.jdbc/mariadb-java-client
127127
{:mvn/version "3.5.3"}
128+
com.mysql/mysql-connector-j
129+
{:mvn/version "9.4.0"}
128130
org.testcontainers/mariadb {:mvn/version "1.21.2"}
131+
org.testcontainers/mysql {:mvn/version "1.21.3"}
129132

130133
;; Other test deps
131134
org.clojure/test.check {:mvn/version "1.1.1"}

dev-resources/mariadb/docker-compose.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ services:
1919
clamav:
2020
image: clamav/clamav:1.2.1
2121
lrs:
22-
build: ../../. # switch to this for active dev
23-
#image: yetanalytics/lrsql:latest
22+
#build: ../../. # switch to this for active dev
23+
image: yetanalytics/lrsql:latest
2424
command:
2525
- /lrsql/bin/run_mariadb.sh
2626
ports:

dev-resources/mysql/docker-compose.yml

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
1-
# Runs SQL LRS with MariaDB - Provided for demonstration purposes only!
1+
# Runs SQL LRS with MySQL - Provided for demonstration purposes only!
22
# To run: docker compose up
33
# See the Docker Compose docs for more info: https://docs.docker.com/compose/
44
volumes:
55
db_data:
66
services:
77
db:
8-
image: mysql:8.0
8+
image: mysql:9.5.0
99
volumes:
1010
- db_data:/var/lib/mysql/
1111
environment:
@@ -19,16 +19,15 @@ services:
1919
clamav:
2020
image: clamav/clamav:1.2.1
2121
lrs:
22-
build: ../../. # switch to this for active dev
23-
#image: yetanalytics/lrsql:latest
22+
#build: ../../. # switch to this for active dev
23+
image: yetanalytics/lrsql:latest
2424
command:
25-
- /lrsql/bin/run_mariadb.sh
25+
- /lrsql/bin/run_mysql.sh
2626
ports:
2727
- "8080:8080"
2828
depends_on:
2929
- db
3030
environment:
31-
LRSQL_DB_TYPE: mysql
3231
LRSQL_API_KEY_DEFAULT: my_key
3332
LRSQL_API_SECRET_DEFAULT: my_secret
3433
LRSQL_ADMIN_USER_DEFAULT: my_username
@@ -37,7 +36,7 @@ services:
3736
LRSQL_DB_NAME: lrsql_db
3837
LRSQL_DB_USER: lrsql_user
3938
LRSQL_DB_PASSWORD: lrsql_password
40-
# If MariaDB is too slow to start, increase this
39+
# If MySQL is too slow to start, increase this
4140
LRSQL_POOL_INITIALIZATION_FAIL_TIMEOUT: 10000
4241
# Set to true if using dev UI, domain name, proxy server, etc.
4342
LRSQL_ALLOW_ALL_ORIGINS: false

doc/dev.md

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,16 @@ The SQL LRS can be built or run with the following Makefile targets. They can be
2323
| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
2424
| `ci` | Called when running continuous integration; runs all test cases in all SQL flavors. |
2525
| `test-sqlite` | Run all tests with SQLite database. |
26-
| `test-mariadb` | Run all tests with Mariadb database version 11.7.2. Set the `LRSQL_TEST_DB_VERSION` env var to a valid Postgres docker tag to use another version. |
26+
| `test-mariadb` | Run all tests with Mariadb database version 11.7.2. Set the `LRSQL_TEST_DB_VERSION` env var to a valid MariaDB docker tag to use another version. |
27+
| `test-mariadb-10.6` | Run all tests with Mariadb database version 10.6. |
28+
| `test-mariadb-10.11` | Run all tests with Mariadb database version 10.11. |
29+
| `test-mariadb-11.4` | Run all tests with Mariadb database version 11.4. |
30+
| `test-mariadb-11.7.2` | Run all tests with Mariadb database version 11.7.2. |
31+
| `test-mariadb-11.8` | Run all tests with Mariadb database version 11.8. |
32+
| `test-mysql` | Run all tests with MySQL database version 8.0.44. Set the `LRSQL_TEST_DB_VERSION` env var to a valid MySQL docker tag to use another version. |
33+
| `test-mysql-8.0.44` | Run all tests with MySQL database version 8.0.44 |
34+
| `test-mysql-8.4` | Run all tests with MySQL database version 8.4 |
35+
| `test-mysql-9.5.0` | Run all tests with MySQL database version 9.5.0 |
2736
| `test-postgres` | Run all tests with Postgres database version 11. Set the `LRSQL_TEST_DB_VERSION` env var to a valid Postgres docker tag to use another version. |
2837
| `test-postgres-11` | Run all tests with Postgres database version 11. |
2938
| `test-postgres-12` | Run all tests with Postgres database version 12. |
@@ -43,10 +52,13 @@ The SQL LRS can be built or run with the following Makefile targets. They can be
4352
| `sqlite` | Start a SQLite-based SQL LRS. |
4453
| `postgres` | Start a Postgres SQL LRS. Requires a running Postgres instance. |
4554
| `mariadb` | Start a Mariadb SQL LRS. Requires a running MariaDB instance. |
55+
| `mysql` | Start a MySQL SQL LRS. Requires a running MySQL instance. |
4656
| `run-jar-sqlite` | Similar to `sqlite` but it runs the finished Jar instead of directly from Clojure. Runs with a predefined default set of env variables. |
4757
| `run-jar-sqlite-ephemeral` | Similar to `ephemeral` but it runs the finished Jar instead of directly from Clojure. Runs with a predefined default set of env variables. |
4858
| `run-jar-postgres` | Similar to `postgres` but it runs the finished Jar instead of directly from Clojure. Runs with a predefined default set of env variables. |
4959
| `run-jar-mariadb` | Similar to `mariadb` but it runs the finished Jar instead of directly from Clojure. Runs with a predefined default set of env variables. |
60+
| `run-jar-mysql` | Similar to `mysql` but it runs the finished Jar instead of directly from Clojure. Runs with a predefined default set of env variables. |
61+
5062

5163
#### Cleanup Targets
5264

doc/index.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
- [Postgres](postgres.md)
2424
- [MariaDB](mariadb.md)
25+
- [MySQL](mysql.md)
2526
- [SQLite](sqlite.md)
2627

2728
### Reference

doc/mysql.md

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
[<- Back to Index](index.md)
2+
3+
# MySQL
4+
5+
SQL-LRS supports MySQL.
6+
7+
To run SQL-LRS with MySQL, you need a properly configured MySQL instance. "Properly configured" really just means setting up a database, user, and password, and configuring SQL-LRS to connect to/with those.
8+
9+
## Environment Variables
10+
11+
You can configure MySQLDB by setting the relevant environment variables. Here is the `environment` entry from our [MySQL docker-compose demo](https://github.com/yetanalytics/lrsql/blob/main/dev-resources/mysql/docker-compose.yml)
12+
13+
```yml
14+
environment:
15+
MYSQL_ROOT_PASSWORD: lrsql_root_password
16+
MYSQL_DATABASE: lrsql_db
17+
MYSQL_USER: lrsql_user
18+
MYSQL_PASSWORD: lrsql_password
19+
```
20+
21+
(`MYSQL_ROOT_PASSWORD` can be anything, as SQL-LRS doesn't use the root account. It is only included here because the MySQL Docker container requires a root password setting. See the [MySQL docs](https://dev.mysql.com/doc/refman/8.4/en/docker-mysql-more-topics.html] for details)
22+
23+
The corresponding `lrsql.json` would look like
24+
25+
```json
26+
{
27+
...
28+
"database": {
29+
"dbHost": "0.0.0.0",
30+
"dbPort": 3306,
31+
"dbName": "lrsql_db",
32+
"dbUser": "lrsql_user",
33+
"dbPassword": "lrsql_password",
34+
}
35+
}
36+
37+
```
38+
39+
## Precision Limitation
40+
41+
SQL-LRS stores statements in its implementation databases as JSON. Due to the way MySQL interprets numbers in JSON, we **cannot guarantee more than 15 significant digits of precision** when using SQL-LRS alongside MySQL. If you need that much precision, consider using SQL-LRS alongside [Postgres](postgres.md) or [MariaDB](mariadb.md) instead.

docker-compose.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ services:
77
db:
88
image: postgres
99
volumes:
10-
- db_data:/var/lib/postgresql/data
10+
- db_data:/var/lib/postgresql
1111
environment:
1212
POSTGRES_USER: lrsql_user
1313
POSTGRES_PASSWORD: lrsql_password

0 commit comments

Comments
 (0)