Skip to content

Commit 39f65da

Browse files
invalidusermiltcliffcaseyyet
authored
Maria (#481)
* maria wip * second pass * after test harness * show&tell * 0-0 * update cognitect test-runner * cleanup * test * added binary * always uses test containers * eliminated trailing space after run-jar-maria * removes cruft * json path and write * Speed Up Tests with Singleton TestContainer (#483) * use shared db for sqlite this likely has no effect, but we do it so the tests will function similarly between backends * singleton tc for postgres * kondo * working testcontainers for mariadb * use ns arg * refactor fixture code * log about container lifecycle * shutdown agents to end tests in a timely fashion * don't truncate for sqlite * spacing * refactor runner for single container var * example of repl container usage * add support for ad-hoc test runs with testcontainers * autocommit off * runs tests in gh * 0-0-2 * maria bows to marie * pr feedback * pr feedback + dockerfile * packaging changes needed to make jar and container (#484) * added multiqueries to prod * excepts emacs file detritus from build * right before change to maria * correct profile name invocation in test support * test-maria -> test-mariadb * removes comment, changes .exe file * docs formatting * maria doc * comment in build.clj, normalization to `mariadb`, addition of dummy `database.edn` file * brings naming in line and makes prod-maria work * adds explanation of base-64 encoding of jwts * tvs * formatting * formatting * make and matrix for LTS versions of maria * all in phony * test-mariadb now uses TEST_MARAIDB_COMMAND * catches deadlocks * correct alias in build script * Jwt time (#487) * decomplects eviction and expiry, sets expiry correctly * clearer comments * MySQL (#488) * consistent mariadb shortname (#489) * consistent mariadb shortname * regenned exes --------- Co-authored-by: Cliff Casey <cliff@yetanalytics.com> Co-authored-by: Cliff Casey <54323005+cliffcaseyyet@users.noreply.github.com> * adds a hash on profile-id to agent_profile_document (#491) * adds a hash on profile-id to agent_profile_document * regenned exes * updates README to include link to maria docs * updatd milady --------- Co-authored-by: Milton Reder <milt@yetanalytics.com> Co-authored-by: Cliff Casey <54323005+cliffcaseyyet@users.noreply.github.com> Co-authored-by: Cliff Casey <cliff@yetanalytics.com>
1 parent 4d1cf3b commit 39f65da

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+2333
-215
lines changed

.github/workflows/test.yml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,12 @@ jobs:
2020
- test-postgres-15
2121
- test-postgres-16
2222
- test-postgres-17
23-
23+
- test-mariadb-10.6
24+
- test-mariadb-10.11
25+
- test-mariadb-11.4
26+
- test-mariadb-11.7.2
27+
- test-mariadb-11.8
28+
2429
steps:
2530
- name: Checkout repository
2631
uses: actions/checkout@v4

Makefile

Lines changed: 54 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,18 @@ resources/public/admin:
2323
# All other phony targets run lrsql instances that can be used and tested
2424
# during development. All start up with fixed DB properties and seed creds.
2525

26-
.phony: clean-dev, ci, ephemeral, ephemeral-prod, sqlite, postgres, bench, bench-async, keycloak-demo, ephemeral-oidc, superset-demo, clamav-demo, test-sqlite, test-postgres, test-postgres-11, test-postgres-12, test-postgres-13, test-postgres-14, test-postgres-15
26+
.phony: clean-dev, ci, ephemeral, ephemeral-prod, sqlite, postgres, bench, bench-async, keycloak-demo, ephemeral-oidc, superset-demo, clamav-demo, test-sqlite, test-postgres, test-postgres-11, test-postgres-12, test-postgres-13, test-postgres-14, test-postgres-15, test-mariadb, test-mariadb-10.6, test-mariadb-10.11, test-mariadb-11.4, test-mariadb-11.7.2, test-mariadb-11.8, mariadb
2727

2828
clean-dev:
2929
rm -rf *.db *.log resources/public tmp
3030

3131
# Tests
3232

3333
test-sqlite:
34-
clojure -M:test -m lrsql.test-runner --database sqlite
34+
clojure -M:test -m lrsql.test-runner --database sqlite $(if $(ns),--ns $(ns))
3535

36-
TEST_PG_COMMAND ?= clojure -M:test -m lrsql.test-runner --database postgres
36+
TEST_PG_COMMAND ?= clojure -M:test -m lrsql.test-runner --database postgres $(if $(ns),--ns $(ns))
37+
TEST_MARIADB_COMMAND ?= clojure -M:test -m lrsql.test-runner --database mariadb $(if $(ns),--ns $(ns))
3738

3839
# Without LRSQL_TEST_DB_VERSION, defaults to version 11
3940
test-postgres:
@@ -54,7 +55,28 @@ test-postgres-16:
5455
test-postgres-17:
5556
LRSQL_TEST_DB_VERSION=17 $(TEST_PG_COMMAND)
5657

57-
ci: test-sqlite test-postgres
58+
test-mariadb:
59+
$(TEST_MARIADB_COMMAND)
60+
61+
test-mariadb-10.6:
62+
LRSQL_TEST_DB_VERSION=10.6 $(TEST_MARIADB_COMMAND)
63+
64+
test-mariadb-10.11:
65+
LRSQL_TEST_DB_VERSION=10.11 $(TEST_MARIADB_COMMAND)
66+
67+
test-mariadb-11.4:
68+
LRSQL_TEST_DB_VERSION=11.4 $(TEST_MARIADB_COMMAND)
69+
70+
test-mariadb-11.7.2:
71+
LRSQL_TEST_DB_VERSION=11.7.2 $(TEST_MARIADB_COMMAND)
72+
73+
test-mariadb-11.8:
74+
LRSQL_TEST_DB_VERSION=11.8 $(TEST_MARIADB_COMMAND)
75+
76+
77+
ci: test-sqlite test-postgres test-mariadb
78+
79+
5880

5981
# Dev
6082

@@ -75,14 +97,17 @@ sqlite: resources/public/admin
7597
postgres: resources/public/admin # Requires a running Postgres instance
7698
clojure -X:db-postgres lrsql.postgres.main/run-test-postgres
7799

100+
mariadb: resources/public/admin # Requires a running MariaDB instance
101+
clojure -X:db-mariadb lrsql.mariadb.main/run-test-mariadb
102+
78103
# Bench - requires a running lrsql instance
79104

80105
bench:
81106
clojure -M:bench -m lrsql.bench \
82107
-e http://0.0.0.0:8080/xapi/statements \
83108
-i dev-resources/bench/insert_input.json \
84109
-q dev-resources/bench/query_input.json \
85-
-u username -p password
110+
-u my_key -p my_secret
86111

87112
bench-async:
88113
clojure -M:bench -m lrsql.bench \
@@ -191,6 +216,10 @@ target/bundle/lrsql_pg.exe: exe/lrsql_pg.exe
191216
mkdir -p target/bundle
192217
cp exe/lrsql_pg.exe target/bundle/lrsql_pg.exe
193218

219+
target/bundle/lrsql_mariadb.exe: exe/lrsql_mariadb.exe
220+
mkdir -p target/bundle
221+
cp exe/lrsql_mariadb.exe target/bundle/lrsql_mariadb.exe
222+
194223
# Copy Admin UI
195224

196225
target/bundle/admin: resources/public/admin
@@ -202,9 +231,9 @@ target/bundle/admin: resources/public/admin
202231
BUNDLE_RUNTIMES ?= true
203232

204233
ifeq ($(BUNDLE_RUNTIMES),true)
205-
target/bundle: target/bundle/config target/bundle/doc target/bundle/bin target/bundle/lrsql.jar target/bundle/admin target/bundle/lrsql.exe target/bundle/lrsql_pg.exe target/bundle/LICENSE target/bundle/NOTICE target/bundle/customization target/bundle/bench.jar target/bundle/bench target/bundle/runtimes
234+
target/bundle: target/bundle/config target/bundle/doc target/bundle/bin target/bundle/lrsql.jar target/bundle/admin target/bundle/lrsql.exe target/bundle/lrsql_pg.exe target/bundle/lrsql_mariadb.exe target/bundle/LICENSE target/bundle/NOTICE target/bundle/customization target/bundle/bench.jar target/bundle/bench target/bundle/runtimes
206235
else
207-
target/bundle: target/bundle/config target/bundle/doc target/bundle/bin target/bundle/lrsql.jar target/bundle/admin target/bundle/lrsql.exe target/bundle/lrsql_pg.exe target/bundle/LICENSE target/bundle/NOTICE target/bundle/customization target/bundle/bench.jar target/bundle/bench
236+
target/bundle: target/bundle/config target/bundle/doc target/bundle/bin target/bundle/lrsql.jar target/bundle/admin target/bundle/lrsql.exe target/bundle/lrsql_pg.exe target/bundle/lrsql_mariadb.exe target/bundle/LICENSE target/bundle/NOTICE target/bundle/customization target/bundle/bench.jar target/bundle/bench
208237
endif
209238

210239
bundle: target/bundle
@@ -247,14 +276,22 @@ else
247276
launch4j exe/config_pg.xml
248277
endif
249278

250-
exe: exe/lrsql.exe exe/lrsql_pg.exe
279+
exe/lrsql_mariadb.exe:
280+
ifeq (,$(shell which launch4j))
281+
$(error "ERROR: launch4j is not installed!")
282+
else
283+
launch4j exe/config_mariadb.xml
284+
endif
285+
286+
287+
exe: exe/lrsql.exe exe/lrsql_pg.exe exe/lrsql_mariadb.exe
251288

252289
# *** Run build ***
253290

254291
# These targets create a bundle containing a lrsql JAR and then runs
255292
# the JAR to create the specific lrsql instance.
256293

257-
.phony: run-jar-sqlite, run-jar-sqlite-ephemeral, run-jar-postgres
294+
.phony: run-jar-sqlite, run-jar-sqlite-ephemeral, run-jar-postgres, run-jar-mariadb
258295

259296
run-jar-sqlite-ephemeral: target/bundle
260297
cd target/bundle; \
@@ -281,6 +318,14 @@ run-jar-postgres: target/bundle
281318
LRSQL_API_SECRET_DEFAULT=password \
282319
bin/run_postgres.sh
283320

321+
run-jar-mariadb: target/bundle
322+
cd target/bundle; \
323+
LRSQL_ADMIN_USER_DEFAULT=username \
324+
LRSQL_ADMIN_PASS_DEFAULT=password \
325+
LRSQL_API_KEY_DEFAULT=username \
326+
LRSQL_API_SECRET_DEFAULT=password \
327+
bin/run_mariadb.sh
328+
284329
# *** Report Dependency Graph to GitHub ***
285330

286331
## This pom.xml file is generated solely in an action to populate the GitHub

README.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ For releases and release notes, see the [Releases](https://github.com/yetanalyti
4040
### DBMS-specific Sections
4141

4242
- [Postgres](doc/postgres.md)
43+
- [MariaDB](doc/mariadb.md)
4344
- [SQLite](doc/sqlite.md)
4445

4546
### Reference
@@ -49,7 +50,7 @@ For releases and release notes, see the [Releases](https://github.com/yetanalyti
4950
- [Developer Documentation](doc/dev.md)
5051
- [Example AWS Deployment](doc/aws.md)
5152
- [Reactions](doc/reactions.md)
52-
- [JSON Spec](doc/reactions/spec.md)
53+
- [JSON Spec](doc/reactions/spec.md)
5354
- [Sending xAPI statement(s) with Postman](doc/postman.md)
5455

5556
### Demos

bin/run_mariadb.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/sh
2+
3+
MACHINE=`bin/machine.sh`
4+
5+
runtimes/$MACHINE/bin/java -Dfile.encoding=UTF-8 -server -cp lrsql.jar lrsql.mariadb.main $@

deps.edn

Lines changed: 67 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,55 +1,56 @@
11
{:paths ["src/main" "resources"]
22
:deps
33
{;; Clojure core deps
4-
org.clojure/clojure {:mvn/version "1.11.2"}
5-
org.clojure/tools.logging {:mvn/version "1.1.0"}
6-
org.clojure/core.memoize {:mvn/version "1.0.250"}
7-
org.clojure/data.csv {:mvn/version "1.1.0"}
8-
clojure-interop/java.security {:mvn/version "1.0.5"}
9-
org.clojure/core.async {:mvn/version "1.6.681"}
4+
org.clojure/clojure {:mvn/version "1.11.2"}
5+
org.clojure/tools.logging {:mvn/version "1.1.0"}
6+
org.clojure/core.memoize {:mvn/version "1.0.250"}
7+
org.clojure/data.csv {:mvn/version "1.1.0"}
8+
clojure-interop/java.security {:mvn/version "1.0.5"}
9+
org.clojure/core.async {:mvn/version "1.6.681"}
1010
;; Util deps
11-
camel-snake-kebab/camel-snake-kebab {:mvn/version "0.4.2"}
12-
cheshire/cheshire {:mvn/version "5.12.0"}
13-
clj-commons/clj-yaml {:mvn/version "1.0.27"}
14-
clojure.java-time/clojure.java-time {:mvn/version "1.2.0"}
15-
danlentz/clj-uuid {:mvn/version "0.1.9"}
16-
metosin/spec-tools {:mvn/version "0.10.6"}
17-
aero/aero {:mvn/version "1.1.6"}
18-
selmer/selmer {:mvn/version "1.12.59"}
19-
ch.qos.logback/logback-classic {:mvn/version "1.5.15"}
20-
commons-io/commons-io {:mvn/version "2.14.0"}
21-
commons-fileupload/commons-fileupload {:mvn/version "1.6.0"}
11+
camel-snake-kebab/camel-snake-kebab {:mvn/version "0.4.2"}
12+
cheshire/cheshire {:mvn/version "5.12.0"}
13+
clj-commons/clj-yaml {:mvn/version "1.0.27"}
14+
clojure.java-time/clojure.java-time {:mvn/version "1.2.0"}
15+
danlentz/clj-uuid {:mvn/version "0.1.9"}
16+
metosin/spec-tools {:mvn/version "0.10.6"}
17+
aero/aero {:mvn/version "1.1.6"}
18+
selmer/selmer {:mvn/version "1.12.59"}
19+
ch.qos.logback/logback-classic {:mvn/version "1.5.15"}
20+
commons-io/commons-io {:mvn/version "2.14.0"}
21+
commons-fileupload/commons-fileupload {:mvn/version "1.6.0"}
22+
2223
;; DB/JDBC deps
2324
;; - HikariCP: Need to exclude slf4j to make logback work properly
2425
;; - HugSql: Use custom version instead of the released version (0.5.1)
25-
com.stuartsierra/component {:mvn/version "1.0.0"}
26-
com.github.seancorfield/next.jdbc {:mvn/version "1.2.709"}
27-
com.yetanalytics/hugsql {:mvn/version "0.6.1-YetAnalytics"}
28-
com.layerware/hugsql-adapter-next-jdbc {:mvn/version "0.5.1"}
29-
com.zaxxer/HikariCP {:mvn/version "5.0.0"
30-
:exclusions [org.slf4j/slf4j-api]}
26+
com.stuartsierra/component {:mvn/version "1.0.0"}
27+
com.github.seancorfield/next.jdbc {:mvn/version "1.2.709"}
28+
com.yetanalytics/hugsql {:mvn/version "0.6.1-YetAnalytics"}
29+
com.layerware/hugsql-adapter-next-jdbc {:mvn/version "0.5.1"}
30+
com.zaxxer/HikariCP {:mvn/version "5.0.0"
31+
:exclusions [org.slf4j/slf4j-api]}
3132
;; Pedestal and Jetty webserver deps
32-
io.pedestal/pedestal.jetty {:mvn/version "0.6.3"}
33-
org.eclipse.jetty/jetty-server {:mvn/version "9.4.58.v20250814"}
34-
org.eclipse.jetty/jetty-servlet {:mvn/version "9.4.58.v20250814"}
35-
org.eclipse.jetty/jetty-alpn-server {:mvn/version "9.4.58.v20250814"}
36-
org.eclipse.jetty.http2/http2-server {:mvn/version "9.4.58.v20250814"}
37-
org.eclipse.jetty/jetty-alpn-java-server {:mvn/version "9.4.58.v20250814"}
33+
io.pedestal/pedestal.jetty {:mvn/version "0.6.3"}
34+
org.eclipse.jetty/jetty-server {:mvn/version "9.4.58.v20250814"}
35+
org.eclipse.jetty/jetty-servlet {:mvn/version "9.4.58.v20250814"}
36+
org.eclipse.jetty/jetty-alpn-server {:mvn/version "9.4.58.v20250814"}
37+
org.eclipse.jetty.http2/http2-server {:mvn/version "9.4.58.v20250814"}
38+
org.eclipse.jetty/jetty-alpn-java-server {:mvn/version "9.4.58.v20250814"}
3839
org.eclipse.jetty.websocket/websocket-api {:mvn/version "9.4.58.v20250814"}
3940
org.eclipse.jetty.websocket/websocket-servlet {:mvn/version "9.4.58.v20250814"}
4041
org.eclipse.jetty.websocket/websocket-server {:mvn/version "9.4.58.v20250814"}
4142
;; Security deps
42-
buddy/buddy-core {:mvn/version "1.11.418"
43-
:exclusions [org.bouncycastle/bcprov-jdk18on
44-
org.bouncycastle/bcpkix-jdk18on]}
45-
buddy/buddy-sign {:mvn/version "3.4.1"
46-
:exclusions [buddy/buddy-core]}
47-
buddy/buddy-hashers {:mvn/version "2.0.162"
48-
:exclusions [buddy/buddy-core]}
49-
org.bouncycastle/bcprov-jdk18on {:mvn/version "1.79"}
50-
org.bouncycastle/bcpkix-jdk18on {:mvn/version "1.79"}
51-
less-awful-ssl/less-awful-ssl {:mvn/version "1.0.6"}
52-
xyz.capybara/clamav-client {:mvn/version "2.1.2"}
43+
buddy/buddy-core {:mvn/version "1.11.418"
44+
:exclusions [org.bouncycastle/bcprov-jdk18on
45+
org.bouncycastle/bcpkix-jdk18on]}
46+
buddy/buddy-sign {:mvn/version "3.4.1"
47+
:exclusions [buddy/buddy-core]}
48+
buddy/buddy-hashers {:mvn/version "2.0.162"
49+
:exclusions [buddy/buddy-core]}
50+
org.bouncycastle/bcprov-jdk18on {:mvn/version "1.79"}
51+
org.bouncycastle/bcpkix-jdk18on {:mvn/version "1.79"}
52+
less-awful-ssl/less-awful-ssl {:mvn/version "1.0.6"}
53+
xyz.capybara/clamav-client {:mvn/version "2.1.2"}
5354
;; Yet Analytics deps
5455
com.yetanalytics/lrs
5556
{:mvn/version "1.3.1"
@@ -72,15 +73,15 @@
7273
buddy/buddy-sign]}
7374
com.yetanalytics/lrs-reactions
7475
{:mvn/version "0.0.1"
75-
:exclusions [org.clojure/clojure]}
76+
:exclusions [org.clojure/clojure]}
7677
com.yetanalytics/gen-openapi
7778
{:mvn/version "0.0.4"
7879
:exclusions [org.clojure/clojure
7980
org.clojure/clojurescript
8081
cheshire/cheshire
8182
clj-commons/clj-yaml]}}
8283

83-
:aliases
84+
:aliases
8485
{:db-sqlite
8586
{:extra-paths ["src/db/sqlite"]
8687
:extra-deps {org.xerial/sqlite-jdbc {:mvn/version "3.42.0.0"}}}
@@ -91,43 +92,56 @@
9192
{:mvn/version "2.6.1"}
9293
com.kohlschutter.junixsocket/junixsocket-native-common
9394
{:mvn/version "2.6.1"}}}
95+
:db-mariadb
96+
{:extra-paths ["src/db/mariadb"]
97+
:extra-deps {org.mariadb.jdbc/mariadb-java-client {:mvn/version "3.5.3"}
98+
;; allow MySQL swap-in
99+
com.mysql/mysql-connector-j {:mvn/version "9.4.0"}}}
100+
94101
:bench
95102
{:extra-paths ["src/bench" "dev-resources"]
96103
:extra-deps {org.clojure/tools.cli {:mvn/version "1.0.194"}
97104
org.clojure/math.numeric-tower {:mvn/version "0.0.5"}
98105
babashka/babashka.curl {:mvn/version "0.0.3"}
99106
com.yetanalytics/datasim
100107
{:mvn/version "0.4.4"
101-
:exclusions [org.clojure/clojure
102-
com.yetanalytics/xapi-schema]}}}
108+
:exclusions [org.clojure/clojure
109+
com.yetanalytics/xapi-schema]}}}
103110
:test
104111
{:extra-paths ["src/db/sqlite"
105112
"src/db/postgres"
113+
"src/db/mariadb"
106114
"src/test"
107115
"dev-resources"]
108116
:extra-deps {;; DB deps
109-
org.xerial/sqlite-jdbc {:mvn/version "3.42.0.0"}
110-
org.postgresql/postgresql {:mvn/version "42.6.1"}
111-
org.testcontainers/postgresql {:mvn/version "1.19.8"}
117+
org.xerial/sqlite-jdbc {:mvn/version "3.42.0.0"}
118+
org.postgresql/postgresql {:mvn/version "42.6.1"}
119+
clj-test-containers/clj-test-containers
120+
{:mvn/version "0.7.3"}
121+
;; org.testcontainers/postgresql {:mvn/version "1.19.8"}
112122
com.kohlschutter.junixsocket/junixsocket-common
113123
{:mvn/version "2.6.1"}
114124
com.kohlschutter.junixsocket/junixsocket-native-common
115125
{:mvn/version "2.6.1"}
126+
org.mariadb.jdbc/mariadb-java-client
127+
{:mvn/version "3.5.3"}
128+
org.testcontainers/mariadb {:mvn/version "1.21.2"}
129+
116130
;; Other test deps
117-
org.clojure/test.check {:mvn/version "1.1.1"}
118-
babashka/babashka.curl {:mvn/version "0.0.3"}
119-
orchestra/orchestra {:mvn/version "2021.01.01-1"}
131+
org.clojure/test.check {:mvn/version "1.1.1"}
132+
babashka/babashka.curl {:mvn/version "0.0.3"}
133+
orchestra/orchestra {:mvn/version "2021.01.01-1"}
120134
io.github.cognitect-labs/test-runner
121135
{:git/url "https://github.com/cognitect-labs/test-runner.git"
122-
:git/sha "2d69f33d7980c3353b246c28f72ffeafbd9f2fab"}
136+
:git/sha "3f288f1f16d167723ad87cc35b1dfee3c1681e10"}
123137
com.yetanalytics.lrs/test-runner
124138
{:git/url "https://github.com/yetanalytics/lrs-test-runner.git"
125139
:git/sha "8bd5be7816288e85f5c07fc11bf8cf53667e72da"
126140
:exclusions [org.clojure/data.json]}
127141
com.yetanalytics/datasim
128142
{:mvn/version "0.4.4"
129-
:exclusions [org.clojure/clojure
130-
com.yetanalytics/xapi-schema]}}}
143+
:exclusions [org.clojure/clojure
144+
com.yetanalytics/xapi-schema]}}}
131145
;; Build alias invoked like clojure -Xbuild uber
132146
:build
133147
{:replace-deps {io.github.clojure/tools.build {:git/tag "v0.10.0"

dev-resources/bench/query_input.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
[
22
{},
33
{"ascending": "true"},
4-
{"agent": "{\"mbox\": \"mailto:bob@example.com\"}"},
5-
{"agent": "{\"mbox\": \"mailto:bob@example.com\"}", "related_agents": "true"},
4+
{"agent": "{\"mbox\": \"mailto:bob@example.org\"}"},
5+
{"agent": "{\"mbox\": \"mailto:bob@example.org\"}", "related_agents": "true"},
66
{"activity": "https://w3id.org/xapi/video/activity-type/video"},
77
{"activity": "https://w3id.org/xapi/video/activity-type/video", "related_activities": "true"},
88
{"registration": "00000000-4000-8000-0000-000000000000"},

0 commit comments

Comments
 (0)