Merge pull request #464 from yetanalytics/csv-export

[SQL-281] CSV export

Author: kelvinqian00

Makefile

@@ -2,7 +2,7 @@
 
 # Version of LRS Admin UI to use
 
-LRS_ADMIN_UI_VERSION ?= v0.2.0
+LRS_ADMIN_UI_VERSION ?= v0.2.2
 LRS_ADMIN_UI_LOCATION ?= https://github.com/yetanalytics/lrs-admin-ui/releases/download/${LRS_ADMIN_UI_VERSION}/lrs-admin-ui.zip
 LRS_ADMIN_ZIPFILE ?= lrs-admin-ui-${LRS_ADMIN_UI_VERSION}.zip
 

deps.edn

@@ -4,6 +4,7 @@
   org.clojure/clojure                      {:mvn/version "1.11.2"}
   org.clojure/tools.logging                {:mvn/version "1.1.0"}
   org.clojure/core.memoize                 {:mvn/version "1.0.250"}
+  org.clojure/data.csv                     {:mvn/version "1.1.0"}
   clojure-interop/java.security            {:mvn/version "1.0.5"}
   org.clojure/core.async                   {:mvn/version "1.6.681"}
   ;; Util deps
@@ -49,13 +50,11 @@
   less-awful-ssl/less-awful-ssl   {:mvn/version "1.0.6"}
   xyz.capybara/clamav-client      {:mvn/version "2.1.2"}
   ;; Yet Analytics deps
-
   com.yetanalytics/lrs
   {:mvn/version "1.3.0"
    :exclusions  [org.clojure/clojure
                  org.clojure/clojurescript
                  com.yetanalytics/xapi-schema]}
-
   com.yetanalytics/xapi-schema
   {:mvn/version "1.4.0"
    :exclusions  [org.clojure/clojure
@@ -64,15 +63,15 @@
   {:mvn/version "0.1.4"
    :exclusions  [org.clojure/clojure
                  org.clojure/clojurescript]}
+  com.yetanalytics/pathetic
+  {:mvn/version "0.5.0"}
   com.yetanalytics/pedestal-oidc
   {:mvn/version "0.0.8"
    :exclusions  [org.clojure/clojure
                  buddy/buddy-sign]}
-
   com.yetanalytics/lrs-reactions
   {:mvn/version "0.0.1"
    :exclusions [org.clojure/clojure]}
-
   com.yetanalytics/gen-openapi
   {:mvn/version "0.0.4"
    :exclusions  [org.clojure/clojure

doc/endpoints.md

@@ -42,6 +42,12 @@ The response body contains a newly generated JSON Web Token (JWT) on success. A
 - `GET http://example.org/admin/creds`: Read all credential pairs and their associated scopes for a particular account (denoted by the JWT).
 - `DELETE http://example.org/admin/creds`: Delete an existing credential pair, given by the `api-key` and `secret-key` properties in the request body, as well as any associated scopes.
 
+#### Statement CSV Download
+
+- `GET http://example.org/admin/csv/auth`: Return a one-time JWT for use for `/admin/csv`, used in order to use the latter endpoint as a `download`
+attribute for HTML anchor tags and authenticate without headers.
+- `GET http://example.org/admin/csv`: Download statements in the LRS as a CSV filestream. This endpoint accepts the statement query parameters defined in the [xAPI spec](https://github.com/adlnet/xAPI-Spec/blob/master/xAPI-Communication.md#213-get-statements), but allows two additional parameters: the one-time JWT `token` string and the URL-encoded `property-path` vector strings.
+
 #### Misc Admin Routes
 
 - `GET http://example.org/admin/env`: Get select environment variables about the configuration which may aid in client-side operations.

doc/env_vars.md

@@ -101,6 +101,7 @@ The following options are used for advanced database performance tuning and may
 | `LRSQL_API_SECRET_DEFAULT`      | `apiSecretDefault`      | The secret API key that seeds the credential table. Optional, and is ignored if no seed admin account is set.                                                                                                                            | Not set                                                                               |
 | `LRSQL_STMT_GET_DEFAULT`        | `stmtGetDefault`        | The default `limit` value in a statement query. Queries default to this value if not explicitly set.                                                                                                                                     | `50`                                                                                  |
 | `LRSQL_STMT_GET_MAX`            | `stmtGetMax`            | The maximum allowed `limit` value for a statement query. If an explicit `limit` value exceeds this value, it will be overridden.                                                                                                         | `50`                                                                                  |
+| `LRSQL_STMT_GET_MAX_CSV`        | `stmtGetMaxCsv`         | The maximum allowed `limit` value applied to a statement download. If an explicit `limit` value exceeds this value, it will be overridden. If negative (e.g. the default value of `-1`), it will be ignored and all statements in the LRS will be downloaded. | -1                                                                                   |
 | `LRSQL_AUTHORITY_TEMPLATE`      | `authorityTemplate`     | The filepath to the Statement authority template file, which describes how authorities are constructed during statement insertion. If the file is not found, the system defaults to a default authority function.                        | <details>`config/authority.json.template`<summary>(Filepath)</summary></details>      |
 | `LRSQL_AUTHORITY_URL`           | `authorityUrl`          | The URL that is set as the `authority-url` value when constructing an authority from a template.                                                                                                                                         | `http://example.org`                                                                  |
 | `LRSQL_OIDC_AUTHORITY_TEMPLATE` | `oidcAuthorityTemplate` | Like `LRSQL_AUTHORITY_TEMPLATE`, but only used when forming an authority from an OIDC access token.                                                                                                                                      | <details>`config/oidc_authority.json.template`<summary>(Filepath)</summary></details> |

resources/lrsql/config/prod/default/lrs.edn

@@ -4,6 +4,7 @@
  :api-secret-default      #or [#env LRSQL_API_SECRET_DEFAULT nil]
  :stmt-get-default        #long #or [#env LRSQL_STMT_GET_DEFAULT 50]
  :stmt-get-max            #long #or [#env LRSQL_STMT_GET_MAX 50]
+ :stmt-get-max-csv        #long #or [#env LRSQL_STMT_GET_MAX_CSV -1]
  :stmt-url-prefix         "/xapi" ; overriden by ::webserver/url-prefix
  :authority-template      #or [#env LRSQL_AUTHORITY_TEMPLATE "config/authority.json.template"]
  :authority-url           #or [#env LRSQL_AUTHORITY_URL "http://example.org"]

resources/lrsql/config/test/default/lrs.edn

@@ -4,6 +4,7 @@
  :api-secret-default      "password"
  :stmt-get-default        50
  :stmt-get-max            50
+ :stmt-get-max-csv        nil
  :stmt-url-prefix         "/xapi"
  :authority-template      "config/authority.json.template"
  :authority-url           "http://example.org"

src/db/postgres/lrsql/postgres/record.clj

@@ -1,6 +1,7 @@
 (ns lrsql.postgres.record
   (:require [com.stuartsierra.component :as cmp]
             [hugsql.core :as hug]
+            [next.jdbc :as jdbc]
             [lrsql.backend.data :as bd]
             [lrsql.backend.protocol :as bp]
             [lrsql.init :refer [init-hugsql-adapter!]]
@@ -18,6 +19,8 @@
 (hug/def-db-fns "lrsql/postgres/sql/update.sql")
 (hug/def-db-fns "lrsql/postgres/sql/delete.sql")
 
+(hug/def-sqlvec-fns "lrsql/postgres/sql/query.sql")
+
 ;; Define record
 #_{:clj-kondo/ignore [:unresolved-symbol]} ; Shut up VSCode warnings
 (defrecord PostgresBackend [tuning]
@@ -77,7 +80,8 @@
       (add-statement-to-actor-cascading-delete! tx))
     (when (some? (query-varchar-exists tx))
       (convert-varchars-to-text! tx))
-    (create-blocked-jwt-table! tx))
+    (create-blocked-jwt-table! tx)
+    (alter-blocked-jwt-add-one-time-id! tx))
 
   bp/BackendUtil
   (-txn-retry? [_ ex]
@@ -103,6 +107,12 @@
     (query-statement-exists tx input))
   (-query-statement-descendants [_ tx input]
     (query-statement-descendants tx input))
+  (-query-statements-lazy [_ tx input]
+    (let [sqlvec (query-statements-sqlvec input)]
+      (jdbc/plan tx sqlvec {:fetch-size  4000
+                            :concurrency :read-only
+                            :cursors     :close
+                            :result-type :forward-only})))
 
   bp/ActorBackend
   (-insert-actor! [_ tx input]
@@ -201,10 +211,16 @@
   bp/JWTBlocklistBackend
   (-insert-blocked-jwt! [_ tx input]
     (insert-blocked-jwt! tx input))
+  (-insert-one-time-jwt! [_ tx input]
+    (insert-one-time-jwt! tx input))
+  (-update-one-time-jwt! [_ tx input]
+    (update-one-time-jwt! tx input))
   (-delete-blocked-jwt-by-time! [_ tx input]
     (delete-blocked-jwt-by-time! tx input))
   (-query-blocked-jwt [_ tx input]
     (query-blocked-jwt-exists tx input))
+  (-query-one-time-jwt [_ tx input]
+    (query-one-time-jwt-exists tx input))
 
   bp/CredentialBackend
   (-insert-credential! [_ tx input]

src/db/postgres/lrsql/postgres/sql/ddl.sql

@@ -499,3 +499,10 @@ CREATE TABLE IF NOT EXISTS blocked_jwt (
   evict_time TIMESTAMP WITH TIME ZONE
 );
 CREATE INDEX IF NOT EXISTS blocked_jwt_evict_time_idx ON blocked_jwt(evict_time);
+
+/* Migration 2025-03-05 - Add One-Time ID to Blocklist Table */
+
+-- :name alter-blocked-jwt-add-one-time-id!
+-- :command :execute
+-- :doc Add the column `blocked_jwt.one_time_id` for one-time JWTs; JWTs with one-time IDs are not considered blocked yet.
+ALTER TABLE IF EXISTS blocked_jwt ADD COLUMN IF NOT EXISTS one_time_id UUID UNIQUE;

src/db/postgres/lrsql/postgres/sql/insert.sql

@@ -171,3 +171,13 @@ INSERT INTO blocked_jwt (
 ) VALUES (
   :jwt, :eviction-time
 );
+
+-- :name insert-one-time-jwt!
+-- :command :insert
+-- :result :affected
+-- :doc Insert a `:jwt` and a `:eviction-time` with `:one-time-id` into the blocklist.
+INSERT INTO blocked_jwt (
+  jwt, evict_time, one_time_id
+) VALUES (
+  :jwt, :eviction-time, :one-time-id
+);

src/db/postgres/lrsql/postgres/sql/query.sql

@@ -67,7 +67,7 @@ WHERE stmt.is_voided = FALSE
 --~ (when (:registration params)   "AND stmt.registration = :registration")
 --~ (when (:authority-ifis params) "AND :frag:postgres-auth-subquery")
 --~ (if (:ascending? params)       "ORDER BY stmt.id ASC" "ORDER BY stmt.id DESC")
-LIMIT :limit
+--~ (when (:limit params)          "LIMIT :limit")
 
 /* Note: We sort by both the PK and statement ID in order to force the query
    planner to avoid scanning on `stmt_a.id` first, which is much slower than
@@ -92,7 +92,7 @@ WHERE stmt_a.is_voided = FALSE
 --~ (when (:authority-ifis params) "AND :frag:postgres-auth-subquery")
 /*~ (if (:ascending? params)       "ORDER BY (stmt_a.id, stmt_a.statement_id) ASC"
                                    "ORDER BY (stmt_a.id, stmt_a.statement_id) DESC") ~*/
-LIMIT :limit
+--~ (when (:limit params)          "LIMIT :limit")
 
 -- :name query-statements
 -- :command :query
@@ -107,7 +107,7 @@ FROM (
   (:frag:postgres-stmt-ref-subquery-frag))
 AS all_stmt
 --~ (if (:ascending? params) "ORDER BY all_stmt.id ASC" "ORDER BY all_stmt.id DESC")
-LIMIT :limit;
+--~ (when (:limit params)    "LIMIT :limit")
 
 /* Statement Object Queries */
 
@@ -433,6 +433,15 @@ WHERE reaction_id IS NOT NULL;
 -- :name query-blocked-jwt-exists
 -- :command :query
 -- :result :one
--- :doc Query that `:jwt` is in the blocklist.
+-- :doc Query that `:jwt` is in the blocklist. Excludes JWTs where `one_time_id` is not null.
 SELECT 1 FROM blocked_jwt
-WHERE jwt = :jwt;
+WHERE jwt = :jwt
+AND one_time_id IS NULL;
+
+-- :name query-one-time-jwt-exists
+-- :command :query
+-- :result :one
+-- :doc Query that `:jwt` with `:one-time-id` exists.
+SELECT 1 FROM blocked_jwt
+WHERE jwt = :jwt
+AND one_time_id = :one-time-id;