Merge pull request #406 from yetanalytics/pg_varchar_to_text

Migration from PG VARCHAR to TEXT

Author: cliffcaseyyet

src/db/postgres/lrsql/postgres/record.clj

@@ -73,7 +73,9 @@
       (migrate-to-jsonb! tx)
       (migrate-to-json! tx))
     (when (nil? (check-statement-to-actor-cascading-delete tx))
-      (add-statement-to-actor-cascading-delete! tx)))
+      (add-statement-to-actor-cascading-delete! tx))
+    (when (some? (query-varchar-exists tx))
+      (convert-varchars-to-text! tx)))
 
   bp/BackendUtil
   (-txn-retry? [_ ex]

src/db/postgres/lrsql/postgres/sql/ddl.sql

@@ -441,3 +441,44 @@ AND pg_get_constraintdef(oid) LIKE '%ON DELETE CASCADE%'
 -- :doc Adds a cascading delete to delete st2actor entries when corresponding statements are deleted
 ALTER TABLE statement_to_actor DROP CONSTRAINT statement_fk;
 ALTER TABLE statement_to_actor ADD CONSTRAINT statement_fk FOREIGN KEY (statement_id) REFERENCES xapi_statement(statement_id) ON DELETE CASCADE;
+
+/* Migration 2024-05-29 - Universally Convert VARCHAR to TEXT */
+
+-- :name query-varchar-exists
+-- :command :query
+-- :result :one
+-- :doc Query to see if varchar->text conversion has not happened yet.
+SELECT 1 FROM information_schema.columns WHERE table_name = 'xapi_statement' AND column_name = 'verb_iri' and data_type = 'character varying';
+
+-- :name convert-varchars-to-text!
+-- :command :execute
+-- :doc Converts all known VARCHAR(255) fields into TEXT fields. Order of execution is critical for ifi constraints
+ALTER TABLE xapi_statement ALTER COLUMN verb_iri TYPE TEXT;
+
+-- Must drop constraints containing ifi (and rebuild after conversion) because conversion in place does not work for actor_fk or actor_idx composites
+ALTER TABLE statement_to_actor DROP CONSTRAINT actor_fk;
+ALTER TABLE actor DROP CONSTRAINT actor_idx;
+ALTER TABLE actor ALTER COLUMN actor_ifi TYPE TEXT;
+ALTER TABLE actor ADD CONSTRAINT actor_idx UNIQUE (actor_ifi, actor_type);
+ALTER TABLE statement_to_actor ALTER COLUMN actor_ifi TYPE TEXT;
+ALTER TABLE statement_to_actor ADD CONSTRAINT actor_fk FOREIGN KEY (actor_ifi, actor_type) REFERENCES actor(actor_ifi, actor_type);
+
+ALTER TABLE activity ALTER COLUMN activity_iri TYPE TEXT;
+ALTER TABLE attachment ALTER COLUMN attachment_sha TYPE TEXT;
+ALTER TABLE attachment ALTER COLUMN content_type TYPE TEXT;
+ALTER TABLE statement_to_activity ALTER COLUMN activity_iri TYPE TEXT;
+ALTER TABLE state_document ALTER COLUMN state_id TYPE TEXT;
+ALTER TABLE state_document ALTER COLUMN activity_iri TYPE TEXT;
+ALTER TABLE state_document ALTER COLUMN agent_ifi TYPE TEXT;
+ALTER TABLE state_document ALTER COLUMN content_type TYPE TEXT;
+ALTER TABLE activity_profile_document ALTER COLUMN profile_id TYPE TEXT;
+ALTER TABLE activity_profile_document ALTER COLUMN activity_iri TYPE TEXT;
+ALTER TABLE activity_profile_document ALTER COLUMN content_type TYPE TEXT;
+ALTER TABLE admin_account ALTER COLUMN username TYPE TEXT;
+ALTER TABLE admin_account ALTER COLUMN passhash TYPE TEXT;
+ALTER TABLE admin_account ALTER COLUMN oidc_issuer TYPE TEXT;
+ALTER TABLE lrs_credential ALTER COLUMN api_key TYPE TEXT;
+ALTER TABLE lrs_credential ALTER COLUMN secret_key TYPE TEXT;
+ALTER TABLE credential_to_scope ALTER COLUMN api_key TYPE TEXT;
+ALTER TABLE credential_to_scope ALTER COLUMN secret_key TYPE TEXT;
+ALTER TABLE reaction ALTER COLUMN title TYPE TEXT;

src/test/lrsql/lrs_test.clj

@@ -183,6 +183,16 @@
    :length      33
    :sha2        "7063d0a4cfa93373753ad2f5a6ffcf684559fb1df3c2f0473a14ece7d4edb06a"})
 
+;; Extremely long IRIs
+(def stmt-7
+  {"id"     "00000000-0000-4000-8000-000000000127"
+   "actor"  {"account"    {"homePage" "https://www.example.com/users/h894hf8934hf8934h8934hf8934h89h89f3h894hf8934hf8943gh8f34h89fh8934h8f934h8f943h89f34h89f34h89f34h89f34h89fh348fh3489fh438fh8934hf880234hg89234gh8934gh8349gh8349gh8349gh8943hg8934gh8934hg8493hg843hg8934h89g34hg8943h8934h8g943h89g34h89g34h8934h89g34h89g34h89g34h89g34h89g34h89g34h8g34h89g34h89g34h89g34h8g43h89gh3489gh3489g934h89fh3489fh8349fh8934h89f34h89f34h89fh43834h89fh4893fh8349fh8934fh8934hf8934hf8934h89f34h89f34h89f34h89f34h89f34h89fh3489f",
+                           "name"     "NothingToSeeHere"}
+             "objectType" "Agent"}
+   "verb"   {"id"         "http://example.com/verbs/24g890gh348h8934gh8349h89g34hg8934hg8934h3489gh8934hg3489h8g34h89g34h4389g34gh83h89gh8934hg894hgfuifbgnusrbgjrjkgbneruiognhuio34bnhuobgh4389gh48gh34g8493ghu34wenhgberuiohegr89h3458ghjerihertyuioernhjkogernguioh89gh348gh84ibhgiohsdrioghb349uih3489ghbu934hgi9erhjgiheru9gh943h89gh3894hu9ghu89werhguibhdsfuijgbnerjinuigernhuighjeriohjgioehuirgh4ioghjioerhg34uiogh349hgf3489gh34u89gh3489hg8943hg8934h89gh23489g34h89g34h89g34hg8934hg3489h4389gh3489hyg3489hg8934hg893h489gh3489gh89",
+             "display"    {"en-US" "Very Normal Verb"}}
+   "object" {"id"         "http://www.example.com/activities/38902fh23890fh2389fh238hf8923fh8239hf8923h829hf8923hf87923h89hf8h2389fh8239h238fh2389fh8923fh2389fh2389fh823fh3892fh8923hf2389hf8239fh8239fh8923h8392h823f9hf823h89f32hf8932h89f23h89f23h89f23h89f32h8923fh8f23hf23h823fh89f23h3892hf2389fh2389hf8932hf8923h89f3h8932hf893hf8923hf8932h238hf328hf8923h23f8ifh23uifh23uibfh23ubf23ifb23yi23bfyuifui23b23fuib3fui2b23fuifb23bfu32bfui23bui32bf23uibfui23bfui23bfui23bfui32bfui23bgh23uifbui23bfuib23uibf2ui3bfui23buifb23uibfu32b3uifbui"}})
+
 (deftest test-statement-fns
   (let [sys   (support/test-system)
         sys'  (component/start sys)
@@ -193,6 +203,7 @@
         id-2  (get stmt-2 "id")
         id-3  (get stmt-3 "id")
         id-4  (get stmt-4 "id")
+        id-7  (get stmt-7 "id")
         ts    "3000-01-01T01:00:00Z" ; Date far into the future
         agt-0 (-> stmt-0 (get "actor"))
         agt-1 (-> stmt-1 (get "actor") (dissoc "name"))
@@ -411,6 +422,10 @@
                              "description" {"en-US" "Multi Part Activity Description"
                                             "zh-CN" "多元部分Activity的简述"}}}}
              (lrsp/-get-activity lrs auth-ident {:activityId act-1}))))
+    
+    (testing "Extremely long IRIs"
+      (is (= {:statement-ids [id-7]}
+             (lrsp/-store-statements lrs auth-ident [stmt-7] []))))
 
     (component/stop sys')
     (support/unstrument-lrsql)))