Merge pull request #4886 from mathesar-foundation/release-0.7.0

Release 0.7.0

Author: zackkrida

.github/ISSUE_TEMPLATE/tech_debt.md

@@ -0,0 +1,33 @@
+---
+name: Tech debt
+about: Improve Mathesar's codebase
+labels: "affects: technical debt, needs: triage"
+---
+
+## Description
+<!-- A clear and concise description of what the problem is. -->
+
+### Affects users
+<!-- Give a rating. Delete all but one -->
+**High**
+**Medium**
+**Low**
+**None**
+
+<!-- If not "None", how does this debt affect users? -->
+
+### Affects developers
+<!-- Give a rating. Delete all but one -->
+**High**
+**Medium**
+**Low**
+**None**
+
+<!-- If not "None", how does this debt affect developers? -->
+
+## Suggested change
+<!-- What is the proposed change to the codebase. -->
+
+## Additional context
+<!-- Add any other context about the problem or screenshots here. -->
+

DEVELOPER_GUIDE.md

@@ -10,7 +10,7 @@ Mathesar is built using:
 - [Python](https://www.python.org/) for the backend
 - [Django](https://www.djangoproject.com/) for the web application
 - [SQLAlchemy](https://www.sqlalchemy.org/) to talk to the database
-- [Django REST Framework](https://www.django-rest-framework.org/) for the API 
+- [Django REST Framework](https://www.django-rest-framework.org/) for the API
 - [Svelte](https://svelte.dev/) and [TypeScript](https://www.typescriptlang.org/) for the frontend
 
 ## Local development setup
@@ -77,7 +77,7 @@ We use [pytest](https://docs.pytest.org) for our backend tests.
     ```
     docker exec mathesar_service_dev pytest -k "test_name"
     ```
-    
+
 - See the [pytest documentation](https://docs.pytest.org/en/latest/how-to/usage.html), or run pytest with the `--help` flag to learn about more options for running tests.
 
 - Run all SQL tests:
@@ -183,7 +183,7 @@ Django uses gettext, which require the `.po` files to be compiled into a more ef
 ## Translation process
 
 - We use [Transifex](https://app.transifex.com/mathesar/mathesar/dashboard/) for managing our translation process.
-- You'll need to be a member of the Mathesar organization in Transifex, inorder to work with translations. Please reach out to us for information on how to join. 
+- You'll need to be a member of the Mathesar organization in Transifex, inorder to work with translations. Please reach out to us for information on how to join.
 
 ### For Translators
 
@@ -241,12 +241,34 @@ If you'd like to manually push or pull translations, follow the instructions in
     ```
     docker exec -it mathesar_dev_db psql -U mathesar
     ```
- 
+
+## Working with file attachments
+
+Mathesar has a feature which allows users to upload files through the UI and store references to those file in data. Uploaded files are stored in an S3-compatible storage provider.
+
+To work with this feature in your local development environment, you'll need to take some extra steps to enable it:
+
+1. Copy `file_storage.yml.example` to `file_storage.yml`.
+
+1. Restart docker with the following command
+
+    ```
+    docker compose -f docker-compose.dev.yml up dev-service obj-store
+    ```
+
+    Notice that we start the `obj-store` service too, in addition to the normal `dev-service`.
+
+1. Log in to the file storage backend UI at http://localhost:9001 with `mathesar`/`mathesar`. Create a new storage bucket in the UI called "mathesar-storages".
+
+At this point you can run Mathesar and create new "File" columns in tables. Those columns will present a UI that allows you to attach files to table cells.
+
+With the `obj-store` service running, you can return to http://localhost:9001 to view and manage your uploaded files.
+
 ## Troubleshooting
 
 ### Permissions within Windows
 
-- Running Script in powershell is disabled by default in windows , you have to change permission to run scripts  [Official Docs ](https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.security/set-executionpolicy?view=powershell-7.2) 
+- Running Script in powershell is disabled by default in windows , you have to change permission to run scripts  [Official Docs ](https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.security/set-executionpolicy?view=powershell-7.2)
 
 ### Fixing line endings on Windows
 

db/columns.py

@@ -8,6 +8,7 @@
 DESCRIPTION = "description"
 NAME = "name"
 NULLABLE = "nullable"
+CAST_OPTIONS = "cast_options"
 
 
 def get_column_info_for_table(table, conn):
@@ -77,6 +78,7 @@ def _transform_column_alter_dict(data):
         "id": <int>,
         "name": <str>,
         "type": <str>,
+        "cast_options": <dict>,
         "type_options": <dict>,
         "nullable": <bool>,
         "default": {"value": <any>}
@@ -87,6 +89,7 @@ def _transform_column_alter_dict(data):
     {
         "attnum": <int>,
         "type": {"name": <str>, "options": <dict>},
+        "cast_options": {"curr_pref": <str>, "curr_suff": <str>, "decimal_p": <str>, "group_sep": <str>, "mathesar_casting": <bool>},
         "name": <str>,
         "not_null": <bool>,
         "default": <any>,
@@ -99,12 +102,14 @@ def _transform_column_alter_dict(data):
     """
     type_ = {"name": data.get('type'), "options": data.get('type_options')}
     new_type = {k: v for k, v in type_.items() if v} or None
+    cast_options = data.get(CAST_OPTIONS)
     nullable = data.get(NULLABLE)
     not_null = not nullable if nullable is not None else None
     column_name = (data.get(NAME) or '').strip() or None
     raw_alter_def = {
         "attnum": data["id"],
         "type": new_type,
+        "cast_options": cast_options,
         "not_null": not_null,
         "name": column_name,
         "description": data.get("description")

db/records.py

@@ -195,3 +195,35 @@ def patch_record_in_table(
         _json_or_none(table_record_summary_templates),
     ).fetchone()[0]
     return result
+
+
+def insert_from_select(
+    conn,
+    src_table_id,
+    dst_table_id,
+    mappings
+):
+    """
+    Insert multiple records from a given source table to a destination/target table,
+    returning the number of records inserted.
+
+    Args:
+      src_tab_id: The OID of the source table.(OID if temp table if inserting into existing table).
+      dst_tab_id: The OID of the destination/target table.
+      mappings: The column mappings b/w src and dst tables based on which data will be inserted.
+
+    mappings should have the following form:
+    [
+      {"src_table_attnum": 1, "dst_table_attnum": 2},
+      {"src_table_attnum": 3, "dst_table_attnum": 3},
+      {...}
+    ]
+    """
+    result = db_conn.exec_msar_func(
+        conn,
+        'insert_from_select',
+        src_table_id,
+        dst_table_id,
+        json.dumps(mappings)
+    ).fetchone()[0]
+    return result

db/sql/00_msar_all_objects_table.sql

@@ -38,6 +38,7 @@ INSERT INTO msar.all_mathesar_objects VALUES
   ('__msar', '__msar.add_constraints(text,__msar.con_def[])', 'FUNCTION', NULL),
   ('__msar', '__msar.add_table(text,__msar.col_def[],__msar.con_def[])', 'FUNCTION', NULL),
   ('__msar', '__msar.build_cast_expr(text,text)', 'FUNCTION', NULL),
+  ('__msar', '__msar.build_cast_expr(text,text,jsonb)', 'FUNCTION', NULL),
   ('__msar', '__msar.build_col_def_text(__msar.col_def)', 'FUNCTION', NULL),
   ('__msar', '__msar.build_col_default_expr(oid,integer,text,jsonb,text)', 'FUNCTION', NULL),
   ('__msar', '__msar.build_col_drop_default_expr(oid,integer,text,jsonb)', 'FUNCTION', NULL),
@@ -574,6 +575,7 @@ INSERT INTO msar.all_mathesar_objects VALUES
   ('msar', 'msar.add_pkey_column(regclass,msar.pkey_kind,text)', 'FUNCTION', NULL),
   ('msar', 'msar.add_record_to_table(oid,jsonb,boolean)', 'FUNCTION', NULL),
   ('msar', 'msar.add_record_to_table(oid,jsonb,boolean,jsonb)', 'FUNCTION', NULL),
+  ('msar', 'msar.add_temp_table(text,__msar.col_def[])', 'FUNCTION', NULL),
   ('msar', 'msar.add_time_to_vector(point,time without time zone)', 'FUNCTION', NULL),
   ('msar', 'msar.all_mathesar_objects', 'TABLE', NULL),
   ('msar', 'msar.all_mathesar_objects', 'TYPE', NULL),
@@ -595,6 +597,7 @@ INSERT INTO msar.all_mathesar_objects VALUES
   ('msar', 'msar.build_grouping_results_jsonb_expr(oid,text,jsonb)', 'FUNCTION', NULL),
   ('msar', 'msar.build_groups_cte_expr(oid,text,jsonb)', 'FUNCTION', NULL),
   ('msar', 'msar.build_groups_cte_expr(oid,text,text,jsonb)', 'FUNCTION', NULL),
+  ('msar', 'msar.insert_from_select(regclass,regclass,jsonb)', 'FUNCTION', NULL),
   ('msar', 'msar.build_insert_lookup_table(jsonb,jsonb)', 'FUNCTION', NULL),
   ('msar', 'msar.build_linked_record_summaries_ctes(oid,jsonb)', 'FUNCTION', NULL),
   ('msar', 'msar.build_order_by_expr(oid,jsonb)', 'FUNCTION', NULL),
@@ -1042,6 +1045,7 @@ INSERT INTO msar.all_mathesar_objects VALUES
   ('msar', 'msar.delete_records_from_table(oid,jsonb)', 'FUNCTION', NULL),
   ('msar', 'msar.describe_column_default(regclass,smallint)', 'FUNCTION', NULL),
   ('msar', 'msar.downsize_table_sample(numeric)', 'FUNCTION', NULL),
+  ('msar', 'msar.drop_col_default(regclass,smallint)', 'FUNCTION', NULL),
   ('msar', 'msar.drop_columns(oid,integer[])', 'FUNCTION', NULL),
   ('msar', 'msar.drop_columns(text,text,text[])', 'FUNCTION', NULL),
   ('msar', 'msar.drop_constraint(oid,oid)', 'FUNCTION', NULL),
@@ -1171,6 +1175,7 @@ INSERT INTO msar.all_mathesar_objects VALUES
   ('msar', 'msar.point_to_time(point)', 'FUNCTION', NULL),
   ('msar', 'msar.prepare_table_for_import(oid,text,jsonb,boolean,text,text,text,text,text)', 'FUNCTION', NULL),
   ('msar', 'msar.prepare_table_for_import(oid,text,text[],text)', 'FUNCTION', NULL),
+  ('msar', 'msar.prepare_temp_table_for_import(text,text[])', 'FUNCTION', NULL),
   ('msar', 'msar.process_col_alter_jsonb(oid,jsonb)', 'FUNCTION', NULL),
   ('msar', 'msar.process_col_def_jsonb(oid,jsonb,boolean,boolean)', 'FUNCTION', NULL),
   ('msar', 'msar.process_con_def_jsonb(oid,jsonb)', 'FUNCTION', NULL),
@@ -1183,6 +1188,8 @@ INSERT INTO msar.all_mathesar_objects VALUES
   ('msar', 'msar.replace_database_privileges_for_roles(jsonb)', 'FUNCTION', NULL),
   ('msar', 'msar.replace_schema_privileges_for_roles(regnamespace,jsonb)', 'FUNCTION', NULL),
   ('msar', 'msar.replace_table_privileges_for_roles(regclass,jsonb)', 'FUNCTION', NULL),
+  ('msar', 'msar.retype_column(regclass,smallint,text)', 'FUNCTION', NULL),
+  ('msar', 'msar.retype_column(regclass,smallint,text,jsonb)', 'FUNCTION', NULL),
   ('msar', 'msar.reset_mash(regclass,smallint,jsonb)', 'FUNCTION', NULL),
   ('msar', 'msar.role_info_table()', 'FUNCTION', NULL),
   ('msar', 'msar.sanitize_direction(text)', 'FUNCTION', NULL),
@@ -1191,8 +1198,11 @@ INSERT INTO msar.all_mathesar_objects VALUES
   ('msar', 'msar.search_records_from_table(oid,jsonb,integer,boolean)', 'FUNCTION', NULL),
   ('msar', 'msar.search_records_from_table(oid,jsonb,integer,boolean,jsonb)', 'FUNCTION', NULL),
   ('msar', 'msar.search_records_from_table(oid,jsonb,integer,integer,boolean,jsonb)', 'FUNCTION', NULL),
+  ('msar', 'msar.set_col_default(regclass,smallint,text)', 'FUNCTION', NULL),
   ('msar', 'msar.set_members_to_role(regrole,oid[])', 'FUNCTION', NULL),
   ('msar', 'msar.set_not_null(regclass,smallint,boolean)', 'FUNCTION', NULL),
+  ('msar', 'msar.set_old_col_default(regclass,smallint,text,text,boolean)', 'FUNCTION', NULL),
+  ('msar', 'msar.set_old_col_default(regclass,smallint,text,text,boolean,jsonb)', 'FUNCTION', NULL),
   ('msar', 'msar.set_pkey_column(regclass,integer,msar.pkey_kind,boolean)', 'FUNCTION', NULL),
   ('msar', 'msar.set_schema_description(oid,text)', 'FUNCTION', NULL),
   ('msar', 'msar.table_info_table()', 'FUNCTION', NULL),