feat: wip replace json endpoints with htmx

Author: spwoodcock

docs/images/qfield-xlsformconverter.png

@@ -0,0 +1,89 @@
+# Setting Up QField Manually
+
+- FieldTM should be able to configure a QField project automatically.
+- Failing this, the steps below help to configure a QField project
+  manually.
+
+## 1. Create An XLSForm
+
+- An XLSForm should be created, with question types as normal
+  (choice fields, text input, image upload, etc).
+- It is **essential** to ensure that a question of type
+  `geopoint`, `geotrace`, or `geoshape` is included in the form,
+  so that QGIS knows which type of geometries you wish to collect.
+  - Alternatively, if you are mapping existing geometries, the
+    geometry type can be determined by QGIS by selecting the
+    geometry later in a later stage.
+
+## 2. Convert XLSForm To QGIS
+
+- Open QGIS and install the XLSFormConverter plugin (by OpenGISch).
+- Go to the processing toolbox, and open the XLSFormConverter
+  processing tool.
+
+You will be presented with a set of field that need to be filled
+out:
+
+![xlsformconverter](../images/qfield-xlsformconverter.png)
+
+- Select the XLSForm you have created as the first field.
+- Ensure the 'Project language' matches exactly what is defined
+  in the XLSForm.
+- If you are mapping existing geometries, here you can load the
+  layer into QGIS, then select this layer to 'Pre-fill the project...'.
+- It is also a good idea to load the project AOI into QGIS, then
+  select this layer as the 'Project Extent'. This will ensure
+  QField zooms to this location on first load (if you don’t do this,
+  users will have to navigate to the AOI manually or using GPS).
+- It's good to select the project CRS as standard (EPSG:4326 generally).
+- Once you have completed this, a survey layer will be added to your
+  QGIS project. By default, this will output to a temporary directory.
+  Open the project in the temporary directory to continue.
+- You can then add a boundary for the AOI and a basemap as you would
+  in any normal QGIS project.
+
+## 3. Configure The Project Further
+
+- You can configure the project in any way that you like, within the
+  constraints of what you can do in QGIS (eg. styling, labels, etc).
+  QField users will see those same styles when they use the app.
+- Basemaps such as satellite or drone imagery can be added as layers,
+  into a 'mutually exclusive' basemap group if preferred.
+- Symbology can be updated to vary based on data type. For example,
+  if a question asks the user to define the feature 'category', the
+  symbol on the map can reflect the categories (with a legend).
+- You may wish to add additional layers to the project, e.g.:
+  - Task boundaries with mapper assignments.
+  - Location of cell towers to get phone signal.
+  - Contours to show terrain.
+  - Other georeferenced data, such as scanned in archive maps,
+    or drawn community maps.
+
+## 4. Syncing The Project To The Cloud
+
+- Install QField Sync in QGIS.
+- You need to register an account at
+  [https://app.qfield.cloud](https://app.qfield.cloud).
+- QField Cloud is the bridge between QGIS and the users'
+  interaction with the project.
+- Log in to QField via the QField Sync plugin in QGIS.
+- Click on the cloud icon and click create new project.
+- Choose 'Convert currently open project to a cloud project'.
+- Add a name, choose HOTOSM as the owner, then convert and upload.
+
+## 5. Open In The QField App
+
+- Load the QField app and log in with your account.
+- Download the newly created project and open it.
+- It's possible to share the project with other users
+  by a QRCode to scan.
+- Mapped data can be synced at an interval (defined in the
+  QFieldSync project settings), or manually by the user.
+- The manager can then sync the uploaded data into their
+  desktop QGIS project, to track progress.
+
+## Useful reference and resources
+
+- <https://xlsform.org/en/>
+- <https://docs.getodk.org/form-question-types/>
+- <https://docs.qfield.org/>

docs/manuals/qfield-by-hand.md

@@ -83,6 +83,7 @@ nav:
       - Field Mapping Examples: manuals/field-mapping-examples.md
       - Visualising Data Externally: https://docs.getodk.org/tutorial-mapping-households
       - ODKCollect Offline Maps: https://docs.getodk.org/collect-offline-maps
+      - QField Project By Hand: manuals/qfield-by-hand.md
   - Developer Guide:
       - Practices:
           - Dev Practices: https://docs.hotosm.org/dev-practices

mkdocs.yml

@@ -39,7 +39,7 @@
 
 from app.central import central_deps, central_schemas
 from app.config import settings
-from app.db.enums import DbGeomType, EntityState
+from app.db.enums import DbGeomType
 from app.db.models import DbProject, DbTemplateXLSForm
 from app.db.postgis_utils import (
     geojson_to_javarosa_geom,
@@ -52,25 +52,6 @@
 log = logging.getLogger(__name__)
 
 
-STATUS_VISUALS = {
-    EntityState.MARKED_BAD.value: {
-        "fill": "#ff0000",
-        "marker-color": "#ff0000",
-        "stroke": "#cc0000",
-    },
-    EntityState.SURVEY_SUBMITTED.value: {
-        "fill": "#00ff00",
-        "marker-color": "#00ff00",
-        "stroke": "#00cc00",
-    },
-    EntityState.VALIDATED.value: {
-        "fill": "#008000",
-        "marker-color": "#008000",
-        "stroke": "#006600",
-    },
-}
-
-
 def get_odk_project(odk_central: Optional[central_schemas.ODKCentral] = None):
     """Helper function to get the OdkProject with credentials."""
     if odk_central:

src/backend/app/central/central_crud.py

@@ -200,22 +200,24 @@ async def upload_project_xlsform(
     )
 
     # Write XLS form content to db
+    # Ensure BytesIO is at the beginning before reading
+    project_xlsform.seek(0)
     xlsform_db_bytes = project_xlsform.getvalue()
     if len(xlsform_db_bytes) == 0 or not xform_id:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail="There was an error modifying the XLSForm!",
         )
-    log.debug(f"Setting project XLSForm db data for xFormId: {xform_id}")
+    log.debug(
+        f"Setting project XLSForm db data for xFormId: {xform_id}, bytes length: {len(xlsform_db_bytes)}"
+    )
     await DbProject.update(
         db,
         project_id,
-        ProjectUpdate(
-            xlsform_content=xlsform_db_bytes,
-            odk_form_id=xform_id,
-        ),
+        ProjectUpdate(xlsform_content=xlsform_db_bytes),
     )
     await db.commit()
+    log.debug(f"Successfully saved XLSForm to database for project {project_id}")
 
     return {"message": "Your form is valid"}
 
@@ -318,7 +320,6 @@ async def refresh_appuser_token(
     project = current_user.get("project")
     project_id = project.id
     project_odk_id = project.external_project_id
-    project_xform_id = project.odk_form_id
     # ODK credentials not stored on project, use None to fall back to env vars
     project_odk_creds = None
 
@@ -371,7 +372,6 @@ async def upload_form_media(
     project = current_user.get("project")
     project_id = project.id
     project_odk_id = project.external_project_id
-    project_xform_id = project.odk_form_id
     # ODK credentials not stored on project, use None to fall back to env vars
     project_odk_creds = None
 
@@ -453,7 +453,6 @@ async def list_form_media(
     project = current_user.get("project")
     project_id = project.id
     project_odk_id = project.external_project_id
-    project_xform_id = project.odk_form_id
     # ODK credentials not stored on project, use None to fall back to env vars
     project_odk_creds = None
 
@@ -495,7 +494,6 @@ async def get_form_media(
     project = current_user.get("project")
     project_id = project.id
     project_odk_id = project.external_project_id
-    project_xform_id = project.odk_form_id
     # ODK credentials not stored on project, use None to fall back to env vars
     project_odk_creds = None
 

src/backend/app/central/central_routes.py

@@ -17,7 +17,7 @@
 #
 """Enum definitions to translate values into human enum strings."""
 
-from enum import Enum, IntEnum, StrEnum
+from enum import Enum, StrEnum
 
 
 class ProjectStatus(StrEnum, Enum):
@@ -29,14 +29,6 @@ class ProjectStatus(StrEnum, Enum):
     COMPLETED = "COMPLETED"
 
 
-class OrganisationType(StrEnum, Enum):
-    """An organisation's subscription type."""
-
-    FREE = "FREE"
-    DISCOUNTED = "DISCOUNTED"
-    FULL_FEE = "FULL_FEE"
-
-
 class ProjectPriority(StrEnum, Enum):
     """All possible project priority levels."""
 
@@ -58,59 +50,6 @@ class ProjectRole(StrEnum, Enum):
     PROJECT_ADMIN = "PROJECT_ADMIN"
 
 
-class MappingLevel(StrEnum, Enum):
-    """The mapping level the mapper has achieved."""
-
-    BEGINNER = "BEGINNER"
-    INTERMEDIATE = "INTERMEDIATE"
-    ADVANCED = "ADVANCED"
-
-
-class TaskEvent(StrEnum, Enum):
-    """Task events via API.
-
-    `MAP` -- Set to *locked for mapping*, i.e. mapping in progress.
-    `FINISH` -- Set to *unlocked to validate*, i.e. is mapped.
-    `VALIDATE` -- Set to *locked for validation*, i.e. validation in progress.
-    `GOOD` -- Set the state to *unlocked done*.
-    `BAD` -- Set the state *unlocked to map* again, to be mapped once again.
-    `SPLIT` -- Set the state *unlocked done* then generate additional
-        subdivided task areas.
-    `MERGE` -- Set the state *unlocked done* then generate additional
-        merged task area.
-    `ASSIGN` -- For a requester user to assign a task to another user.
-        Set the state *locked for mapping* passing in the required user id.
-        Also notify the user they should map the area.
-    `COMMENT` -- Keep the state the same, but simply add a comment.
-    """
-
-    MAP = "MAP"
-    FINISH = "FINISH"
-    VALIDATE = "VALIDATE"
-    GOOD = "GOOD"
-    BAD = "BAD"
-    SPLIT = "SPLIT"
-    MERGE = "MERGE"
-    ASSIGN = "ASSIGN"
-    COMMENT = "COMMENT"
-    RESET = "RESET"
-
-
-class EntityState(IntEnum, Enum):
-    """State options for Entities in ODK.
-
-    NOTE here we started with int enums and it's hard to migrate.
-    NOTE we will continue to use int values in the form.
-    NOTE we keep BAD=6 for legacy reasons too.
-    """
-
-    READY = 0
-    OPENED_IN_ODK = 1
-    SURVEY_SUBMITTED = 2
-    VALIDATED = 5
-    MARKED_BAD = 6
-
-
 class TaskSplitType(StrEnum, Enum):
     """Task splitting type for area-splitter."""
 

src/backend/app/db/enums.py

@@ -323,6 +323,8 @@ class DbProject:
     updated_at: Optional[AwareDatetime] = None
     # Encrypted ODK appuser token (may be null until generated)
     odk_token: Optional[str] = None
+    # GeoJSON data extract stored directly in database (replaces S3 URL approach)
+    data_extract_geojson: Optional[dict] = None
 
     @classmethod
     async def one(
@@ -447,6 +449,10 @@ async def create(cls, db: AsyncConnection, project_in: Self) -> Self:
                 value_placeholders.append(f"ST_GeomFromGeoJSON(%({key})s)")
                 # Must be string json for db input
                 model_dump[key] = json.dumps(model_dump[key])
+            elif key == "data_extract_geojson" and isinstance(model_dump[key], dict):
+                # Convert GeoJSON dict to JSON string for JSONB column
+                value_placeholders.append(f"%({key})s::jsonb")
+                model_dump[key] = json.dumps(model_dump[key])
             else:
                 value_placeholders.append(f"%({key})s")
 
@@ -527,6 +533,12 @@ async def update(
             # Remove plaintext password if present
             model_dump.pop("external_project_password", None)
 
+        # Convert dict/JSONB fields to JSON strings for database
+        for key in list(model_dump.keys()):
+            if key == "data_extract_geojson" and isinstance(model_dump[key], dict):
+                # Convert GeoJSON dict to JSON string for JSONB column
+                model_dump[key] = json.dumps(model_dump[key])
+
         placeholders = [f"{key} = %({key})s" for key in model_dump.keys()]
 
         # NOTE we want a trackable hashtag DOMAIN-PROJECT_ID