Merge pull request #644 from MerginMaps/master

Backport 5.0 to maps

Author: MarcelGeo

.github/workflows/auto_tests.yml

@@ -3,9 +3,18 @@ name: Auto Tests
 on: push
 
 jobs:
-  server_tests:
+  tests:
     runs-on: ubuntu-24.04
 
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - suite: server
+            pytest_args: "-v --cov=mergin --cov-report=lcov mergin/tests"
+          - suite: migration
+            pytest_args: "-v mergin/test_migrations"
+
     services:
       postgres:
         image: postgres:14
@@ -15,13 +24,18 @@ jobs:
           POSTGRES_USER: postgres
         ports:
           - 5435:5432
-        # Set health checks to wait until postgres has started
         options: >-
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
 
+    env:
+      DB_USER: postgres
+      DB_PASSWORD: postgres
+      DB_HOST: localhost
+      DB_PORT: 5435
+
     steps:
       - name: Check out repository
         uses: actions/checkout@v3
@@ -36,9 +50,10 @@ jobs:
       - name: Run tests
         run: |
           cd server
-          pipenv run pytest -v --cov=mergin --cov-report=lcov mergin/tests
+          pipenv run pytest ${{ matrix.pytest_args }}
 
       - name: Coveralls
+        if: matrix.suite == 'server'
         uses: coverallsapp/github-action@v2
         with:
           base-path: server

server/mergin/sync/interfaces.py

@@ -110,6 +110,12 @@ def get_by_name(self, name):
         """
         pass
 
+    def get_by_names(self, names):
+        """
+        Return list of workspaces whose names are in the given collection.
+        """
+        pass
+
     @abstractmethod
     def get_by_project(self, project):
         """

server/mergin/sync/models.py

@@ -6,6 +6,7 @@
 import json
 import logging
 import os
+import re
 import threading
 import time
 import uuid
@@ -18,7 +19,9 @@
 from blinker import signal
 from flask_login import current_user
 from pygeodiff import GeoDiff
+from functools import cached_property
 from sqlalchemy import text, null, desc, nullslast, tuple_
+from sqlalchemy.orm import contains_eager, joinedload, load_only
 from sqlalchemy.dialects.postgresql import ARRAY, BIGINT, UUID, JSONB, ENUM, insert
 from sqlalchemy.types import String
 from sqlalchemy.ext.hybrid import hybrid_property
@@ -137,6 +140,12 @@ def workspace(self):
         project_workspace = current_app.ws_handler.get(self.workspace_id)
         return project_workspace
 
+    @cached_property
+    def _has_conflict(self) -> bool:
+        """True if any current project file matches a known conflict-copy pattern."""
+        pattern = r"(\.gpkg|\.qgs|.qgz)(.*conflict.*)|( \(.*conflict.*)"
+        return any(re.search(pattern, f.path) for f in self.files)
+
     def get_latest_files_cache(self) -> List[int]:
         """Get latest file history ids either from cached table or calculate them on the fly"""
         if self.latest_project_files.file_history_ids is not None:
@@ -669,7 +678,7 @@ def __init__(
     def path(self) -> str:
         return self.file.path
 
-    @property
+    @cached_property
     def diff(self) -> Optional[FileDiff]:
         """Diff file pushed with UPDATE_DIFF change type.
 
@@ -724,9 +733,37 @@ def changes(
         if not (is_versioned_file(file) and since is not None and to is not None):
             return []
 
-        history = []
+        # when since=1 the range spans the entire project history; narrow it to
+        # the most recent CREATE/DELETE so we don't load records from previous
+        # file lifecycles that the Python break would discard anyway
+        if since == 1:
+            boundary = (
+                FileHistory.query.join(ProjectFilePath)
+                .filter(
+                    ProjectFilePath.project_id == project_id,
+                    ProjectFilePath.path == file,
+                    FileHistory.project_version_name <= to,
+                    FileHistory.change.in_(
+                        [PushChangeType.CREATE.value, PushChangeType.DELETE.value]
+                    ),
+                )
+                .order_by(desc(FileHistory.project_version_name))
+                .with_entities(FileHistory.project_version_name)
+                .first()
+            )
+            since = boundary[0] if boundary else since
+
         full_history = (
             FileHistory.query.join(ProjectFilePath)
+            .join(FileHistory.version)
+            .join(ProjectVersion.project)
+            .options(
+                contains_eager(FileHistory.file).load_only(ProjectFilePath.path),
+                contains_eager(FileHistory.version)
+                .load_only(ProjectVersion.name, ProjectVersion.project_id)
+                .contains_eager(ProjectVersion.project)
+                .load_only(Project.storage_params),
+            )
             .filter(
                 ProjectFilePath.project_id == project_id,
                 FileHistory.project_version_name <= to,
@@ -737,6 +774,7 @@ def changes(
             .all()
         )
 
+        history = []
         for item in full_history:
             history.append(item)
 
@@ -1792,11 +1830,15 @@ def diff_summary(self):
 
     def changes_count(self) -> Dict:
         """Return number of changes by type"""
-        query = f"SELECT change, COUNT(change) FROM file_history WHERE version_id = :version_id GROUP BY change;"
+        query = "SELECT change, COUNT(change) FROM file_history WHERE version_id = :version_id GROUP BY change;"
         params = {"version_id": self.id}
         result = db.session.execute(text(query), params).fetchall()
         return {row[0]: row[1] for row in result}
 
+    @cached_property
+    def _changes_count(self) -> Dict:
+        return self.changes_count()
+
     @property
     def zip_path(self):
         return os.path.join(

server/mergin/sync/public_api.yaml

@@ -1124,11 +1124,6 @@ components:
                           - added
                           - updated
                           - removed
-                      expiration:
-                        nullable: true
-                        type: string
-                        format: date-time
-                        example: 2019-02-26T08:47:58.636074Z
     UploadFileInfo:
       allOf:
         - $ref: "#/components/schemas/FileInfo"