diff --git a/server/.test.env b/server/.test.env index bdaa7bfa..63294a3f 100644 --- a/server/.test.env +++ b/server/.test.env @@ -24,3 +24,4 @@ SECURITY_BEARER_SALT='bearer' SECURITY_EMAIL_SALT='email' SECURITY_PASSWORD_SALT='password' DIAGNOSTIC_LOGS_DIR=/tmp/diagnostic_logs +GEVENT_WORKER=0 \ No newline at end of file diff --git a/server/mergin/sync/config.py b/server/mergin/sync/config.py index 7200dae5..e616a0ca 100644 --- a/server/mergin/sync/config.py +++ b/server/mergin/sync/config.py @@ -75,3 +75,6 @@ class Configuration(object): UPLOAD_CHUNKS_EXPIRATION = config( "UPLOAD_CHUNKS_EXPIRATION", default=86400, cast=int ) + EXCLUDED_CLONE_FILENAMES = config( + "EXCLUDED_CLONE_FILENAMES", default="qgis_cfg.xml", cast=Csv() + ) diff --git a/server/mergin/sync/permissions.py b/server/mergin/sync/permissions.py index 4305a15f..7dd042d5 100644 --- a/server/mergin/sync/permissions.py +++ b/server/mergin/sync/permissions.py @@ -209,7 +209,21 @@ def require_project(ws, project_name, permission) -> Project: return project -def require_project_by_uuid(uuid: str, permission: ProjectPermissions, scheduled=False): +def require_project_by_uuid( + uuid: str, permission: ProjectPermissions, scheduled=False, expose=True +) -> Project: + """ + Retrieves a project by UUID after validating existence, workspace status, and permissions. + + Args: + uuid (str): The unique identifier of the project. + permission (ProjectPermissions): The permission level required to access the project. + scheduled (bool, optional): If ``True``, bypasses the check for projects marked for deletion. + expose (bool, optional): Controls security disclosure behavior on permission failure. + - If `True`: Returns 403 Forbidden (reveals project exists but access is denied). + - If `False`: Returns 404 Not Found (hides project existence for security). + Standard is that reading results in 404, while writing results in 403 + """ if not is_valid_uuid(uuid): abort(404) @@ -219,6 +233,10 @@ def require_project_by_uuid(uuid: str, permission: ProjectPermissions, scheduled if not scheduled: project = project.filter(Project.removed_at.is_(None)) project = project.first_or_404() + if not expose and current_user.is_anonymous and not project.public: + # we don't want to tell anonymous user if a private project exists + abort(404) + workspace = project.workspace if not workspace: abort(404) @@ -226,6 +244,7 @@ def require_project_by_uuid(uuid: str, permission: ProjectPermissions, scheduled abort(404, "Workspace doesn't exist") if not permission.check(project, current_user): abort(403, "You do not have permissions for this project") + return project diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index 0b487874..f8b88cd1 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -1136,9 +1136,12 @@ def clone_project(namespace, project_name): # noqa: E501 ) p.updated = datetime.utcnow() db.session.add(p) + files_to_exclude = current_app.config.get("EXCLUDED_CLONE_FILENAMES", []) try: - p.storage.initialize(template_project=cloned_project) + p.storage.initialize( + template_project=cloned_project, excluded_files=files_to_exclude + ) except InitializationError as e: abort(400, f"Failed to clone project: {str(e)}") @@ -1149,6 +1152,8 @@ def clone_project(namespace, project_name): # noqa: E501 # transform source files to new uploaded files file_changes = [] for file in cloned_project.files: + if os.path.basename(file.path) in files_to_exclude: + continue file_changes.append( ProjectFileChange( file.path, diff --git a/server/mergin/sync/public_api_v2.yaml b/server/mergin/sync/public_api_v2.yaml index bf3db007..c1c74f68 100644 --- a/server/mergin/sync/public_api_v2.yaml +++ b/server/mergin/sync/public_api_v2.yaml @@ -87,8 +87,7 @@ paths: description: Include list of files at specific version required: false schema: - type: string - example: v3 + $ref: "#/components/schemas/VersionName" responses: "200": description: Success @@ -305,9 +304,7 @@ paths: default: false example: true version: - type: string - pattern: '^$|^v\d+$' - example: v2 + $ref: "#/components/schemas/VersionName" changes: type: object required: @@ -849,3 +846,7 @@ components: - editor - writer - owner + VersionName: + type: string + pattern: '^$|^v\d+$' + example: v2 diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index 217204c1..1bfd8738 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -44,7 +44,6 @@ from .public_api_controller import catch_sync_failure from .schemas import ( ProjectMemberSchema, - ProjectVersionSchema, UploadChunkSchema, ProjectSchema, ) @@ -167,7 +166,7 @@ def remove_project_collaborator(id, user_id): def get_project(id, files_at_version=None): """Get project info. Include list of files at specific version if requested.""" - project = require_project_by_uuid(id, ProjectPermissions.Read) + project = require_project_by_uuid(id, ProjectPermissions.Read, expose=False) data = ProjectSchemaV2().dump(project) if files_at_version: pv = ProjectVersion.query.filter_by( diff --git a/server/mergin/sync/storages/disk.py b/server/mergin/sync/storages/disk.py index 4491ad98..7b038755 100644 --- a/server/mergin/sync/storages/disk.py +++ b/server/mergin/sync/storages/disk.py @@ -178,7 +178,7 @@ def _project_dir(self): ) return project_dir - def initialize(self, template_project=None): + def initialize(self, template_project=None, excluded_files=None): if os.path.exists(self.project_dir): raise InitializationError( "Project directory already exists: {}".format(self.project_dir) @@ -193,8 +193,12 @@ def initialize(self, template_project=None): if ws.disk_usage() + template_project.disk_usage > ws.storage: self.delete() raise InitializationError("Disk quota reached") + if excluded_files is None: + excluded_files = [] for file in template_project.files: + if os.path.basename(file.path) in excluded_files: + continue src = os.path.join(template_project.storage.project_dir, file.location) dest = os.path.join( self.project_dir, diff --git a/server/mergin/tests/test_middleware.py b/server/mergin/tests/test_middleware.py index 82b9cf26..2f5cbe4f 100644 --- a/server/mergin/tests/test_middleware.py +++ b/server/mergin/tests/test_middleware.py @@ -6,6 +6,7 @@ import psycogreen.gevent import pytest import sqlalchemy +from unittest.mock import patch from ..app import create_simple_app, GeventTimeoutMiddleware, db from ..config import Configuration @@ -14,58 +15,74 @@ @pytest.mark.parametrize("use_middleware", [True, False]) def test_use_middleware(use_middleware): """Test using middleware""" - Configuration.GEVENT_WORKER = use_middleware - Configuration.GEVENT_REQUEST_TIMEOUT = 1 - application = create_simple_app() + with patch.object( + Configuration, + "GEVENT_WORKER", + use_middleware, + ), patch.object( + Configuration, + "GEVENT_REQUEST_TIMEOUT", + 1, + ): + application = create_simple_app() - def ping(): - gevent.sleep(Configuration.GEVENT_REQUEST_TIMEOUT + 1) - return "pong" + def ping(): + gevent.sleep(Configuration.GEVENT_REQUEST_TIMEOUT + 1) + return "pong" - application.add_url_rule("/test", "ping", ping) - app_context = application.app_context() - app_context.push() + application.add_url_rule("/test", "ping", ping) + app_context = application.app_context() + app_context.push() - assert isinstance(application.wsgi_app, GeventTimeoutMiddleware) == use_middleware - # in case of gevent, dummy endpoint it set to time out - assert application.test_client().get("/test").status_code == ( - 504 if use_middleware else 200 - ) + assert ( + isinstance(application.wsgi_app, GeventTimeoutMiddleware) == use_middleware + ) + # in case of gevent, dummy endpoint it set to time out + assert application.test_client().get("/test").status_code == ( + 504 if use_middleware else 200 + ) def test_catch_timeout(): """Test proper handling of gevent timeout with db.session.rollback""" psycogreen.gevent.patch_psycopg() - Configuration.GEVENT_WORKER = True - Configuration.GEVENT_REQUEST_TIMEOUT = 1 - application = create_simple_app() + with patch.object( + Configuration, + "GEVENT_WORKER", + True, + ), patch.object( + Configuration, + "GEVENT_REQUEST_TIMEOUT", + 1, + ): + application = create_simple_app() - def unhandled(): - try: - db.session.execute("SELECT pg_sleep(1.1);") - finally: - db.session.execute("SELECT 1;") - return "" + def unhandled(): + try: + db.session.execute("SELECT pg_sleep(1.1);") + finally: + db.session.execute("SELECT 1;") + return "" - def timeout(): - try: - db.session.execute("SELECT pg_sleep(1.1);") - except gevent.timeout.Timeout: - db.session.rollback() - raise - finally: - db.session.execute("SELECT 1;") - return "" + def timeout(): + try: + db.session.execute("SELECT pg_sleep(1.1);") + except gevent.timeout.Timeout: + db.session.rollback() + raise + finally: + db.session.execute("SELECT 1;") + return "" - application.add_url_rule("/unhandled", "unhandled", unhandled) - application.add_url_rule("/timeout", "timeout", timeout) - app_context = application.app_context() - app_context.push() + application.add_url_rule("/unhandled", "unhandled", unhandled) + application.add_url_rule("/timeout", "timeout", timeout) + app_context = application.app_context() + app_context.push() - assert application.test_client().get("/timeout").status_code == 504 + assert application.test_client().get("/timeout").status_code == 504 - # in case of missing rollback sqlalchemy would raise error - with pytest.raises(sqlalchemy.exc.PendingRollbackError): - application.test_client().get("/unhandled") + # in case of missing rollback sqlalchemy would raise error + with pytest.raises(sqlalchemy.exc.PendingRollbackError): + application.test_client().get("/unhandled") - db.session.rollback() + db.session.rollback() diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index c7a0550e..054ba063 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -1728,6 +1728,8 @@ def test_clone_project(client, data, username, expected): assert resp.json["code"] == "StorageLimitHit" assert resp.json["detail"] == "You have reached a data limit (StorageLimitHit)" if expected == 200: + excluded_filenames = current_app.config.get("EXCLUDED_CLONE_FILENAMES") + proj = data.get("project", test_project).strip() template = Project.query.filter_by( name=test_project, workspace_id=test_workspace_id @@ -1735,9 +1737,12 @@ def test_clone_project(client, data, username, expected): project = Project.query.filter_by( name=proj, workspace_id=test_workspace_id ).first() + template_files_filtered = [ + f for f in template.files if f.path not in excluded_filenames + ] assert not any( x.checksum != y.checksum and x.path != y.path - for x, y in zip(project.files, template.files) + for x, y in zip(project.files, template_files_filtered) ) assert os.path.exists( os.path.join(project.storage.project_dir, project.files[0].location) @@ -1755,6 +1760,12 @@ def test_clone_project(client, data, username, expected): item for item in changes if item.change == PushChangeType.UPDATE.value ] assert pv.device_id == json_headers["X-Device-Id"] + + assert not any(f.path == excluded_filenames[0] for f in project.files) + assert not os.path.exists( + os.path.join(project.storage.project_dir, excluded_filenames[0]) + ) + assert len(project.files) == len(template.files) - 1 # cleanup shutil.rmtree(project.storage.project_dir) @@ -2000,7 +2011,7 @@ def test_get_projects_by_uuids(client): {"page": 1, "per_page": 5, "desc": False}, 200, "v1", - {"added": 12, "removed": 0, "updated": 0, "updated_diff": 0}, + {"added": 13, "removed": 0, "updated": 0, "updated_diff": 0}, ), ( {"page": 2, "per_page": 3, "desc": True}, diff --git a/server/mergin/tests/test_projects/test/qgis_cfg.xml b/server/mergin/tests/test_projects/test/qgis_cfg.xml new file mode 100644 index 00000000..c2eece89 --- /dev/null +++ b/server/mergin/tests/test_projects/test/qgis_cfg.xml @@ -0,0 +1,4 @@ + +0737fe398eb9f26bd847fb9da2407646a2e8c89dc2f93eba5a059f19eedd8017e50c557d2c7d435a2701d881cdaf4fbbd3a892e4367053b5bfc348b556ae252314d9b06fc70a4f184362d064023c1ed6c4dd7ee14dce10ea91595e8548f7ba3d3eaca1d41063f50d1ccc12bfb90c059271254dca780e0d60e68bd234844fda81a0781977907485b397aa1263aef81863625eb439dc349fca0dbc641b4a606657f17e55d2c02fbc95388bf9f96977c65fcf7b723689d5fcdaf73190a5597425b3d33c858c2c4ef8c334b5f601c98db05557c8f690cdb9f73c725bf7ee420fffb6037cc9e80c7374a55ab55baf4aaa7f1c957fd40bb69b9fcb41ec42b063330bbddcd73f4de69e47772309167cb20ef4fe3250db96c29b71772edd18c7e73c501d569f4f8deda15fb0bcf6701d81902a6fc6c722db9e0d766d18a45297232224738c07a1a4f8fe490954efcae05fc1e43eac4eb5efc8b9008dcff4cf3688db4b7e268c9adf75d88d4d1d3648232c6fc2ac98b49b3bbb19368ce460b4a7a9828558d473eb0f1ba34c09f1ba9ce0170ac6d6c656176760e4012c56daef3f5f05320d5f84260d2e6b5a0b15620c33802d1c8c2f28084eef63b32f8130edd4789972b25960e12eb79351d11316f78aea3a941b9e7f1f4042f708d873ac7807ce0652819b2e2f77f9aac1c50cf72d2341118c41419f5f4c31474d8dbe56558dab9cb4b8e4fe8df9c8b4a057d9c6fe6b098b78e150aadd2a45cf1ea15e02f8f1f8b1b46d1a5513c26a63ea08788675feaa912e884ceee57adc120393c8a5bc42988f7b210195f6eff5de3e332d0d67321d05b907f836eb0f0f9e97388f89b699638804978639ad8b4c889f1a56952f949242679506cba5cc35538ea01b1621dd6a154f92b721b5247e294a5394df9c87765675b737dcb28346fc4032b68f87f46150aa4aa136378903036aff61fd41cf0cbcdd0865660f26d7f1d49f29ff5962adc209b9db71d12bf49bf67950496f18ca1de0a5cd7186e1bc0fcf826ffd1bb91ab36412c43730db5ff9ec57990fee27c5158446294bf0d8e61e12ee53e80b606b541c754ed45b2289079df8b647a8ca12fb1706e371523a581af50d333adfe5e84bcce2a60e84e24bdc1eb74610bc28b279b15c4f2020b045d2e4a7f846e488d74e761d98c05f105452235f602b3fe8beccf4b11d35ba6042dcc97f68090f40edbd6e8497434c193343cca98ebeabdd8620ec7eec642efda7cd45f0a9547ea821ac193eb1a8fb8c9c71d2e607b4651de5b8b613bc38aa4ba06bdb65a3d6b6e92546f1a4113e0bbce99aadbab3bbb07f31d6f90b3ff58b4494815e97a265c1c5e8a826bf14177427e03247395a18941753c0e580c42661a9c959ad57b93b97fb4adeca49927f3bec95eff361e95c324623f1c7c4d39e71250938d4189461cd6c1e978a5445f88eaf47670f23145cb7c8faf42ac83158743004fefb17a37a25edcf2425d530dd12ca52fdcfc399542cd288773c06931ce9aaac94df69dc6514fa3b1b8629dcfe725c0dcd77b5db967c5620dbc2444f4b78fb247e33a54ed2cbcaba3b92833b6d75b4900697da646f04da9a04d6353556b0ab70f8dd952eed9bd9cee1d53e760b292080862a74f625eb402662aadd94efaa6cce0727d3ccab5b6e112f25562effadfcf70307800e0d28976327576e99380facea2828ddb6a85addb4d4c0cfdb73cd848a9f707f8f978caf5de82756c80f42d53719987d5b4826397de8674d75dc1308dd3e96af37e9b3e42175dca1a5ff58a4aa4881a344113711a93340ee6515e5b9d03d1f4979531c84ec187b9303ea763b2641f530144cf52a81812349511219fc92bb038ec62d438c3beaf723 + + \ No newline at end of file diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index 6a4243fd..34a5a2a1 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -11,6 +11,8 @@ create_workspace, create_project, upload_file_to_project, + login, + file_info, ) from ..auth.models import User @@ -49,7 +51,6 @@ _get_changes_with_diff_0_size, _get_changes_without_added, ) -from .utils import add_user, file_info def test_schedule_delete_project(client): @@ -168,6 +169,7 @@ def test_project_members(client): # access provided by workspace role cannot be removed directly response = client.delete(url + f"/{user.id}") assert response.status_code == 404 + Configuration.GLOBAL_READ = 0 def test_get_project(client): @@ -176,7 +178,12 @@ def test_get_project(client): test_workspace = create_workspace() project = create_project("new_project", test_workspace, admin) logout(client) + # anonymous user cannot access the private resource + response = client.get(f"v2/projects/{project.id}") + assert response.status_code == 404 # lack of permissions + user = add_user("tests", "tests") + login(client, user.username, "tests") response = client.get(f"v2/projects/{project.id}") assert response.status_code == 403 # access public project @@ -235,6 +242,9 @@ def test_get_project(client): ) assert len(response.json["files"]) == 3 assert {f["path"] for f in response.json["files"]} == set(files) + # invalid version format parameter + response = client.get(f"v2/projects/{project.id}?files_at_version=3") + assert response.status_code == 400 push_data = [ diff --git a/server/mergin/version.py b/server/mergin/version.py index cdacf710..10d74e1e 100644 --- a/server/mergin/version.py +++ b/server/mergin/version.py @@ -4,4 +4,4 @@ def get_version(): - return "2025.7.3" + return "2025.8.2" diff --git a/server/setup.py b/server/setup.py index 33f41ea7..7a3d1939 100644 --- a/server/setup.py +++ b/server/setup.py @@ -6,7 +6,7 @@ setup( name="mergin", - version="2025.7.3", + version="2025.8.2", url="https://github.com/MerginMaps/mergin", license="AGPL-3.0-only", author="Lutra Consulting Limited",