From a4a40a85ea2c4bb4e7a16e2fb05949ce85d68f06 Mon Sep 17 00:00:00 2001 From: Elliot Simpson Date: Tue, 13 Oct 2026 07:50:32 +1300 Subject: [PATCH 01/22] feat: DM-4390 add create_project --- HISTORY.rst | 6 +++ datamasque/client/__init__.py | 4 ++ datamasque/client/dmclient.py | 2 + datamasque/client/models/project.py | 39 +++++++++++++++++ datamasque/client/projects.py | 25 +++++++++++ docs/client.models.rst | 8 ++++ docs/client.rst | 8 ++++ tests/test_projects.py | 65 +++++++++++++++++++++++++++++ 8 files changed, 157 insertions(+) create mode 100644 datamasque/client/models/project.py create mode 100644 datamasque/client/projects.py create mode 100644 tests/test_projects.py diff --git a/HISTORY.rst b/HISTORY.rst index 191e613..a4fb912 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,12 @@ History ======= +1.7.0 (unreleased) +------------------ + +* Added ``create_project``, which takes a ``ProjectCreateRequest`` and returns the created ``Project``. + Only admin users can create projects, and API tokens cannot. + 1.6.0 (unreleased) ------------------ diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index 2122c6d..3146bae 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -115,6 +115,7 @@ RulesetPlanUpdateRequest, ) from datamasque.client.models.license import LicenseInfo, SwitchableLicenseMetadata +from datamasque.client.models.project import Project, ProjectCreateRequest, ProjectId from datamasque.client.models.rg_config import RGConfig, RGConfigId from datamasque.client.models.ruleset import Ruleset, RulesetId, RulesetType from datamasque.client.models.ruleset_library import RulesetLibrary, RulesetLibraryId @@ -269,6 +270,9 @@ "PatternComposition", "PatternEntry", "PatternsStatistics", + "Project", + "ProjectCreateRequest", + "ProjectId", "RGConfig", "RGConfigId", "RGConfigNotFoundError", diff --git a/datamasque/client/dmclient.py b/datamasque/client/dmclient.py index 7e8cc7f..486e3e7 100644 --- a/datamasque/client/dmclient.py +++ b/datamasque/client/dmclient.py @@ -5,6 +5,7 @@ from datamasque.client.discovery_configs import DiscoveryConfigClient from datamasque.client.files import FileClient from datamasque.client.license import LicenseClient +from datamasque.client.projects import ProjectClient from datamasque.client.rg_configs import RGConfigClient from datamasque.client.ruleset_libraries import RulesetLibraryClient from datamasque.client.rulesets import RulesetClient @@ -18,6 +19,7 @@ class DataMasqueClient( LicenseClient, + ProjectClient, ConnectionClient, RulesetClient, RulesetLibraryClient, diff --git a/datamasque/client/models/project.py b/datamasque/client/models/project.py new file mode 100644 index 0000000..40b9bcf --- /dev/null +++ b/datamasque/client/models/project.py @@ -0,0 +1,39 @@ +"""Project models for the DataMasque API.""" + +from datetime import datetime +from typing import NewType + +from pydantic import BaseModel, ConfigDict + +ProjectId = NewType("ProjectId", int) + + +class Project(BaseModel): + """A project: the top-level grouping that connections, rulesets and other resources belong to.""" + + model_config = ConfigDict(extra="allow") + + id: ProjectId + reference: str + """The `-` string other resources use to refer to this project.""" + name: str + description: str = "" + created: datetime + updated: datetime + + +class ProjectCreateRequest(BaseModel): + """ + Request body for `POST /api/projects/`. + + Names are unique across active and archived projects, ignoring case. + The server trims leading and trailing whitespace from `name` and `description`, + and rejects a name that contains "/", ".." or a control character. + It may reject other names too, + and its 400 response gives the reason. + """ + + model_config = ConfigDict(extra="forbid") + + name: str + description: str = "" diff --git a/datamasque/client/projects.py b/datamasque/client/projects.py new file mode 100644 index 0000000..1175e75 --- /dev/null +++ b/datamasque/client/projects.py @@ -0,0 +1,25 @@ +import logging + +from datamasque.client.base import BaseClient +from datamasque.client.models.project import Project, ProjectCreateRequest + +logger = logging.getLogger(__name__) + + +class ProjectClient(BaseClient): + """Project API methods. Mixed into `DataMasqueClient`.""" + + def create_project(self, request: ProjectCreateRequest) -> Project: + """ + Creates a new project on the server, and returns it as the server stored it. + + Only admin users can create projects, and API tokens cannot. + A name the server rejects (see `ProjectCreateRequest`) raises `DataMasqueApiError` with status 400, + and a caller who is not an admin user raises it with status 403. + """ + + data = request.model_dump(mode="json") + response = self.make_request("POST", "/api/projects/", data=data) + project = Project.model_validate(response.json()) + logger.info('Creation of project "%s" successful', project.name) + return project diff --git a/docs/client.models.rst b/docs/client.models.rst index e98789b..0b2f872 100644 --- a/docs/client.models.rst +++ b/docs/client.models.rst @@ -60,6 +60,14 @@ datamasque.client.models.license module :undoc-members: :show-inheritance: +datamasque.client.models.project module +--------------------------------------- + +.. automodule:: datamasque.client.models.project + :members: + :undoc-members: + :show-inheritance: + datamasque.client.models.rg_config module ----------------------------------------- diff --git a/docs/client.rst b/docs/client.rst index f013c64..b0f5a69 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -84,6 +84,14 @@ datamasque.client.license module :undoc-members: :show-inheritance: +datamasque.client.projects module +--------------------------------- + +.. automodule:: datamasque.client.projects + :members: + :undoc-members: + :show-inheritance: + datamasque.client.rg_configs module ----------------------------------- diff --git a/tests/test_projects.py b/tests/test_projects.py new file mode 100644 index 0000000..998121f --- /dev/null +++ b/tests/test_projects.py @@ -0,0 +1,65 @@ +"""Tests for project support in the DataMasque client.""" + +from datetime import datetime + +import pytest +import requests_mock +from pydantic import ValidationError + +from datamasque.client import DataMasqueApiError, DataMasqueClient, ProjectCreateRequest, ProjectId + +CREATE_RESPONSE = { + "id": 7, + "reference": "7-banking", + "name": "Banking", + "description": "Customer banking data", + "created": "2026-10-01T10:00:00Z", + "updated": "2026-10-01T10:00:00Z", +} + + +def test_create_request_rejects_unknown_fields() -> None: + with pytest.raises(ValidationError, match="archived"): + ProjectCreateRequest(name="Banking", archived=True) # type: ignore[call-arg] + + +def test_create_project(client: DataMasqueClient) -> None: + request = ProjectCreateRequest(name="Banking", description="Customer banking data") + + with requests_mock.Mocker() as m: + m.post("http://test-server/api/projects/", json=CREATE_RESPONSE, status_code=201) + project = client.create_project(request) + + assert m.last_request.json() == {"name": "Banking", "description": "Customer banking data"} + assert project.id == ProjectId(7) + assert project.reference == "7-banking" + assert project.name == "Banking" + assert project.description == "Customer banking data" + assert project.created == datetime.fromisoformat("2026-10-01T10:00:00+00:00") + assert project.updated == datetime.fromisoformat("2026-10-01T10:00:00+00:00") + + +def test_create_project_returns_the_name_and_description_as_stored(client: DataMasqueClient) -> None: + """The server trims surrounding whitespace from the name and description.""" + request = ProjectCreateRequest(name=" Banking ", description=" Customer banking data ") + + with requests_mock.Mocker() as m: + m.post("http://test-server/api/projects/", json=CREATE_RESPONSE, status_code=201) + project = client.create_project(request) + + assert m.last_request.json() == {"name": " Banking ", "description": " Customer banking data "} + assert project.name == "Banking" + assert project.description == "Customer banking data" + + +def test_create_project_rejected_name_raises(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.post( + "http://test-server/api/projects/", + json={"name": ['Project name must not contain "/".']}, + status_code=400, + ) + with pytest.raises(DataMasqueApiError) as exc_info: + client.create_project(ProjectCreateRequest(name="bank/au")) + + assert exc_info.value.response.status_code == 400 From 9681cbe00217f9a53c00b280b8b607907b4ac9cd Mon Sep 17 00:00:00 2001 From: Elliot Simpson Date: Fri, 16 Oct 2026 12:21:31 +1300 Subject: [PATCH 02/22] test: DM-4390 describe what the trim test checks --- tests/test_projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_projects.py b/tests/test_projects.py index 998121f..383f5a1 100644 --- a/tests/test_projects.py +++ b/tests/test_projects.py @@ -40,7 +40,7 @@ def test_create_project(client: DataMasqueClient) -> None: def test_create_project_returns_the_name_and_description_as_stored(client: DataMasqueClient) -> None: - """The server trims surrounding whitespace from the name and description.""" + """Whitespace in the name and description is sent as given, and the server's stored values are returned.""" request = ProjectCreateRequest(name=" Banking ", description=" Customer banking data ") with requests_mock.Mocker() as m: From b8f8296441dbc1dbd8640a2cd849b02c223bc23f Mon Sep 17 00:00:00 2001 From: Colin Haywood Date: Fri, 16 Oct 2026 12:24:56 +1300 Subject: [PATCH 03/22] feat: add delete_project and get_project_delete_conflicts --- HISTORY.rst | 3 +- datamasque/client/__init__.py | 12 ++- datamasque/client/exceptions.py | 17 ++++ datamasque/client/models/project.py | 39 +++++++- datamasque/client/projects.py | 69 ++++++++++++++- tests/test_projects.py | 133 +++++++++++++++++++++++++++- 6 files changed, 267 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a4fb912..9d28c46 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,8 +5,7 @@ History 1.7.0 (unreleased) ------------------ -* Added ``create_project``, which takes a ``ProjectCreateRequest`` and returns the created ``Project``. - Only admin users can create projects, and API tokens cannot. +* Added project models and management APIs. 1.6.0 (unreleased) ------------------ diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index 3146bae..27e6fac 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -23,6 +23,7 @@ InvalidDiscoveryConfigError, InvalidLibraryError, InvalidRulesetError, + ProjectDeleteConflictError, RGConfigNotFoundError, RunNotCancellableError, ) @@ -115,7 +116,13 @@ RulesetPlanUpdateRequest, ) from datamasque.client.models.license import LicenseInfo, SwitchableLicenseMetadata -from datamasque.client.models.project import Project, ProjectCreateRequest, ProjectId +from datamasque.client.models.project import ( + Project, + ProjectCreateRequest, + ProjectDeleteConflict, + ProjectDeleteConflictReason, + ProjectId, +) from datamasque.client.models.rg_config import RGConfig, RGConfigId from datamasque.client.models.ruleset import Ruleset, RulesetId, RulesetType from datamasque.client.models.ruleset_library import RulesetLibrary, RulesetLibraryId @@ -272,6 +279,9 @@ "PatternsStatistics", "Project", "ProjectCreateRequest", + "ProjectDeleteConflict", + "ProjectDeleteConflictError", + "ProjectDeleteConflictReason", "ProjectId", "RGConfig", "RGConfigId", diff --git a/datamasque/client/exceptions.py b/datamasque/client/exceptions.py index 9943b8b..747fd64 100644 --- a/datamasque/client/exceptions.py +++ b/datamasque/client/exceptions.py @@ -1,5 +1,7 @@ from requests import Response +from datamasque.client.models.project import ProjectDeleteConflict + class DataMasqueException(Exception): """Generic exception base class.""" @@ -66,6 +68,21 @@ class RGConfigNotFoundError(DataMasqueApiError): """ +class ProjectDeleteConflictError(DataMasqueApiError): + """ + Raised when `delete_project` cannot move the project's resources to the destination project. + + The server deletes and moves nothing in this case. + + Attributes: + conflicts: Every reason the resources cannot move. + """ + + def __init__(self, message: str, *, response: Response, conflicts: list[ProjectDeleteConflict]) -> None: + super().__init__(message, response=response) + self.conflicts = conflicts + + class DataMasqueTransportError(DataMasqueException): """ Raised when a request to the DataMasque server fails before any response is received. diff --git a/datamasque/client/models/project.py b/datamasque/client/models/project.py index 40b9bcf..6eedc1b 100644 --- a/datamasque/client/models/project.py +++ b/datamasque/client/models/project.py @@ -1,7 +1,8 @@ """Project models for the DataMasque API.""" from datetime import datetime -from typing import NewType +from enum import Enum +from typing import NewType, Optional, Union from pydantic import BaseModel, ConfigDict @@ -22,6 +23,11 @@ class Project(BaseModel): updated: datetime +def unwrap_project_id(project: Union[ProjectId, Project]) -> ProjectId: + """Return the id of `project`, which a caller may give as a `Project` or as its `ProjectId`.""" + return project.id if isinstance(project, Project) else project + + class ProjectCreateRequest(BaseModel): """ Request body for `POST /api/projects/`. @@ -37,3 +43,34 @@ class ProjectCreateRequest(BaseModel): name: str description: str = "" + + +class ProjectDeleteConflictReason(str, Enum): + """Why a project's resources cannot move to the destination project when the project is deleted.""" + + name_in_use = "name_in_use" + """The destination project already contains a resource of the same type and name.""" + file_exists = "file_exists" + """The destination project already has a file or directory where the resource's files would move to.""" + unexpected_file = "unexpected_file" + """The project's directory contains a file or directory that belongs to none of its resources.""" + + +class ProjectDeleteConflict(BaseModel): + """A conflict that prevents a project's resources moving to the destination project when the project is deleted.""" + + model_config = ConfigDict(extra="allow") + + reason: ProjectDeleteConflictReason + detail: str + """A message that describes the conflict.""" + type: Optional[str] = None + """ + For `name_in_use` and `file_exists`: the type of the resource that would move, such as `connections` or `rulesets`. + """ + id: Optional[str] = None + """For `name_in_use` and `file_exists`: the ID of the resource that would move.""" + name: Optional[str] = None + """For `name_in_use` and `file_exists`: the name of the resource that would move.""" + path: Optional[str] = None + """For `file_exists` and `unexpected_file`: the path of the file or directory.""" diff --git a/datamasque/client/projects.py b/datamasque/client/projects.py index 1175e75..1646b13 100644 --- a/datamasque/client/projects.py +++ b/datamasque/client/projects.py @@ -1,11 +1,25 @@ import logging +from typing import Union + +from requests import Response from datamasque.client.base import BaseClient -from datamasque.client.models.project import Project, ProjectCreateRequest +from datamasque.client.exceptions import ProjectDeleteConflictError +from datamasque.client.models.project import ( + Project, + ProjectCreateRequest, + ProjectDeleteConflict, + ProjectId, + unwrap_project_id, +) logger = logging.getLogger(__name__) +def _get_conflicts(response: Response) -> list[ProjectDeleteConflict]: + return [ProjectDeleteConflict.model_validate(conflict) for conflict in response.json()["conflicts"]] + + class ProjectClient(BaseClient): """Project API methods. Mixed into `DataMasqueClient`.""" @@ -23,3 +37,56 @@ def create_project(self, request: ProjectCreateRequest) -> Project: project = Project.model_validate(response.json()) logger.info('Creation of project "%s" successful', project.name) return project + + def get_project_delete_conflicts( + self, + project: Union[ProjectId, Project], + destination: Union[ProjectId, Project], + ) -> list[ProjectDeleteConflict]: + """ + Returns the conflicts that would prevent `delete_project` moving the project's resources to the destination. + + An empty list means `delete_project` can delete the project with this destination, + unless the resources in either project change first. + + Raises: + DataMasqueApiError: the destination does not exist, is deleted, or is the project; + the project does not exist or is already deleted; or any other non-2xx response. + """ + + response = self.make_request( + "GET", + f"/api/projects/{unwrap_project_id(project)}/delete-check/", + params={"destination": unwrap_project_id(destination)}, + ) + return _get_conflicts(response) + + def delete_project(self, project: Union[ProjectId, Project], destination: Union[ProjectId, Project]) -> None: + """ + Deletes a project, after moving every resource in it to the destination project. + + When the resources cannot move, the server changes nothing. + `get_project_delete_conflicts` lists the same conflicts without trying to delete the project. + + Raises: + ProjectDeleteConflictError: the resources cannot move to the destination; its `conflicts` says why. + DataMasqueApiError: the destination does not exist, is deleted, or is the project; + the project does not exist or is already deleted; or any other non-2xx response. + """ + + project_id = unwrap_project_id(project) + response = self.make_request( + "DELETE", + f"/api/projects/{project_id}/", + params={"destination": unwrap_project_id(destination)}, + require_status_check=False, + ) + if response.status_code == 409: + raise ProjectDeleteConflictError( + str(response.json()["detail"]), + response=response, + conflicts=_get_conflicts(response), + ) + + self._raise_for_status(response) + logger.info("Deletion of project %s successful", project_id) diff --git a/tests/test_projects.py b/tests/test_projects.py index 383f5a1..17131e8 100644 --- a/tests/test_projects.py +++ b/tests/test_projects.py @@ -6,7 +6,16 @@ import requests_mock from pydantic import ValidationError -from datamasque.client import DataMasqueApiError, DataMasqueClient, ProjectCreateRequest, ProjectId +from datamasque.client import ( + DataMasqueApiError, + DataMasqueClient, + Project, + ProjectCreateRequest, + ProjectDeleteConflict, + ProjectDeleteConflictError, + ProjectDeleteConflictReason, + ProjectId, +) CREATE_RESPONSE = { "id": 7, @@ -17,6 +26,29 @@ "updated": "2026-10-01T10:00:00Z", } +CONFLICTS = [ + { + "reason": "name_in_use", + "detail": 'The destination project already contains a ruleset named "3407".', + "type": "rulesets", + "id": "0b6c2f0e-4a0d-4d8e-9a57-2f0f3c1d9e11", + "name": "3407", + }, + { + "reason": "file_exists", + "detail": 'The destination project already has a file at "projects/8-retail/seeds/names.csv".', + "type": "seeds", + "id": "5e3b8a40-7c2d-4f61-b1a9-0d4e6f2c8b77", + "name": "names.csv", + "path": "projects/8-retail/seeds/names.csv", + }, + { + "reason": "unexpected_file", + "detail": 'The project directory contains "projects/7-banking/notes.txt", which belongs to no resource.', + "path": "projects/7-banking/notes.txt", + }, +] + def test_create_request_rejects_unknown_fields() -> None: with pytest.raises(ValidationError, match="archived"): @@ -63,3 +95,102 @@ def test_create_project_rejected_name_raises(client: DataMasqueClient) -> None: client.create_project(ProjectCreateRequest(name="bank/au")) assert exc_info.value.response.status_code == 400 + + +def test_get_project_delete_conflicts_parses_each_reason(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/7/delete-check/", json={"conflicts": CONFLICTS}) + conflicts = client.get_project_delete_conflicts(ProjectId(7), ProjectId(8)) + + assert m.last_request.qs == {"destination": ["8"]} + assert conflicts == [ + ProjectDeleteConflict( + reason=ProjectDeleteConflictReason.name_in_use, + detail='The destination project already contains a ruleset named "3407".', + type="rulesets", + id="0b6c2f0e-4a0d-4d8e-9a57-2f0f3c1d9e11", + name="3407", + ), + ProjectDeleteConflict( + reason=ProjectDeleteConflictReason.file_exists, + detail='The destination project already has a file at "projects/8-retail/seeds/names.csv".', + type="seeds", + id="5e3b8a40-7c2d-4f61-b1a9-0d4e6f2c8b77", + name="names.csv", + path="projects/8-retail/seeds/names.csv", + ), + ProjectDeleteConflict( + reason=ProjectDeleteConflictReason.unexpected_file, + detail='The project directory contains "projects/7-banking/notes.txt", which belongs to no resource.', + path="projects/7-banking/notes.txt", + ), + ] + assert conflicts[0].path is None + assert conflicts[2].type is None + assert conflicts[2].id is None + assert conflicts[2].name is None + + +def test_get_project_delete_conflicts_returns_empty_list_when_project_can_be_deleted( + client: DataMasqueClient, +) -> None: + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/7/delete-check/", json={"conflicts": []}) + conflicts = client.get_project_delete_conflicts(ProjectId(7), ProjectId(8)) + + assert conflicts == [] + + +def test_project_delete_calls_accept_projects_as_well_as_ids(client: DataMasqueClient) -> None: + project = Project.model_validate(CREATE_RESPONSE) + destination = Project.model_validate(CREATE_RESPONSE | {"id": 8, "reference": "8-retail", "name": "Retail"}) + + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/7/delete-check/", json={"conflicts": []}) + m.delete("http://test-server/api/projects/7/", status_code=204) + client.get_project_delete_conflicts(project, destination) + client.delete_project(project, destination) + + assert [request.qs for request in m.request_history] == [{"destination": ["8"]}, {"destination": ["8"]}] + + +def test_delete_project(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.delete("http://test-server/api/projects/7/", status_code=204) + client.delete_project(ProjectId(7), ProjectId(8)) + + assert m.last_request.method == "DELETE" + assert m.last_request.qs == {"destination": ["8"]} + + +def test_delete_project_conflict_raises_with_conflicts(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.delete( + "http://test-server/api/projects/7/", + json={"detail": 'Project "Banking" cannot be deleted.', "conflicts": CONFLICTS}, + status_code=409, + ) + with pytest.raises(ProjectDeleteConflictError) as exc_info: + client.delete_project(ProjectId(7), ProjectId(8)) + + assert exc_info.value.response.status_code == 409 + assert str(exc_info.value) == 'Project "Banking" cannot be deleted.' + assert [conflict.reason for conflict in exc_info.value.conflicts] == [ + ProjectDeleteConflictReason.name_in_use, + ProjectDeleteConflictReason.file_exists, + ProjectDeleteConflictReason.unexpected_file, + ] + assert exc_info.value.conflicts[0].name == "3407" + + +@pytest.mark.parametrize("status_code", [400, 404]) +def test_delete_project_raises_a_plain_api_error_for_a_status_other_than_409( + client: DataMasqueClient, status_code: int +) -> None: + with requests_mock.Mocker() as m: + m.delete("http://test-server/api/projects/7/", json={"detail": "Not found."}, status_code=status_code) + with pytest.raises(DataMasqueApiError) as exc_info: + client.delete_project(ProjectId(7), ProjectId(8)) + + assert not isinstance(exc_info.value, ProjectDeleteConflictError) + assert exc_info.value.response.status_code == status_code From 6d286c45bb52c45ebe1eaa0b81576d88bee4498d Mon Sep 17 00:00:00 2001 From: Colin Haywood Date: Fri, 16 Oct 2026 12:54:47 +1300 Subject: [PATCH 04/22] feat: add list_resources --- HISTORY.rst | 1 + datamasque/client/__init__.py | 12 +++ datamasque/client/base.py | 10 +- datamasque/client/dmclient.py | 2 + datamasque/client/models/resource.py | 71 +++++++++++++ datamasque/client/resources.py | 56 ++++++++++ docs/client.models.rst | 8 ++ docs/client.rst | 8 ++ tests/test_resources.py | 152 +++++++++++++++++++++++++++ 9 files changed, 318 insertions(+), 2 deletions(-) create mode 100644 datamasque/client/models/resource.py create mode 100644 datamasque/client/resources.py create mode 100644 tests/test_resources.py diff --git a/HISTORY.rst b/HISTORY.rst index 9d28c46..7eb1002 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,7 @@ History ------------------ * Added project models and management APIs. +* Added ``list_resources``, which lists resources of every type across projects. 1.6.0 (unreleased) ------------------ diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index 27e6fac..e4ba6c3 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -123,6 +123,13 @@ ProjectDeleteConflictReason, ProjectId, ) +from datamasque.client.models.resource import ( + ConnectionResource, + DiscoveryConfigResource, + Resource, + ResourceTypeName, + RulesetResource, +) from datamasque.client.models.rg_config import RGConfig, RGConfigId from datamasque.client.models.ruleset import Ruleset, RulesetId, RulesetType from datamasque.client.models.ruleset_library import RulesetLibrary, RulesetLibraryId @@ -196,6 +203,7 @@ "CommonStatistics", "ConnectionConfig", "ConnectionId", + "ConnectionResource", "ConstraintColumns", "DataMasqueApiError", "DataMasqueClient", @@ -216,6 +224,7 @@ "DiscoveryConfigLibrary", "DiscoveryConfigLibraryId", "DiscoveryConfigNotFoundError", + "DiscoveryConfigResource", "DiscoveryConfigType", "DiscoveryMatch", "CosmosDbConnectionConfig", @@ -287,6 +296,8 @@ "RGConfigId", "RGConfigNotFoundError", "ReferencingForeignKey", + "Resource", + "ResourceTypeName", "Ruleset", "RulesetGenerationRequest", "RulesetGenerationWithRGConfigRequest", @@ -298,6 +309,7 @@ "RulesetPlanOptions", "RulesetPlanPartialUpdateRequest", "RulesetPlanUpdateRequest", + "RulesetResource", "RulesetType", "RunConnectionRef", "RunId", diff --git a/datamasque/client/base.py b/datamasque/client/base.py index 673356d..f1ce266 100644 --- a/datamasque/client/base.py +++ b/datamasque/client/base.py @@ -321,8 +321,14 @@ def _iter_paginated( params: Optional[dict] = None, page_size: int = 100, ) -> Iterator[_T]: + """Iterate every `T` across all pages of an admin-server list endpoint.""" + + for results in self._iter_pages(path, params=params, page_size=page_size): + yield from (model.model_validate(item) for item in results) + + def _iter_pages(self, path: str, *, params: Optional[dict] = None, page_size: int = 100) -> Iterator[Any]: """ - Iterate every `T` across all pages of an admin-server list endpoint. + Iterate the `results` of each page of an admin-server list endpoint. Opts into pagination by sending `limit`/`offset` on the first request, then follows the absolute `next` URL returned by the server. @@ -338,7 +344,7 @@ def _iter_paginated( while url: response = self.make_request("GET", url, params=current_params) data = response.json() - yield from (model.model_validate(item) for item in data["results"]) + yield data["results"] url = data.get("next") # The `next` URL is absolute and already contains the pagination cursor; # do not re-send our initial params alongside it. diff --git a/datamasque/client/dmclient.py b/datamasque/client/dmclient.py index 486e3e7..0d98dca 100644 --- a/datamasque/client/dmclient.py +++ b/datamasque/client/dmclient.py @@ -6,6 +6,7 @@ from datamasque.client.files import FileClient from datamasque.client.license import LicenseClient from datamasque.client.projects import ProjectClient +from datamasque.client.resources import ResourceClient from datamasque.client.rg_configs import RGConfigClient from datamasque.client.ruleset_libraries import RulesetLibraryClient from datamasque.client.rulesets import RulesetClient @@ -20,6 +21,7 @@ class DataMasqueClient( LicenseClient, ProjectClient, + ResourceClient, ConnectionClient, RulesetClient, RulesetLibraryClient, diff --git a/datamasque/client/models/resource.py b/datamasque/client/models/resource.py new file mode 100644 index 0000000..a4abec2 --- /dev/null +++ b/datamasque/client/models/resource.py @@ -0,0 +1,71 @@ +"""Resource listing models for the DataMasque API.""" + +from datetime import datetime +from enum import Enum +from typing import Optional + +from pydantic import BaseModel, ConfigDict + +from datamasque.client.models.discovery_config import DiscoveryConfigType +from datamasque.client.models.runs import MaskType + + +class ResourceTypeName(str, Enum): + """A type of resource that belongs to a project, named as the resources listing groups it.""" + + connections = "connections" + connection_filesets = "connection_filesets" + oracle_wallets = "oracle_wallets" + snowflake_keys = "snowflake_keys" + table_references = "table_references" + rulesets = "rulesets" + ruleset_libraries = "ruleset_libraries" + discovery_configs = "discovery_configs" + discovery_config_libraries = "discovery_config_libraries" + ruleset_generation_configs = "ruleset_generation_configs" + seeds = "seeds" + + +class Resource(BaseModel): + """ + One resource in the resources listing: the summary the listing gives, not the resource's full model. + + Connections, rulesets and discovery configs are listed as the subclasses that add the field each carries. + """ + + model_config = ConfigDict(extra="allow") + + id: str + name: str + project: Optional[str] = None + """Its project's `-` reference, or `None` for a resource in no project, such as a built-in seed.""" + created: Optional[datetime] = None + """`None` for a type that does not record it, currently connections.""" + modified: Optional[datetime] = None + """`None` for a type that does not record it, currently connections.""" + + +class ConnectionResource(Resource): + """A connection in the resources listing.""" + + mask_type: MaskType + + +class RulesetResource(Resource): + """A ruleset in the resources listing.""" + + mask_type: MaskType + + +class DiscoveryConfigResource(Resource): + """A discovery config in the resources listing.""" + + config_type: DiscoveryConfigType + + +# The model each type's rows are listed as; a type not named here is listed as `Resource`. +RESOURCE_MODEL_BY_TYPE_NAME: dict[str, type[Resource]] = { + ResourceTypeName.connections.value: ConnectionResource, + ResourceTypeName.rulesets.value: RulesetResource, + ResourceTypeName.discovery_configs.value: DiscoveryConfigResource, +} diff --git a/datamasque/client/resources.py b/datamasque/client/resources.py new file mode 100644 index 0000000..d7215aa --- /dev/null +++ b/datamasque/client/resources.py @@ -0,0 +1,56 @@ +from collections import defaultdict +from collections.abc import Sequence +from typing import Optional, Union + +from datamasque.client.base import BaseClient +from datamasque.client.models.project import Project, ProjectId, unwrap_project_id +from datamasque.client.models.resource import RESOURCE_MODEL_BY_TYPE_NAME, Resource, ResourceTypeName + + +class ResourceClient(BaseClient): + """Resource listing API methods. Mixed into `DataMasqueClient`.""" + + def list_resources( + self, + *, + projects: Optional[Sequence[Union[ProjectId, Project]]] = None, + types: Optional[Sequence[ResourceTypeName]] = None, + search: Optional[str] = None, + page_size: int = 500, + ) -> dict[str, list[Resource]]: + """ + Returns the resources the caller can manage, grouped by type name such as `rulesets`, across every page. + + `projects` limits them to those projects, each given as a `Project` or its `ProjectId`; + `types` to those types; and `search` to resources whose name contains every word of it, ignoring case. + The groups are the requested types, or every type the server has when `types` is not given. + An empty `projects` or `types` matches no resources, without a request to the server. + Connections, rulesets and discovery configs are listed as `ConnectionResource`, `RulesetResource` + and `DiscoveryConfigResource`. + + Raises: + DataMasqueApiError: a project does not exist, or any other non-2xx response. + """ + + # `requests` leaves out a parameter whose value is an empty list, which would remove the filter. + if projects is not None and not projects: + return {type_name.value: [] for type_name in types or []} + + if types is not None and not types: + return {} + + params: dict[str, object] = {} + if projects is not None: + params["project"] = [unwrap_project_id(project) for project in projects] + if types is not None: + params["type"] = [type_name.value for type_name in types] + if search is not None: + params["search"] = search + + resources: defaultdict[str, list[Resource]] = defaultdict(list) + for results in self._iter_pages("/api/resources/", params=params, page_size=page_size): + for type_name, rows in results.items(): + model = RESOURCE_MODEL_BY_TYPE_NAME.get(type_name, Resource) + resources[type_name].extend(model.model_validate(row) for row in rows) + + return dict(resources) diff --git a/docs/client.models.rst b/docs/client.models.rst index 0b2f872..bd00a65 100644 --- a/docs/client.models.rst +++ b/docs/client.models.rst @@ -68,6 +68,14 @@ datamasque.client.models.project module :undoc-members: :show-inheritance: +datamasque.client.models.resource module +---------------------------------------- + +.. automodule:: datamasque.client.models.resource + :members: + :undoc-members: + :show-inheritance: + datamasque.client.models.rg_config module ----------------------------------------- diff --git a/docs/client.rst b/docs/client.rst index b0f5a69..276a982 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -92,6 +92,14 @@ datamasque.client.projects module :undoc-members: :show-inheritance: +datamasque.client.resources module +---------------------------------- + +.. automodule:: datamasque.client.resources + :members: + :undoc-members: + :show-inheritance: + datamasque.client.rg_configs module ----------------------------------- diff --git a/tests/test_resources.py b/tests/test_resources.py new file mode 100644 index 0000000..7fde2eb --- /dev/null +++ b/tests/test_resources.py @@ -0,0 +1,152 @@ +"""Tests for listing resources with the DataMasque client.""" + +from datetime import datetime +from typing import Optional + +import pytest +import requests_mock + +from datamasque.client import ( + ConnectionResource, + DataMasqueApiError, + DataMasqueClient, + MaskType, + Project, + ProjectId, + Resource, + ResourceTypeName, + RulesetResource, +) + +RULESET = { + "id": "0b6c2f0e-4a0d-4d8e-9a57-2f0f3c1d9e11", + "name": "mask_customers", + "project": "7-banking", + "created": "2026-10-01T10:00:00Z", + "modified": "2026-10-02T10:00:00Z", + "mask_type": "database", +} +CONNECTION = { + "id": "5e3b8a40-7c2d-4f61-b1a9-0d4e6f2c8b77", + "name": "customers_db", + "project": "7-banking", + "created": None, + "modified": None, + "mask_type": "database", +} +BUILT_IN_SEED = { + "id": "9a1d3c55-2b7e-4c0a-8f16-3e5d7b9c1a24", + "name": "first_names.csv", + "project": None, + "created": "2026-01-01T00:00:00Z", + "modified": "2026-01-01T00:00:00Z", +} + + +def page(results: dict[str, list[dict]], next_url: Optional[str] = None) -> dict: + return { + "count": sum(len(rows) for rows in results.values()), + "next": next_url, + "previous": None, + "results": results, + } + + +def test_list_resources_groups_them_by_type_name(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/resources/", + json=page({"rulesets": [RULESET], "connections": [CONNECTION], "seeds": [BUILT_IN_SEED]}), + ) + resources = client.list_resources() + + assert sorted(resources) == ["connections", "rulesets", "seeds"] + (ruleset,) = resources["rulesets"] + assert isinstance(ruleset, RulesetResource) + assert ruleset.id == RULESET["id"] + assert ruleset.name == "mask_customers" + assert ruleset.project == "7-banking" + assert ruleset.created == datetime.fromisoformat("2026-10-01T10:00:00+00:00") + assert ruleset.mask_type is MaskType.database + (connection,) = resources["connections"] + assert isinstance(connection, ConnectionResource) + assert connection.created is None + (seed,) = resources["seeds"] + assert type(seed) is Resource + assert seed.project is None + + +def test_list_resources_sends_its_filters(client: DataMasqueClient) -> None: + retail = Project.model_validate( + { + "id": 8, + "reference": "8-retail", + "name": "Retail", + "created": "2026-10-01T10:00:00Z", + "updated": "2026-10-01T10:00:00Z", + } + ) + with requests_mock.Mocker() as m: + m.get("http://test-server/api/resources/", json=page({"rulesets": [], "connections": []})) + client.list_resources( + projects=[ProjectId(7), retail], + types=[ResourceTypeName.rulesets, ResourceTypeName.connections], + search="customers", + ) + + assert m.last_request.qs == { + "project": ["7", "8"], + "type": ["rulesets", "connections"], + "search": ["customers"], + "limit": ["500"], + "offset": ["0"], + } + + +@pytest.mark.parametrize( + ("projects", "types", "expected"), + [ + ([], None, {}), + ([], [ResourceTypeName.rulesets], {"rulesets": []}), + ([ProjectId(7)], [], {}), + ], +) +def test_list_resources_with_an_empty_filter_returns_no_resources_without_a_request( + client: DataMasqueClient, + projects: list[ProjectId], + types: Optional[list[ResourceTypeName]], + expected: dict[str, list[Resource]], +) -> None: + with requests_mock.Mocker() as m: + resources = client.list_resources(projects=projects, types=types) + + assert resources == expected + assert m.call_count == 0 + + +def test_list_resources_follows_every_page_and_merges_the_groups(client: DataMasqueClient) -> None: + next_url = "http://test-server/api/resources/?limit=2&offset=2" + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/resources/", + [ + {"json": page({"rulesets": [RULESET], "connections": [CONNECTION]}, next_url=next_url)}, + {"json": page({"rulesets": [RULESET | {"id": "c3d1e5f7-0000-4000-8000-000000000001"}]})}, + ], + ) + resources = client.list_resources(page_size=2) + + assert [ruleset.id for ruleset in resources["rulesets"]] == [RULESET["id"], "c3d1e5f7-0000-4000-8000-000000000001"] + assert [connection.id for connection in resources["connections"]] == [CONNECTION["id"]] + assert m.call_count == 2 + assert m.request_history[1].qs == {"limit": ["2"], "offset": ["2"]} + + +def test_list_resources_raises_for_a_caller_who_cannot_list_them(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get("http://test-server/api/resources/", status_code=403, json={"detail": "Forbidden"}) + + with pytest.raises(DataMasqueApiError) as raised: + client.list_resources() + + assert raised.value.response.status_code == 403 From f9b7a9e486fab511c39efdccdbd6aac7f912e6e2 Mon Sep 17 00:00:00 2001 From: Colin Haywood Date: Fri, 16 Oct 2026 13:48:04 +1300 Subject: [PATCH 05/22] docs: follow the client's docstring patterns for create_project --- datamasque/client/models/project.py | 8 +++----- datamasque/client/projects.py | 5 ++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/datamasque/client/models/project.py b/datamasque/client/models/project.py index 6eedc1b..b264100 100644 --- a/datamasque/client/models/project.py +++ b/datamasque/client/models/project.py @@ -32,11 +32,9 @@ class ProjectCreateRequest(BaseModel): """ Request body for `POST /api/projects/`. - Names are unique across active and archived projects, ignoring case. - The server trims leading and trailing whitespace from `name` and `description`, - and rejects a name that contains "/", ".." or a control character. - It may reject other names too, - and its 400 response gives the reason. + A name must not contain "/", ".." or a control character, + and must differ, ignoring case, from the name of every other project, including deleted ones. + The server trims leading and trailing whitespace from `name` and `description`. """ model_config = ConfigDict(extra="forbid") diff --git a/datamasque/client/projects.py b/datamasque/client/projects.py index 1646b13..7a454a8 100644 --- a/datamasque/client/projects.py +++ b/datamasque/client/projects.py @@ -27,9 +27,8 @@ def create_project(self, request: ProjectCreateRequest) -> Project: """ Creates a new project on the server, and returns it as the server stored it. - Only admin users can create projects, and API tokens cannot. - A name the server rejects (see `ProjectCreateRequest`) raises `DataMasqueApiError` with status 400, - and a caller who is not an admin user raises it with status 403. + Raises: + DataMasqueApiError: the server rejects the name (see `ProjectCreateRequest`), or any other non-2xx response. """ data = request.model_dump(mode="json") From 66c7b449846daaa6841ad2894dc57dcace194a6e Mon Sep 17 00:00:00 2001 From: Elliot Simpson Date: Mon, 19 Oct 2026 10:47:42 +1300 Subject: [PATCH 06/22] fix!: DM-4394 key unfinished runs by connection ID --- HISTORY.rst | 2 + datamasque/client/runs.py | 27 +++++-- tests/test_runs.py | 163 ++++++++++++++++++++++---------------- 3 files changed, 117 insertions(+), 75 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7eb1002..7be9c26 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,8 @@ History 1.7.0 (unreleased) ------------------ +* **Breaking:** ``get_unfinished_runs`` returns its runs keyed by ``ConnectionId`` instead of connection name, + because two connections in different projects can share a name. * Added project models and management APIs. * Added ``list_resources``, which lists resources of every type across projects. diff --git a/datamasque/client/runs.py b/datamasque/client/runs.py index 9f4c87c..3668731 100644 --- a/datamasque/client/runs.py +++ b/datamasque/client/runs.py @@ -4,11 +4,13 @@ from datamasque.client.base import BaseClient from datamasque.client.exceptions import ( + DataMasqueException, FailedToStartError, InvalidLibraryError, InvalidRulesetError, RunNotCancellableError, ) +from datamasque.client.models.connection import ConnectionId from datamasque.client.models.runs import MaskingRunRequest, RunId, RunInfo, UnfinishedRun from datamasque.client.models.status import MaskingRunStatus @@ -63,10 +65,10 @@ def get_db_discovery_result_report(self, run_id: RunId, include_selection_column return response.content return response.text - def get_unfinished_runs(self) -> dict[str, UnfinishedRun]: - """Queries the DM instance for unfinished runs, and returns them organised by connection name.""" + def get_unfinished_runs(self) -> dict[ConnectionId, UnfinishedRun]: + """Queries the DM instance for unfinished runs, and returns them organised by connection ID.""" - unfinished_runs = {} + unfinished_runs: dict[ConnectionId, UnfinishedRun] = {} for status in ( MaskingRunStatus.queued, MaskingRunStatus.running, @@ -89,9 +91,22 @@ def get_unfinished_runs(self) -> dict[str, UnfinishedRun]: for run in data.get("results", []): unfinished_run = UnfinishedRun.model_validate(run) - unfinished_runs[unfinished_run.source_connection.name] = unfinished_run - if unfinished_run.destination_connection is not None: - unfinished_runs[unfinished_run.destination_connection.name] = unfinished_run + source = unfinished_run.source_connection + if source.id is None: + raise DataMasqueException( + f"Server returned unfinished run {unfinished_run.id} without a source connection ID." + ) + + unfinished_runs[source.id] = unfinished_run + + destination = unfinished_run.destination_connection + if destination is not None: + if destination.id is None: + raise DataMasqueException( + f"Server returned unfinished run {unfinished_run.id} without a destination connection ID." + ) + + unfinished_runs[destination.id] = unfinished_run return unfinished_runs diff --git a/tests/test_runs.py b/tests/test_runs.py index 295409a..b31aa7b 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -3,8 +3,10 @@ import pytest import requests_mock +from datamasque.client import DataMasqueClient from datamasque.client.exceptions import ( DataMasqueApiError, + DataMasqueException, FailedToStartError, InvalidLibraryError, InvalidRulesetError, @@ -92,78 +94,101 @@ def test_unfinished_run_str_without_destination(): assert str(run) == '"source_db": Run ID 42 in status `queued`, ruleset "my_ruleset"' -def test_get_unfinished_runs(client): +QUEUED_SOURCE_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a01") +QUEUED_DESTINATION_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a02") +RUNNING_SOURCE_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a03") +RUNNING_DESTINATION_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a04") +VALIDATING_SOURCE_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a05") +VALIDATING_DESTINATION_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a06") +CANCELLING_SOURCE_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a07") + +UNFINISHED_RUNS_URL = ( + "http://test-server/api/runs/?connection_ruleset_name=&ruleset_name=&run_status={}&limit=1&offset=0" +) + + +def unfinished_run_json( + run_id: int, status: str, source: tuple[str, str], destination: tuple[str, str] +) -> dict[str, object]: + """ + Builds an unfinished-run list entry from `(id, name)` pairs. + + An empty ID is sent as `None`, and an empty destination name means none. + """ + return { + "id": run_id, + "source_connection": source[0] or None, + "source_connection_name": source[1], + "destination_connection": destination[0] or None, + "destination_connection_name": destination[1], + "ruleset_name": f"ruleset_{run_id}", + "project": "1-default", + "status": status, + } + + +def test_get_unfinished_runs(client: DataMasqueClient) -> None: + runs = { + "queued": unfinished_run_json( + 1, "queued", (QUEUED_SOURCE_ID, "queued_src"), (QUEUED_DESTINATION_ID, "queued_dst") + ), + "running": unfinished_run_json( + 2, "running", (RUNNING_SOURCE_ID, "running_src"), (RUNNING_DESTINATION_ID, "running_dst") + ), + "validating": unfinished_run_json( + 3, "validating", (VALIDATING_SOURCE_ID, "validating_src"), (VALIDATING_DESTINATION_ID, "validating_dst") + ), + "cancelling": unfinished_run_json(4, "cancelling", (CANCELLING_SOURCE_ID, "cancelling_src"), ("", "")), + } with requests_mock.Mocker() as m: - m.get( - "http://test-server/api/runs/?connection_ruleset_name=&ruleset_name=&run_status=queued&limit=1&offset=0", - json={ - "results": [ - { - "source_connection_name": "queued_src", - "destination_connection_name": "queued_dst", - "id": 1, - "ruleset_name": "ruleset_1", - "project": "1-default", - "status": "queued", - } - ] - }, - status_code=200, - ) - m.get( - "http://test-server/api/runs/?connection_ruleset_name=&ruleset_name=&run_status=running&limit=1&offset=0", - json={ - "results": [ - { - "source_connection_name": "running_src", - "destination_connection_name": "running_dst", - "id": 2, - "ruleset_name": "ruleset_2", - "project": "1-default", - "status": "running", - } - ] - }, - status_code=200, - ) - m.get( - "http://test-server/api/runs/?connection_ruleset_name=&ruleset_name=&run_status=validating&limit=1&offset=0", - json={ - "results": [ - { - "source_connection_name": "validating_src", - "destination_connection_name": "validating_dst", - "id": 3, - "ruleset_name": "ruleset_3", - "project": "1-default", - "status": "validating", - } - ] - }, - status_code=200, - ) - m.get( - "http://test-server/api/runs/?connection_ruleset_name=&ruleset_name=&run_status=cancelling&limit=1&offset=0", - json={ - "results": [ - { - "source_connection_name": "cancelling_src", - "destination_connection_name": "", - "id": 4, - "ruleset_name": "ruleset_4", - "project": "1-default", - "status": "cancelling", - } - ] - }, - status_code=200, - ) + for status, run in runs.items(): + m.get(UNFINISHED_RUNS_URL.format(status), json={"results": [run]}, status_code=200) + + ur = client.get_unfinished_runs() + + # 3 statuses have both source and destination keys, cancelling has only source (empty destination) + assert {connection_id: run.id for connection_id, run in ur.items()} == { + QUEUED_SOURCE_ID: 1, + QUEUED_DESTINATION_ID: 1, + RUNNING_SOURCE_ID: 2, + RUNNING_DESTINATION_ID: 2, + VALIDATING_SOURCE_ID: 3, + VALIDATING_DESTINATION_ID: 3, + CANCELLING_SOURCE_ID: 4, + } + + +def test_get_unfinished_runs_keeps_same_named_connections_apart(client: DataMasqueClient) -> None: + """Two connections with the same name in different projects each keep their own run.""" + runs = { + "queued": unfinished_run_json(1, "queued", (QUEUED_SOURCE_ID, "orders_db"), ("", "")), + "running": unfinished_run_json(2, "running", (RUNNING_SOURCE_ID, "orders_db"), ("", "")), + } + with requests_mock.Mocker() as m: + for status in ("queued", "running", "validating", "cancelling"): + results = [runs[status]] if status in runs else [] + m.get(UNFINISHED_RUNS_URL.format(status), json={"results": results}, status_code=200) ur = client.get_unfinished_runs() - # 3 statuses have both source and destination keys, cancelling has only source (empty destination) - assert len(ur) == 7 - for run in ur.values(): - assert isinstance(run, UnfinishedRun) + + assert {connection_id: run.id for connection_id, run in ur.items()} == {QUEUED_SOURCE_ID: 1, RUNNING_SOURCE_ID: 2} + + +@pytest.mark.parametrize( + "source, destination, missing", + [ + (("", "orders_db"), ("", ""), "source"), + ((QUEUED_SOURCE_ID, "orders_db"), ("", "orders_copy"), "destination"), + ], +) +def test_get_unfinished_runs_without_connection_id_raises( + client: DataMasqueClient, source: tuple[str, str], destination: tuple[str, str], missing: str +) -> None: + run = unfinished_run_json(1, "queued", source, destination) + with requests_mock.Mocker() as m: + m.get(UNFINISHED_RUNS_URL.format("queued"), json={"results": [run]}, status_code=200) + with pytest.raises(DataMasqueException, match=f"without a {missing} connection ID"): + client.get_unfinished_runs() def test_start_masking_run(client): From 90655145b2439b1ae58bb5e1e293969addb6db4c Mon Sep 17 00:00:00 2001 From: Elliot Simpson Date: Mon, 19 Oct 2026 10:12:05 +1300 Subject: [PATCH 07/22] feat: DM-4394 add list_projects and get_project --- datamasque/client/projects.py | 31 ++++++++++++- tests/test_projects.py | 82 ++++++++++++++++++++++++++++++++--- 2 files changed, 107 insertions(+), 6 deletions(-) diff --git a/datamasque/client/projects.py b/datamasque/client/projects.py index 7a454a8..149ba0e 100644 --- a/datamasque/client/projects.py +++ b/datamasque/client/projects.py @@ -1,5 +1,5 @@ import logging -from typing import Union +from typing import Iterator, Union from requests import Response @@ -23,6 +23,35 @@ def _get_conflicts(response: Response) -> list[ProjectDeleteConflict]: class ProjectClient(BaseClient): """Project API methods. Mixed into `DataMasqueClient`.""" + def iter_projects(self, *, include_archived: bool = False) -> Iterator[Project]: + """ + Lazily iterate projects via the paginated endpoint. + + Archived projects are skipped unless `include_archived` is set. + """ + + params = {"include_archived": "true"} if include_archived else None + return self._iter_paginated("/api/projects/", model=Project, params=params) + + def list_projects(self, *, include_archived: bool = False) -> list[Project]: + """ + Lists projects. + + Archived projects are left out unless `include_archived` is set. + """ + + return list(self.iter_projects(include_archived=include_archived)) + + def get_project(self, project: Union[ProjectId, Project]) -> Project: + """ + Retrieves a single project, given as a `Project` or its `ProjectId`. + + Archived projects can be retrieved too. + """ + + response = self.make_request("GET", f"/api/projects/{unwrap_project_id(project)}/") + return Project.model_validate(response.json()) + def create_project(self, request: ProjectCreateRequest) -> Project: """ Creates a new project on the server, and returns it as the server stored it. diff --git a/tests/test_projects.py b/tests/test_projects.py index 17131e8..38b1556 100644 --- a/tests/test_projects.py +++ b/tests/test_projects.py @@ -1,6 +1,7 @@ """Tests for project support in the DataMasque client.""" from datetime import datetime +from typing import Union import pytest import requests_mock @@ -17,7 +18,7 @@ ProjectId, ) -CREATE_RESPONSE = { +BANKING_PROJECT = { "id": 7, "reference": "7-banking", "name": "Banking", @@ -26,6 +27,15 @@ "updated": "2026-10-01T10:00:00Z", } +LEGACY_PROJECT = { + "id": 8, + "reference": "8-legacy", + "name": "Legacy", + "description": "", + "created": "2026-09-01T10:00:00Z", + "updated": "2026-10-01T10:00:00Z", +} + CONFLICTS = [ { "reason": "name_in_use", @@ -59,7 +69,7 @@ def test_create_project(client: DataMasqueClient) -> None: request = ProjectCreateRequest(name="Banking", description="Customer banking data") with requests_mock.Mocker() as m: - m.post("http://test-server/api/projects/", json=CREATE_RESPONSE, status_code=201) + m.post("http://test-server/api/projects/", json=BANKING_PROJECT, status_code=201) project = client.create_project(request) assert m.last_request.json() == {"name": "Banking", "description": "Customer banking data"} @@ -76,7 +86,7 @@ def test_create_project_returns_the_name_and_description_as_stored(client: DataM request = ProjectCreateRequest(name=" Banking ", description=" Customer banking data ") with requests_mock.Mocker() as m: - m.post("http://test-server/api/projects/", json=CREATE_RESPONSE, status_code=201) + m.post("http://test-server/api/projects/", json=BANKING_PROJECT, status_code=201) project = client.create_project(request) assert m.last_request.json() == {"name": " Banking ", "description": " Customer banking data "} @@ -142,8 +152,8 @@ def test_get_project_delete_conflicts_returns_empty_list_when_project_can_be_del def test_project_delete_calls_accept_projects_as_well_as_ids(client: DataMasqueClient) -> None: - project = Project.model_validate(CREATE_RESPONSE) - destination = Project.model_validate(CREATE_RESPONSE | {"id": 8, "reference": "8-retail", "name": "Retail"}) + project = Project.model_validate(BANKING_PROJECT) + destination = Project.model_validate(BANKING_PROJECT | {"id": 8, "reference": "8-retail", "name": "Retail"}) with requests_mock.Mocker() as m: m.get("http://test-server/api/projects/7/delete-check/", json={"conflicts": []}) @@ -194,3 +204,65 @@ def test_delete_project_raises_a_plain_api_error_for_a_status_other_than_409( assert not isinstance(exc_info.value, ProjectDeleteConflictError) assert exc_info.value.response.status_code == status_code + + +def test_list_projects_follows_pages(client: DataMasqueClient) -> None: + """With more active projects than fit on a page of 100, the client follows `next` to the last page.""" + projects = [ + { + **BANKING_PROJECT, + "id": project_id, + "reference": f"{project_id}-project-{project_id}", + "name": f"Project {project_id}", + } + for project_id in range(1, 102) + ] + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/projects/?limit=100&offset=0", + json={ + "count": 101, + "next": "http://test-server/api/projects/?limit=100&offset=100", + "results": projects[:100], + }, + complete_qs=True, + ) + m.get( + "http://test-server/api/projects/?limit=100&offset=100", + json={"count": 101, "next": None, "results": projects[100:]}, + complete_qs=True, + ) + result = client.list_projects() + + assert [project.id for project in result] == list(range(1, 102)) + + +def test_list_projects_include_archived(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/projects/?include_archived=true&limit=100&offset=0", + json={"count": 2, "next": None, "results": [BANKING_PROJECT, LEGACY_PROJECT]}, + complete_qs=True, + ) + projects = client.list_projects(include_archived=True) + + assert [project.id for project in projects] == [ProjectId(7), ProjectId(8)] + + +@pytest.mark.parametrize("project", [ProjectId(7), Project.model_validate(BANKING_PROJECT)]) +def test_get_project(client: DataMasqueClient, project: Union[ProjectId, Project]) -> None: + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/7/", json=BANKING_PROJECT, complete_qs=True) + result = client.get_project(project) + + assert result.id == ProjectId(7) + assert result.name == "Banking" + + +def test_get_missing_project_raises(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/99/", json={"detail": "Not found."}, status_code=404) + with pytest.raises(DataMasqueApiError) as exc_info: + client.get_project(ProjectId(99)) + + assert exc_info.value.response.status_code == 404 From 1e63a0bd3ba10a886e5da17d395c18938a3926f7 Mon Sep 17 00:00:00 2001 From: Elliot Simpson Date: Mon, 19 Oct 2026 10:32:58 +1300 Subject: [PATCH 08/22] feat!: DM-4394 replace namespace with project and scope lookups by project --- HISTORY.rst | 18 ++ datamasque/client/base.py | 24 +++ datamasque/client/connections.py | 43 +++-- datamasque/client/discovery.py | 11 +- .../client/discovery_config_libraries.py | 44 ++--- datamasque/client/discovery_configs.py | 53 +++--- datamasque/client/files.py | 42 +++-- datamasque/client/models/connection.py | 10 +- datamasque/client/models/discovery_config.py | 8 +- .../client/models/discovery_config_library.py | 18 +- datamasque/client/models/files.py | 2 + datamasque/client/models/project.py | 14 ++ datamasque/client/models/rg_config.py | 8 +- datamasque/client/models/ruleset.py | 8 +- datamasque/client/models/ruleset_library.py | 14 +- datamasque/client/models/table_reference.py | 8 +- datamasque/client/rg_configs.py | 32 ++-- datamasque/client/ruleset_libraries.py | 55 +++--- datamasque/client/rulesets.py | 30 ++-- datamasque/client/table_references.py | 49 ++++-- tests/conftest.py | 6 +- tests/helpers.py | 36 ++++ tests/test_connections.py | 162 ++++++++++++++++-- tests/test_discovery.py | 37 +++- tests/test_discovery_config_libraries.py | 135 ++++++--------- tests/test_discovery_configs.py | 93 +++++----- tests/test_files.py | 88 +++++++++- tests/test_model_project.py | 29 +++- tests/test_projects.py | 153 ++++++++++++++++- tests/test_rg_configs.py | 43 +++-- tests/test_ruleset_library.py | 121 +++++++------ tests/test_rulesets.py | 58 ++++--- tests/test_runs.py | 10 +- tests/test_table_references.py | 75 +++++++- 34 files changed, 1121 insertions(+), 416 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7be9c26..7a7a4ae 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,8 +5,26 @@ History 1.7.0 (unreleased) ------------------ +* ``ConnectionConfig``, ``Ruleset``, ``RulesetLibrary``, ``DiscoveryConfig``, ``DiscoveryConfigLibrary``, ``RGConfig`` + and ``TableReference`` send their ``project`` to the server when it is set. + It takes a project's ``-`` reference, such as ``Project.reference``, or its ID as a string. + Left unset, a create goes into the server's default project. + Each create sets the model's ``project`` to the project the server stored it in. +* **Breaking:** ``upload_file`` and ``upload_file_if_not_exists`` take a required ``project``. +* **Breaking:** ``namespace`` is removed from ``RulesetLibrary`` and ``DiscoveryConfigLibrary``, + and the client drops it from server responses so that it is never sent back. + The library by-name helpers take ``project`` in its place. +* **Breaking:** names are unique per project, so every by-name helper takes a required ``project``: + ``get_*_by_name``, ``delete_*_by_name_if_exists`` and ``get_file_of_type_by_name``. + ``create_or_update_*`` updates an existing resource only when it has the same name in the same project. + With the model's ``project`` unset, that is the server's only active project, + and ``DataMasqueException`` is raised when there is no active project or more than one. + The methods' ``project`` parameters take a ``Project`` or its ``ProjectId``. * **Breaking:** ``get_unfinished_runs`` returns its runs keyed by ``ConnectionId`` instead of connection name, because two connections in different projects can share a name. +* ``list_connections``, ``list_table_references`` and ``list_files_of_type`` take an optional ``project`` filter. +* ``DataMasqueFile`` gains a read-only ``project`` field, which is ``None`` for built-in seeds. +* ``get_generated_rulesets`` sets each ruleset's ``project`` to the connection's project. * Added project models and management APIs. * Added ``list_resources``, which lists resources of every type across projects. diff --git a/datamasque/client/base.py b/datamasque/client/base.py index f1ce266..68db356 100644 --- a/datamasque/client/base.py +++ b/datamasque/client/base.py @@ -17,10 +17,13 @@ from datamasque.client.exceptions import ( DataMasqueApiError, + DataMasqueException, DataMasqueNotReadyError, DataMasqueTransportError, ) from datamasque.client.models.dm_instance import DataMasqueInstanceConfig +from datamasque.client.models.pagination import Page +from datamasque.client.models.project import Project, ProjectId from datamasque.client.spcs import install_spcs_gateway_auth logger = logging.getLogger(__name__) @@ -313,6 +316,27 @@ def _delete_if_exists(self, path: str, *, params: Optional[dict] = None) -> None self._raise_for_status(response) + def _resolve_project(self, project: Optional[Union[ProjectId, str]]) -> Union[ProjectId, str]: + """ + Returns `project`, or when it is `None`, the ID of the only active project. + + That is the project the server puts a create into when it is sent no `project`. + With no active project, or more than one, the server rejects such a create, + so this raises `DataMasqueException` instead. + """ + + if project is not None: + return project + + response = self.make_request("GET", "/api/projects/", params={"limit": 1}) + page = Page[Project].model_validate(response.json()) + if page.count != 1: + raise DataMasqueException( + f"No `project` is set and the server has {page.count} active projects. Set `project` to choose one." + ) + + return page.results[0].id + def _iter_paginated( self, path: str, diff --git a/datamasque/client/connections.py b/datamasque/client/connections.py index e41e46b..ddf4570 100644 --- a/datamasque/client/connections.py +++ b/datamasque/client/connections.py @@ -1,8 +1,10 @@ import logging +from typing import Optional, Union from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueException from datamasque.client.models.connection import ConnectionConfig, ConnectionId, validate_connection +from datamasque.client.models.project import Project, ProjectId, unwrap_project_id logger = logging.getLogger(__name__) @@ -10,25 +12,39 @@ class ConnectionClient(BaseClient): """Connection-related API methods. Mixed into `DataMasqueClient`.""" - def list_connections(self) -> list[ConnectionConfig]: + def list_connections(self, project: Optional[Union[ProjectId, Project]] = None) -> list[ConnectionConfig]: """ - Lists all configured connections. + Lists all configured connections, or only those in `project` when it is given. + `project` is a `Project` or its `ProjectId`. Note that database passwords and connection strings are returned encrypted over the API and so are `None` on the returned `ConnectionConfig` objects. """ - response = self.make_request("GET", "/api/connections/") + return self._list_connections(unwrap_project_id(project) if project is not None else None) + + def _list_connections(self, project: Optional[Union[ProjectId, str]]) -> list[ConnectionConfig]: + """Lists the connections in `project`, given as an ID or `-` reference, or in every project.""" + + params = {"project": project} if project is not None else None + response = self.make_request("GET", "/api/connections/", params=params) return [validate_connection(payload) for payload in response.json()] def create_or_update_connection(self, connection_config: ConnectionConfig) -> ConnectionConfig: - """Creates or updates the connection in DM, and sets the `id` field on the given `connection_config`.""" + """ + Creates or updates the connection in DM, and sets the `id` field on the given `connection_config`. + + An existing connection with the same name in the same project is updated. + With no `project` set, the server's only active project is used, + and `DataMasqueException` is raised when it has no active project or more than one. + """ connection_id = connection_config.id - all_connections = self.list_connections() connections_matching_name = [ - connection for connection in all_connections if connection.name == connection_config.name + connection + for connection in self._list_connections(self._resolve_project(connection_config.project)) + if connection.name == connection_config.name ] if connections_matching_name: connection_id = connections_matching_name[0].id @@ -45,6 +61,7 @@ def create_or_update_connection(self, connection_config: ConnectionConfig) -> Co server_connection_id = ConnectionId(connection_data["id"]) logger.debug("%s creation successful", type(connection_config).__name__) connection_config.id = server_connection_id + connection_config.project = connection_data.get("project") return connection_config def delete_connection_by_id_if_exists(self, connection_id: ConnectionId) -> None: @@ -52,11 +69,17 @@ def delete_connection_by_id_if_exists(self, connection_id: ConnectionId) -> None self._delete_if_exists(f"/api/connections/{connection_id}/") - def delete_connection_by_name_if_exists(self, connection_name: str) -> None: - """Deletes the connection with the given name. No-op if the connection does not exist.""" + def delete_connection_by_name_if_exists(self, connection_name: str, project: Union[ProjectId, Project]) -> None: + """ + Deletes the connection with the given name in the given project. + + `project` is a `Project` or its `ProjectId`. + No-op if the connection does not exist. + """ - all_connections = self.list_connections() - connections_matching_name = [connection for connection in all_connections if connection.name == connection_name] + connections_matching_name = [ + connection for connection in self.list_connections(project) if connection.name == connection_name + ] for connection in connections_matching_name: if connection.id is None: raise DataMasqueException(f'Server returned a connection named "{connection.name}" without an `id`.') diff --git a/datamasque/client/discovery.py b/datamasque/client/discovery.py index 2faaaad..9145298 100644 --- a/datamasque/client/discovery.py +++ b/datamasque/client/discovery.py @@ -15,7 +15,7 @@ InvalidDiscoveryConfigError, RGConfigNotFoundError, ) -from datamasque.client.models.connection import ConnectionId +from datamasque.client.models.connection import ConnectionId, validate_connection from datamasque.client.models.data_selection import ( SelectedColumns, SelectedData, @@ -247,6 +247,7 @@ def get_generated_rulesets(self, connection_id: ConnectionId) -> list[Ruleset]: and `DataMasqueException` if it failed or was cancelled. Note that the ruleset(s) have autogenerated names, which you may want to customize before uploading. + Their `project` is the connection's project. """ status = self.get_async_ruleset_generation_task_status(connection_id) @@ -273,6 +274,8 @@ def get_generated_rulesets(self, connection_id: ConnectionId) -> list[Ruleset]: # (which carries the generated ruleset inline as `generated_ruleset`). # `requests` follows the 303 transparently, so we distinguish by the presence of # a `Content-Disposition: attachment` header, which Django's `FileResponse` sets on the ZIP response. + project = validate_connection(self.make_request("GET", f"/api/connections/{connection_id}/").json()).project + response = self.make_request( method="GET", path=f"/api/async-generate-ruleset/{connection_id}/download-rulesets/", @@ -285,7 +288,9 @@ def get_generated_rulesets(self, connection_id: ConnectionId) -> list[Ruleset]: if file_info.filename.endswith((".yml", ".yaml")): with zip_file.open(file_info) as file: yaml_content = file.read().decode("utf-8") - rulesets.append(Ruleset(name=Path(file_info.filename).stem, yaml=yaml_content)) + rulesets.append( + Ruleset(name=Path(file_info.filename).stem, project=project, yaml=yaml_content) + ) if not rulesets: raise DataMasqueException( @@ -302,7 +307,7 @@ def get_generated_rulesets(self, connection_id: ConnectionId) -> list[Ruleset]: f"but no ruleset was returned on the task-status record." ) - return [Ruleset(name="generated_ruleset", yaml=generated)] + return [Ruleset(name="generated_ruleset", project=project, yaml=generated)] def start_schema_discovery_run(self, discovery_config: SchemaDiscoveryRequest) -> RunId: """ diff --git a/datamasque/client/discovery_config_libraries.py b/datamasque/client/discovery_config_libraries.py index bd1e1fb..781d277 100644 --- a/datamasque/client/discovery_config_libraries.py +++ b/datamasque/client/discovery_config_libraries.py @@ -1,9 +1,10 @@ import logging -from typing import Optional +from typing import Optional, Union from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary, DiscoveryConfigLibraryId +from datamasque.client.models.project import Project, ProjectId, unwrap_project_id logger = logging.getLogger(__name__) @@ -30,22 +31,18 @@ def get_discovery_config_library(self, library_id: DiscoveryConfigLibraryId) -> response = self.make_request("GET", f"/api/discovery/config-libraries/{library_id}/") return DiscoveryConfigLibrary.model_validate(response.json()) - def _get_discovery_config_library_id_by_name(self, name: str, namespace: str) -> Optional[DiscoveryConfigLibraryId]: - """ - Returns the ID of the library with the given name and namespace, or `None` if there is none. - - The listing is filtered by name to keep the response small; - the namespace is matched client-side, - since the API's namespace filter cannot select the default (empty) namespace. - """ + def _get_discovery_config_library_id_by_name( + self, name: str, project: Union[ProjectId, str] + ) -> Optional[DiscoveryConfigLibraryId]: + """Returns the ID of the library with the given name in the given project, or `None` if there is none.""" response = self.make_request( "GET", "/api/discovery/config-libraries/", - params={"name_exact": name}, + params={"name_exact": name, "project": project}, ) entries = [DiscoveryConfigLibrary.model_validate(item) for item in response.json()] - matches = [entry for entry in entries if entry.name == name and entry.namespace == namespace] + matches = [entry for entry in entries if entry.name == name] if not matches: return None @@ -57,15 +54,17 @@ def _get_discovery_config_library_id_by_name(self, name: str, namespace: str) -> return matches[0].id - def get_discovery_config_library_by_name(self, name: str, namespace: str = "") -> Optional[DiscoveryConfigLibrary]: + def get_discovery_config_library_by_name( + self, name: str, project: Union[ProjectId, Project] + ) -> Optional[DiscoveryConfigLibrary]: """ - Looks for a discovery config library matching the given name and namespace (case-sensitive, exact match). + Looks for a discovery config library matching the given name (case-sensitive, exact match) in the given project. - Library names are unique within a namespace. + `project` is a `Project` or its `ProjectId`. Returns it (with full YAML content) if found, otherwise `None`. """ - library_id = self._get_discovery_config_library_id_by_name(name, namespace) + library_id = self._get_discovery_config_library_id_by_name(name, unwrap_project_id(project)) if library_id is None: return None @@ -86,6 +85,7 @@ def create_discovery_config_library(self, library: DiscoveryConfigLibrary) -> Di response = self.make_request("POST", "/api/discovery/config-libraries/", data=data) created = DiscoveryConfigLibrary.model_validate(response.json()) library.id = created.id + library.project = created.project library.is_valid = created.is_valid library.validation_error = created.validation_error library.usage_count = created.usage_count @@ -123,12 +123,14 @@ def update_discovery_config_library(self, library: DiscoveryConfigLibrary) -> Di def create_or_update_discovery_config_library(self, library: DiscoveryConfigLibrary) -> DiscoveryConfigLibrary: """ - Creates the library, or updates the existing one with the same name and namespace. + Creates the library, or updates the existing one with the same name in the same project. + With no `project` set, the server's only active project is used, + and `DataMasqueException` is raised when it has no active project or more than one. Sets the library's `id` property. """ - library_id = self._get_discovery_config_library_id_by_name(library.name, library.namespace) + library_id = self._get_discovery_config_library_id_by_name(library.name, self._resolve_project(library.project)) if library_id is not None: library.id = library_id return self.update_discovery_config_library(library) @@ -151,15 +153,15 @@ def delete_discovery_config_library_by_id_if_exists( self._delete_if_exists(f"/api/discovery/config-libraries/{library_id}/", params=params) def delete_discovery_config_library_by_name_if_exists( - self, name: str, namespace: str = "", *, force: bool = False + self, name: str, project: Union[ProjectId, Project], *, force: bool = False ) -> None: """ - Deletes the discovery config library with the given name and namespace. + Deletes the discovery config library with the given name in the given project. - Library names are unique within a namespace. + `project` is a `Project` or its `ProjectId`. No-op if no such library exists. """ - library_id = self._get_discovery_config_library_id_by_name(name, namespace) + library_id = self._get_discovery_config_library_id_by_name(name, unwrap_project_id(project)) if library_id is not None: self.delete_discovery_config_library_by_id_if_exists(library_id, force=force) diff --git a/datamasque/client/discovery_configs.py b/datamasque/client/discovery_configs.py index d19f900..cea748f 100644 --- a/datamasque/client/discovery_configs.py +++ b/datamasque/client/discovery_configs.py @@ -1,10 +1,11 @@ import logging -from typing import Iterator, Optional +from typing import Iterator, Optional, Union from datamasque.client.base import BaseClient -from datamasque.client.exceptions import DataMasqueApiError, DataMasqueException +from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType from datamasque.client.models.pagination import Page +from datamasque.client.models.project import Project, ProjectId, unwrap_project_id logger = logging.getLogger(__name__) @@ -34,14 +35,14 @@ def get_discovery_config(self, config_id: DiscoveryConfigId) -> DiscoveryConfig: return DiscoveryConfig.model_validate(response.json()) def _get_discovery_config_id_by_name( - self, name: str, config_type: DiscoveryConfigType + self, name: str, config_type: DiscoveryConfigType, project: Union[ProjectId, str] ) -> Optional[DiscoveryConfigId]: - """Return the id of the config matching name and type via a single list request, or `None`.""" + """Return the id of the config matching name, type and project via a single list request, or `None`.""" response = self.make_request( "GET", "/api/discovery/configs/", - params={"name_exact": name, "config_type": config_type.value, "limit": 1}, + params={"name_exact": name, "config_type": config_type.value, "project": project, "limit": 1}, ) page = Page[DiscoveryConfig].model_validate(response.json()) if not page.results: @@ -56,15 +57,18 @@ def _get_discovery_config_id_by_name( return config_id - def get_discovery_config_by_name(self, name: str, config_type: DiscoveryConfigType) -> Optional[DiscoveryConfig]: + def get_discovery_config_by_name( + self, name: str, config_type: DiscoveryConfigType, project: Union[ProjectId, Project] + ) -> Optional[DiscoveryConfig]: """ - Looks for a discovery config matching the given name and type (case-sensitive, exact match). + Looks for a discovery config matching the given name (case-sensitive, exact match), type and project. - Config names are unique per type, so a type is required to identify a single config. + Config names are unique per type and project, so both are required to identify a single config. + `project` is a `Project` or its `ProjectId`. Returns it if found, otherwise `None`. """ - config_id = self._get_discovery_config_id_by_name(name, config_type) + config_id = self._get_discovery_config_id_by_name(name, config_type, unwrap_project_id(project)) if config_id is None: return None @@ -86,6 +90,7 @@ def create_discovery_config(self, config: DiscoveryConfig) -> DiscoveryConfig: response = self.make_request("POST", "/api/discovery/configs/", data=data) created = DiscoveryConfig.model_validate(response.json()) config.id = created.id + config.project = created.project config.is_valid = created.is_valid config.validation_error = created.validation_error config.validation_error_details = created.validation_error_details @@ -123,12 +128,16 @@ def update_discovery_config(self, config: DiscoveryConfig) -> DiscoveryConfig: def create_or_update_discovery_config(self, config: DiscoveryConfig) -> DiscoveryConfig: """ - Creates the config if it doesn't exist, or updates it if one with the same name already exists. + Creates the config, or updates the existing one with the same name and type in the same project. + With no `project` set, the server's only active project is used, + and `DataMasqueException` is raised when it has no active project or more than one. Sets the config's `id` property. """ - existing_id = self._get_discovery_config_id_by_name(config.name, config.config_type) + existing_id = self._get_discovery_config_id_by_name( + config.name, config.config_type, self._resolve_project(config.project) + ) if existing_id is not None: config.id = existing_id return self.update_discovery_config(config) @@ -144,24 +153,20 @@ def delete_discovery_config_by_id_if_exists(self, config_id: DiscoveryConfigId) self._delete_if_exists(f"/api/discovery/configs/{config_id}/") - def delete_discovery_config_by_name_if_exists(self, name: str, config_type: DiscoveryConfigType) -> None: + def delete_discovery_config_by_name_if_exists( + self, name: str, config_type: DiscoveryConfigType, project: Union[ProjectId, Project] + ) -> None: """ - Deletes the discovery config with the given name and type. + Deletes the discovery config with the given name and type in the given project. - Config names are unique per type, so a type is required to identify a single config. + Config names are unique per type and project, so both are required to identify a single config. + `project` is a `Project` or its `ProjectId`. No-op if no such config exists. """ - matching = [ - config - for config in self.list_discovery_configs() - if config.name == name and config.config_type is config_type - ] - for config in matching: - if config.id is None: - raise DataMasqueException(f'Server returned a discovery config named "{config.name}" without an `id`.') - - self.delete_discovery_config_by_id_if_exists(config.id) + config_id = self._get_discovery_config_id_by_name(name, config_type, unwrap_project_id(project)) + if config_id is not None: + self.delete_discovery_config_by_id_if_exists(config_id) def get_default_discovery_config_yaml(self) -> str: """Returns the server's built-in default discovery configuration as a YAML string.""" diff --git a/datamasque/client/files.py b/datamasque/client/files.py index 89d1a67..d7f196c 100644 --- a/datamasque/client/files.py +++ b/datamasque/client/files.py @@ -3,6 +3,7 @@ from datamasque.client.base import BaseClient, UploadFile, read_file_or_content from datamasque.client.models.files import DataMasqueFile +from datamasque.client.models.project import Project, ProjectId, unwrap_project_id FileTypeT = TypeVar("FileTypeT", bound=DataMasqueFile) @@ -15,13 +16,15 @@ def upload_file( file_type: Type[FileTypeT], file_name: str, file_path_or_content: Union[str, bytes, Path], + project: Union[ProjectId, Project], ) -> FileTypeT: """ - Uploads a file of the given type to the DataMasque server. + Uploads a file of the given type to the DataMasque server, into the given project. `file_type` must be a concrete subclass of `DataMasqueFile` (`SeedFile`, `OracleWalletFile`, `SslZipFile`, `SnowflakeKeyFile`). `file_path_or_content` may be a path (as `str` or `Path`), raw `bytes`, or a file-like object. + `project` is a `Project` or its `ProjectId`. """ name, content = read_file_or_content(file_path_or_content, file_name) @@ -30,7 +33,7 @@ def upload_file( response = self.make_request( "POST", file_type.get_url(), - data={"name": file_name, **file_type.get_extra_form_data()}, + data={"name": file_name, "project": unwrap_project_id(project), **file_type.get_extra_form_data()}, files=[ UploadFile( field_name=file_type.get_content_param_name(), @@ -56,37 +59,50 @@ def delete_file_if_exists(self, file: DataMasqueFile) -> None: # file.get_url() ends with a slash so no need to insert one before the id self._delete_if_exists(f"{file.get_url()}{file.id}/") - def list_files_of_type(self, file_type: Type[FileTypeT]) -> list[FileTypeT]: - """Returns all files of the given type (a concrete subclass of `DataMasqueFile`).""" + def list_files_of_type( + self, file_type: Type[FileTypeT], project: Optional[Union[ProjectId, Project]] = None + ) -> list[FileTypeT]: + """ + Returns all files of the given type (a concrete subclass of `DataMasqueFile`). + + Only files in `project` are returned when it is given. + `project` is a `Project` or its `ProjectId`. + """ - response = self.make_request("GET", file_type.get_url()) + params = {"project": unwrap_project_id(project)} if project is not None else None + response = self.make_request("GET", file_type.get_url(), params=params) return [file_type.model_validate(file) for file in response.json()] - def get_file_of_type_by_name(self, file_type: Type[FileTypeT], name: str) -> Optional[FileTypeT]: + def get_file_of_type_by_name( + self, file_type: Type[FileTypeT], name: str, project: Union[ProjectId, Project] + ) -> Optional[FileTypeT]: """ - Looks for a file of the given type (a concrete subclass of `DataMasqueFile`) with the given `name`. + Looks for a file of the given type (a concrete subclass of `DataMasqueFile`) by name in the given project. Returns it if found, otherwise `None`. """ - matching_files = [f for f in self.list_files_of_type(file_type) if f.name == name] + matching_files = [f for f in self.list_files_of_type(file_type, project) if f.name == name] return matching_files[0] if matching_files else None - def upload_file_if_not_exists(self, file_type: Type[FileTypeT], file_path: Union[str, Path]) -> Optional[FileTypeT]: + def upload_file_if_not_exists( + self, file_type: Type[FileTypeT], file_path: Union[str, Path], project: Union[ProjectId, Project] + ) -> Optional[FileTypeT]: """ - Upload a file only if one with the same name doesn't already exist. + Upload a file only if one with the same name doesn't already exist in the project. Args: file_type: A concrete subclass of `DataMasqueFile` (e.g., SeedFile, OracleWalletFile). file_path: Path to the file to upload. + project: A `Project` or its `ProjectId`. Returns: The uploaded file object if a new file was uploaded, or None if a file - with the same name already exists. + with the same name already exists in the project. """ file_path = Path(file_path) - if self.get_file_of_type_by_name(file_type, file_path.name) is not None: + if self.get_file_of_type_by_name(file_type, file_path.name, project) is not None: return None - return self.upload_file(file_type, file_path.name, file_path) + return self.upload_file(file_type, file_path.name, file_path, project) diff --git a/datamasque/client/models/connection.py b/datamasque/client/models/connection.py index 5bec97b..086b166 100644 --- a/datamasque/client/models/connection.py +++ b/datamasque/client/models/connection.py @@ -121,13 +121,15 @@ class ConnectionConfig(BaseModel): model_config = ConfigDict(extra="allow", populate_by_name=True) name: str + project: Optional[str] = None + """ + The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, + or the project's ID as a string. + Left as `None`, a create goes into the server's default project. + """ id: Optional[ConnectionId] = None # Server-populated and read-only, so it is excluded when serializing a config back to the API. license_lock: Optional[LicenseLock] = Field(default=None, exclude=True) - # Its project's `-` reference, e.g. `1-default`; - # `None` until it is created on the server. - # Server-populated, so excluded when serializing. - project: Optional[str] = Field(default=None, exclude=True) # Whether the connection's protected fields are currently locked under the license edit lock. target_frozen: bool = Field(default=False, exclude=True) # The protected fields currently locked, as server config keys (e.g. `schema`, not `db_schema`); diff --git a/datamasque/client/models/discovery_config.py b/datamasque/client/models/discovery_config.py index 086257f..6a5a909 100644 --- a/datamasque/client/models/discovery_config.py +++ b/datamasque/client/models/discovery_config.py @@ -40,13 +40,17 @@ class DiscoveryConfig(BaseModel): model_config = ConfigDict(extra="allow", populate_by_name=True) name: str + project: Optional[str] = None + """ + The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, + or the project's ID as a string. + Left as `None`, a create goes into the server's default project. + """ yaml: Optional[str] = Field(default=None, alias="config_yaml") config_type: DiscoveryConfigType # Server-populated read-only fields, excluded from request bodies. id: Optional[DiscoveryConfigId] = Field(default=None, exclude=True) - project: Optional[str] = Field(default=None, exclude=True) - """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True) """Validation status; may be `in_progress` briefly after creating a large config.""" validation_error: Optional[str] = Field(default=None, exclude=True) diff --git a/datamasque/client/models/discovery_config_library.py b/datamasque/client/models/discovery_config_library.py index 629160f..c4ef39d 100644 --- a/datamasque/client/models/discovery_config_library.py +++ b/datamasque/client/models/discovery_config_library.py @@ -1,8 +1,9 @@ from datetime import datetime from typing import NewType, Optional -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator +from datamasque.client.models.project import drop_namespace from datamasque.client.models.status import ValidationStatus DiscoveryConfigLibraryId = NewType("DiscoveryConfigLibraryId", str) @@ -10,21 +11,24 @@ class DiscoveryConfigLibrary(BaseModel): """ - Represents a named, namespaced, persisted YAML discovery config library. + Represents a named, persisted YAML discovery config library. A library is untyped: the same library may be imported by both database and - file discovery configs, and its name is unique within a namespace. + file discovery configs, and its name is unique within its project. """ model_config = ConfigDict(extra="allow", populate_by_name=True) name: str - namespace: str = "" + project: Optional[str] = None + """ + The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, + or the project's ID as a string. + Left as `None`, a create goes into the server's default project. + """ yaml: Optional[str] = Field(default=None, alias="config_yaml") # Server-populated read-only fields, excluded from request bodies. id: Optional[DiscoveryConfigLibraryId] = Field(default=None, exclude=True) - project: Optional[str] = Field(default=None, exclude=True) - """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True) """Validation status; libraries are validated synchronously on create/update.""" validation_error: Optional[str] = Field(default=None, exclude=True) @@ -33,3 +37,5 @@ class DiscoveryConfigLibrary(BaseModel): """Number of active discovery configs that import this library.""" created: Optional[datetime] = Field(default=None, exclude=True) modified: Optional[datetime] = Field(default=None, exclude=True) + + _drop_namespace = model_validator(mode="before")(drop_namespace) diff --git a/datamasque/client/models/files.py b/datamasque/client/models/files.py index 13c9846..6b8dc65 100644 --- a/datamasque/client/models/files.py +++ b/datamasque/client/models/files.py @@ -16,6 +16,8 @@ class DataMasqueFile(BaseModel): created_date: datetime modified_date: Optional[datetime] = None id: Optional[FileId] = None + project: Optional[str] = None + """Its project's `-` reference, or `None` for a built-in seed, which belongs to no project.""" @model_validator(mode="before") @classmethod diff --git a/datamasque/client/models/project.py b/datamasque/client/models/project.py index b264100..758314d 100644 --- a/datamasque/client/models/project.py +++ b/datamasque/client/models/project.py @@ -9,6 +9,20 @@ ProjectId = NewType("ProjectId", int) +def drop_namespace(cls: type, data: object) -> object: + """ + Drops the deprecated `namespace` field that the server still returns on libraries. + + The models allow extra fields, so a kept `namespace` would be sent back on update, + and the server lets `namespace` override `project`. + """ + + if isinstance(data, dict) and "namespace" in data: + data = {key: value for key, value in data.items() if key != "namespace"} + + return data + + class Project(BaseModel): """A project: the top-level grouping that connections, rulesets and other resources belong to.""" diff --git a/datamasque/client/models/rg_config.py b/datamasque/client/models/rg_config.py index 1403937..f5de6ac 100644 --- a/datamasque/client/models/rg_config.py +++ b/datamasque/client/models/rg_config.py @@ -39,12 +39,16 @@ class RGConfig(BaseModel): model_config = ConfigDict(extra="allow", populate_by_name=True) name: str + project: Optional[str] = None + """ + The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, + or the project's ID as a string. + Left as `None`, a create goes into the server's default project. + """ yaml: Optional[str] = Field(default=None, alias="config_yaml") # Server-populated read-only fields, excluded from request bodies. id: Optional[RGConfigId] = Field(default=None, exclude=True) - project: Optional[str] = Field(default=None, exclude=True) - """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True) validation_errors: list[ValidationErrorDetails] = Field(default_factory=list, exclude=True) created: Optional[datetime] = Field(default=None, exclude=True) diff --git a/datamasque/client/models/ruleset.py b/datamasque/client/models/ruleset.py index d14c2d4..2c6275d 100644 --- a/datamasque/client/models/ruleset.py +++ b/datamasque/client/models/ruleset.py @@ -38,13 +38,17 @@ class Ruleset(GitTrackedEntity): """Represents a ruleset.""" name: str + project: Optional[str] = None + """ + The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, + or the project's ID as a string. + Left as `None`, a create goes into the server's default project. + """ yaml: str = Field(default="", alias="config_yaml") ruleset_type: RulesetType = Field(default=RulesetType.database, alias="mask_type") # Server-populated read-only fields, excluded from request bodies. id: Optional[RulesetId] = Field(default=None, exclude=True) - project: Optional[str] = Field(default=None, exclude=True) - """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True) validation_errors: list[ValidationErrorDetails] = Field(default_factory=list, exclude=True) """Validation errors surfaced by the server; empty when valid.""" diff --git a/datamasque/client/models/ruleset_library.py b/datamasque/client/models/ruleset_library.py index 68929c2..d0b75d9 100644 --- a/datamasque/client/models/ruleset_library.py +++ b/datamasque/client/models/ruleset_library.py @@ -1,9 +1,10 @@ from datetime import datetime from typing import NewType, Optional -from pydantic import Field +from pydantic import Field, model_validator from datamasque.client.models.git import GitTrackedEntity +from datamasque.client.models.project import drop_namespace from datamasque.client.models.status import ValidationErrorDetails, ValidationStatus RulesetLibraryId = NewType("RulesetLibraryId", str) @@ -13,13 +14,16 @@ class RulesetLibrary(GitTrackedEntity): """Represents a ruleset library.""" name: str - namespace: str = "" + project: Optional[str] = None + """ + The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, + or the project's ID as a string. + Left as `None`, a create goes into the server's default project. + """ yaml: Optional[str] = Field(default=None, alias="config_yaml") # Server-populated read-only fields, excluded from request bodies. id: Optional[RulesetLibraryId] = Field(default=None, exclude=True) - project: Optional[str] = Field(default=None, exclude=True) - """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True) validation_errors: list[ValidationErrorDetails] = Field(default_factory=list, exclude=True) """Validation errors surfaced by the server; empty when valid.""" @@ -32,3 +36,5 @@ class RulesetLibrary(GitTrackedEntity): """ created: Optional[datetime] = Field(default=None, exclude=True) modified: Optional[datetime] = Field(default=None, exclude=True) + + _drop_namespace = model_validator(mode="before")(drop_namespace) diff --git a/datamasque/client/models/table_reference.py b/datamasque/client/models/table_reference.py index e03bb38..3bcac90 100644 --- a/datamasque/client/models/table_reference.py +++ b/datamasque/client/models/table_reference.py @@ -73,6 +73,12 @@ class TableReference(BaseModel): model_config = ConfigDict(extra="allow", populate_by_name=True) name: str + project: Optional[str] = None + """ + The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, + or the project's ID as a string. + Left as `None`, a create goes into the server's default project. + """ connection: Union[ConnectionId, ConnectionConfig] """The connection the data lives in. Accepts a `ConnectionId` or a `ConnectionConfig` (its `id` is extracted).""" source: str @@ -81,8 +87,6 @@ class TableReference(BaseModel): # Server-populated read-only fields, excluded from request bodies. id: Optional[TableReferenceId] = Field(default=None, exclude=True) - project: Optional[str] = Field(default=None, exclude=True) - """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" created: Optional[datetime] = Field(default=None, exclude=True) modified: Optional[datetime] = Field(default=None, exclude=True) diff --git a/datamasque/client/rg_configs.py b/datamasque/client/rg_configs.py index adb6066..4ba96fe 100644 --- a/datamasque/client/rg_configs.py +++ b/datamasque/client/rg_configs.py @@ -1,9 +1,10 @@ import logging -from typing import Iterator, Optional +from typing import Iterator, Optional, Union from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.pagination import Page +from datamasque.client.models.project import Project, ProjectId, unwrap_project_id from datamasque.client.models.rg_config import RGConfig, RGConfigId logger = logging.getLogger(__name__) @@ -33,13 +34,13 @@ def get_rg_config(self, config_id: RGConfigId) -> RGConfig: response = self.make_request("GET", f"/api/ruleset-generation-configs/{config_id}/") return RGConfig.model_validate(response.json()) - def _get_rg_config_id_by_name(self, name: str) -> Optional[RGConfigId]: - """Return the id of the config matching the name via a single list request, or `None`.""" + def _get_rg_config_id_by_name(self, name: str, project: Union[ProjectId, str]) -> Optional[RGConfigId]: + """Return the id of the config matching the name and project via a single list request, or `None`.""" response = self.make_request( "GET", "/api/ruleset-generation-configs/", - params={"name_exact": name, "limit": 1}, + params={"name_exact": name, "project": project, "limit": 1}, ) page = Page[RGConfig].model_validate(response.json()) if not page.results: @@ -54,15 +55,16 @@ def _get_rg_config_id_by_name(self, name: str) -> Optional[RGConfigId]: return config_id - def get_rg_config_by_name(self, name: str) -> Optional[RGConfig]: + def get_rg_config_by_name(self, name: str, project: Union[ProjectId, Project]) -> Optional[RGConfig]: """ - Looks for an RG config matching the given name (case-sensitive, exact match). + Looks for an RG config matching the given name (case-sensitive, exact match) in the given project. - RG configs are untyped and their names are unique, so the name alone identifies a single config. + RG configs are untyped and their names are unique per project. + `project` is a `Project` or its `ProjectId`. Returns it if found, otherwise `None`. """ - config_id = self._get_rg_config_id_by_name(name) + config_id = self._get_rg_config_id_by_name(name, unwrap_project_id(project)) if config_id is None: return None @@ -80,6 +82,7 @@ def create_rg_config(self, config: RGConfig) -> RGConfig: response = self.make_request("POST", "/api/ruleset-generation-configs/", data=data) created = RGConfig.model_validate(response.json()) config.id = created.id + config.project = created.project config.is_valid = created.is_valid config.validation_errors = created.validation_errors config.created = created.created @@ -116,12 +119,14 @@ def update_rg_config(self, config: RGConfig) -> RGConfig: def create_or_update_rg_config(self, config: RGConfig) -> RGConfig: """ - Creates the config if it doesn't exist, or updates it if one with the same name already exists. + Creates the config, or updates the existing one with the same name in the same project. + With no `project` set, the server's only active project is used, + and `DataMasqueException` is raised when it has no active project or more than one. Sets the config's `id` property. """ - existing_id = self._get_rg_config_id_by_name(config.name) + existing_id = self._get_rg_config_id_by_name(config.name, self._resolve_project(config.project)) if existing_id is not None: config.id = existing_id return self.update_rg_config(config) @@ -137,14 +142,15 @@ def delete_rg_config_by_id_if_exists(self, config_id: RGConfigId) -> None: self._delete_if_exists(f"/api/ruleset-generation-configs/{config_id}/") - def delete_rg_config_by_name_if_exists(self, name: str) -> None: + def delete_rg_config_by_name_if_exists(self, name: str, project: Union[ProjectId, Project]) -> None: """ - Deletes the RG config with the given name. + Deletes the RG config with the given name in the given project. + `project` is a `Project` or its `ProjectId`. No-op if no such config exists. """ - config_id = self._get_rg_config_id_by_name(name) + config_id = self._get_rg_config_id_by_name(name, unwrap_project_id(project)) if config_id is not None: self.delete_rg_config_by_id_if_exists(config_id) diff --git a/datamasque/client/ruleset_libraries.py b/datamasque/client/ruleset_libraries.py index ce5fddb..e03b6c2 100644 --- a/datamasque/client/ruleset_libraries.py +++ b/datamasque/client/ruleset_libraries.py @@ -1,9 +1,10 @@ import logging -from typing import Iterator, Optional +from typing import Iterator, Optional, Union from datamasque.client.base import BaseClient -from datamasque.client.exceptions import DataMasqueApiError, DataMasqueException +from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.pagination import Page +from datamasque.client.models.project import Project, ProjectId, unwrap_project_id from datamasque.client.models.ruleset import Ruleset from datamasque.client.models.ruleset_library import RulesetLibrary, RulesetLibraryId @@ -34,17 +35,13 @@ def get_ruleset_library(self, library_id: RulesetLibraryId) -> RulesetLibrary: response = self.make_request("GET", f"/api/ruleset-libraries/{library_id}/") return RulesetLibrary.model_validate(response.json()) - def get_ruleset_library_by_name(self, name: str, namespace: str = "") -> Optional[RulesetLibrary]: - """ - Looks for a ruleset library matching the given name and namespace (case-sensitive, exact match). - - Returns it (with full YAML content) if found, otherwise None. - """ + def _get_ruleset_library_id_by_name(self, name: str, project: Union[ProjectId, str]) -> Optional[RulesetLibraryId]: + """Return the ID of the library with the given name in the given project, or `None` if there is none.""" response = self.make_request( "GET", "/api/ruleset-libraries/", - params={"name_exact": name, "namespace_exact": namespace, "limit": 1}, + params={"name_exact": name, "project": project, "limit": 1}, ) page = Page[RulesetLibrary].model_validate(response.json()) if not page.results: @@ -57,6 +54,20 @@ def get_ruleset_library_by_name(self, name: str, namespace: str = "") -> Optiona response=response, ) + return library_id + + def get_ruleset_library_by_name(self, name: str, project: Union[ProjectId, Project]) -> Optional[RulesetLibrary]: + """ + Looks for a ruleset library matching the given name (case-sensitive, exact match) in the given project. + + `project` is a `Project` or its `ProjectId`. + Returns it (with full YAML content) if found, otherwise None. + """ + + library_id = self._get_ruleset_library_id_by_name(name, unwrap_project_id(project)) + if library_id is None: + return None + return self.get_ruleset_library(library_id) def create_ruleset_library(self, library: RulesetLibrary) -> RulesetLibrary: @@ -72,6 +83,7 @@ def create_ruleset_library(self, library: RulesetLibrary) -> RulesetLibrary: response = self.make_request("POST", "/api/ruleset-libraries/", data=data) created_library = RulesetLibrary.model_validate(response.json()) library.id = created_library.id + library.project = created_library.project library.is_valid = created_library.is_valid library.validation_errors = created_library.validation_errors library.unlicensed_feature_warnings = created_library.unlicensed_feature_warnings @@ -104,14 +116,16 @@ def update_ruleset_library(self, library: RulesetLibrary) -> RulesetLibrary: def create_or_update_ruleset_library(self, library: RulesetLibrary) -> RulesetLibrary: """ - Creates the library if it doesn't exist, or updates it if a library with the same name already exists. + Creates the library, or updates the existing one with the same name in the same project. + With no `project` set, the server's only active project is used, + and `DataMasqueException` is raised when it has no active project or more than one. Sets the library's `id` property. """ - existing = self.get_ruleset_library_by_name(library.name, library.namespace) - if existing is not None: - library.id = existing.id + library_id = self._get_ruleset_library_id_by_name(library.name, self._resolve_project(library.project)) + if library_id is not None: + library.id = library_id return self.update_ruleset_library(library) return self.create_ruleset_library(library) @@ -131,21 +145,18 @@ def delete_ruleset_library_by_id_if_exists(self, library_id: RulesetLibraryId, * self._delete_if_exists(f"/api/ruleset-libraries/{library_id}/", params=params) def delete_ruleset_library_by_name_if_exists( - self, library_name: str, namespace: str = "", *, force: bool = False + self, library_name: str, project: Union[ProjectId, Project], *, force: bool = False ) -> None: """ - Deletes the ruleset library with the given name and namespace. + Deletes the ruleset library with the given name in the given project. + `project` is a `Project` or its `ProjectId`. No-op if the library does not exist. """ - all_libraries = self.list_ruleset_libraries() - matching = [lib for lib in all_libraries if lib.name == library_name and lib.namespace == namespace] - for lib in matching: - if lib.id is None: - raise DataMasqueException(f'Server returned a ruleset library named "{lib.name}" without an `id`.') - - self.delete_ruleset_library_by_id_if_exists(lib.id, force=force) + library_id = self._get_ruleset_library_id_by_name(library_name, unwrap_project_id(project)) + if library_id is not None: + self.delete_ruleset_library_by_id_if_exists(library_id, force=force) def iter_rulesets_using_library(self, library_id: RulesetLibraryId) -> Iterator[Ruleset]: """Lazily iterate rulesets that import the given library.""" diff --git a/datamasque/client/rulesets.py b/datamasque/client/rulesets.py index 96fc98c..9a5159c 100644 --- a/datamasque/client/rulesets.py +++ b/datamasque/client/rulesets.py @@ -1,7 +1,9 @@ import logging +from typing import Union from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueException +from datamasque.client.models.project import Project, ProjectId, unwrap_project_id from datamasque.client.models.ruleset import Ruleset, RulesetId, RulesetType logger = logging.getLogger(__name__) @@ -18,16 +20,21 @@ def list_rulesets(self) -> list[Ruleset]: def create_or_update_ruleset(self, ruleset: Ruleset) -> Ruleset: """ - Creates or updates a ruleset. + Creates the ruleset, or updates the existing one with the same name and type in the same project. + With no `project` set, the server's only active project is used, + and `DataMasqueException` is raised when it has no active project or more than one. Populates the given ruleset's `id`, `is_valid`, `validation_errors`, `unlicensed_feature_warnings`, and `git` fields from the server response, and returns the same ruleset instance for convenience. """ + # The server finds the existing ruleset itself, so this only checks that an unset project resolves. + self._resolve_project(ruleset.project) data = ruleset.model_dump(exclude_none=True, by_alias=True, mode="json") response = self.make_request("POST", "/api/rulesets/", data=data, params={"upsert": "true"}) created = Ruleset.model_validate(response.json()) ruleset.id = created.id + ruleset.project = created.project ruleset.is_valid = created.is_valid ruleset.validation_errors = created.validation_errors ruleset.unlicensed_feature_warnings = created.unlicensed_feature_warnings @@ -45,20 +52,23 @@ def delete_ruleset_by_id_if_exists(self, ruleset_id: RulesetId) -> None: self._delete_if_exists(f"/api/rulesets/{ruleset_id}/") - def delete_ruleset_by_name_if_exists(self, ruleset_name: str, ruleset_type: RulesetType) -> None: + def delete_ruleset_by_name_if_exists( + self, ruleset_name: str, ruleset_type: RulesetType, project: Union[ProjectId, Project] + ) -> None: """ - Deletes the ruleset with the given name and type. + Deletes the ruleset with the given name and type in the given project. - Ruleset names are unique per type, so a type is required to identify a single ruleset. + Ruleset names are unique per type and project, so both are required to identify a single ruleset. + `project` is a `Project` or its `ProjectId`. No-op if no such ruleset exists. """ - matching = [ - ruleset - for ruleset in self.list_rulesets() - if ruleset.name == ruleset_name and ruleset.ruleset_type is ruleset_type - ] - for ruleset in matching: + response = self.make_request( + "GET", + "/api/v2/rulesets/", + params={"name_exact": ruleset_name, "mask_type": ruleset_type.value, "project": unwrap_project_id(project)}, + ) + for ruleset in (Ruleset.model_validate(payload) for payload in response.json()): if ruleset.id is None: raise DataMasqueException(f'Server returned a ruleset named "{ruleset.name}" without an `id`.') diff --git a/datamasque/client/table_references.py b/datamasque/client/table_references.py index f191abf..a3c6ff2 100644 --- a/datamasque/client/table_references.py +++ b/datamasque/client/table_references.py @@ -1,8 +1,9 @@ import logging -from typing import Optional +from typing import Optional, Union from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueApiError +from datamasque.client.models.project import Project, ProjectId, unwrap_project_id from datamasque.client.models.table_reference import TableReference, TableReferenceId logger = logging.getLogger(__name__) @@ -11,16 +12,19 @@ class TableReferenceClient(BaseClient): """Table reference CRUD API methods. Mixed into `DataMasqueClient`.""" - def list_table_references(self) -> list[TableReference]: + def list_table_references(self, project: Optional[Union[ProjectId, Project]] = None) -> list[TableReference]: """ - Lists all table references. + Lists all table references, or only those in `project` when it is given. + + `project` is a `Project` or its `ProjectId`. Unlike most list endpoints, this one is not paginated; the server returns every table reference in a single response. Deleted table references are archived rather than removed, and are not listed. """ - response = self.make_request("GET", "/api/table-references/") + params = {"project": unwrap_project_id(project)} if project is not None else None + response = self.make_request("GET", "/api/table-references/", params=params) return [TableReference.model_validate(item) for item in response.json()] def get_table_reference(self, table_reference_id: TableReferenceId) -> TableReference: @@ -29,26 +33,27 @@ def get_table_reference(self, table_reference_id: TableReferenceId) -> TableRefe response = self.make_request("GET", f"/api/table-references/{table_reference_id}/") return TableReference.model_validate(response.json()) - def get_table_reference_by_name(self, name: str) -> Optional[TableReference]: + def get_table_reference_by_name(self, name: str, project: Union[ProjectId, Project]) -> Optional[TableReference]: """ - Looks for a table reference with the given name (case-sensitive, exact match). + Looks for a table reference with the given name (case-sensitive, exact match) in the given project. - Names are unique across live table references, so at most one can match. + Names are unique across a project's live table references, so at most one can match. + `project` is a `Project` or its `ProjectId`. Returns it if found, otherwise `None`. - The endpoint takes no name filter, so the match is made client-side over the full listing. + The endpoint takes no name filter, so the match is made client-side over the project's listing. """ - matches = [reference for reference in self.list_table_references() if reference.name == name] + matches = [reference for reference in self.list_table_references(project) if reference.name == name] if not matches: return None return matches[0] - def _get_table_reference_id_by_name(self, name: str) -> Optional[TableReferenceId]: - """Return the ID of the table reference with the given name, or `None` if there is none.""" + def _get_table_reference_id_by_name(self, name: str, project: Union[ProjectId, str]) -> Optional[TableReferenceId]: + """Return the ID of the table reference with the given name in the given project, or `None` if there is none.""" - response = self.make_request("GET", "/api/table-references/") + response = self.make_request("GET", "/api/table-references/", params={"project": project}) matches = [item for item in response.json() if item.get("name") == name] if not matches: return None @@ -74,9 +79,12 @@ def create_table_reference(self, table_reference: TableReference) -> TableRefere data = table_reference.model_dump(by_alias=True, mode="json") if data.get("options") is None: data.pop("options", None) + if data.get("project") is None: + data.pop("project", None) response = self.make_request("POST", "/api/table-references/", data=data) created = TableReference.model_validate(response.json()) table_reference.id = created.id + table_reference.project = created.project table_reference.options = created.options table_reference.created = created.created table_reference.modified = created.modified @@ -101,6 +109,8 @@ def update_table_reference(self, table_reference: TableReference) -> TableRefere data = table_reference.model_dump(by_alias=True, mode="json") if data.get("options") is None: data.pop("options", None) + if data.get("project") is None: + data.pop("project", None) response = self.make_request("PUT", f"/api/table-references/{table_reference.id}/", data=data) updated = TableReference.model_validate(response.json()) table_reference.options = updated.options @@ -110,12 +120,16 @@ def update_table_reference(self, table_reference: TableReference) -> TableRefere def create_or_update_table_reference(self, table_reference: TableReference) -> TableReference: """ - Creates the table reference, or updates the existing one with the same name. + Creates the table reference, or updates the existing one with the same name in the same project. + With no `project` set, the server's only active project is used, + and `DataMasqueException` is raised when it has no active project or more than one. Sets the table reference's `id` property. """ - existing_id = self._get_table_reference_id_by_name(table_reference.name) + existing_id = self._get_table_reference_id_by_name( + table_reference.name, self._resolve_project(table_reference.project) + ) if existing_id is not None: table_reference.id = existing_id return self.update_table_reference(table_reference) @@ -132,13 +146,14 @@ def delete_table_reference_by_id_if_exists(self, table_reference_id: TableRefere self._delete_if_exists(f"/api/table-references/{table_reference_id}/") - def delete_table_reference_by_name_if_exists(self, name: str) -> None: + def delete_table_reference_by_name_if_exists(self, name: str, project: Union[ProjectId, Project]) -> None: """ - Deletes the table reference with the given name. + Deletes the table reference with the given name in the given project. + `project` is a `Project` or its `ProjectId`. No-op if no such table reference exists. """ - table_reference_id = self._get_table_reference_id_by_name(name) + table_reference_id = self._get_table_reference_id_by_name(name, unwrap_project_id(project)) if table_reference_id is not None: self.delete_table_reference_by_id_if_exists(table_reference_id) diff --git a/tests/conftest.py b/tests/conftest.py index 317778a..1052be1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,7 @@ from datamasque.client import DataMasqueClient from datamasque.client.models.dm_instance import DataMasqueInstanceConfig from datamasque.client.models.ruleset import Ruleset, RulesetType -from tests.helpers import database_connection_config, file_connection_config +from tests.helpers import PROJECT, database_connection_config, file_connection_config @pytest.fixture @@ -36,6 +36,7 @@ def existing_connection_json(): return { "id": "1", "name": "an_existing_connection", + "project": PROJECT, "mask_type": "database", "db_type": "mysql", "host": "my-host", @@ -51,6 +52,7 @@ def existing_rulesets_json(): { "id": "1", "name": "db_masking_ruleset", + "project": PROJECT, "mask_type": "database", "config_yaml": "version: '1.0'", "is_valid": "valid", @@ -58,6 +60,7 @@ def existing_rulesets_json(): { "id": "2", "name": "file_masking_ruleset", + "project": PROJECT, "mask_type": "file", "config_yaml": "version: '1.0'", "is_valid": "invalid", @@ -69,6 +72,7 @@ def existing_rulesets_json(): def ruleset(): return Ruleset( name="test_ruleset", + project=PROJECT, yaml="version: '1.0'\ntasks: []", ruleset_type=RulesetType.database, ) diff --git a/tests/helpers.py b/tests/helpers.py index 2303cfc..5c21f4e 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -9,9 +9,41 @@ S3ConnectionConfig, SnowflakeConnectionConfig, ) +from datamasque.client.models.project import Project, ProjectId fake = Faker() +PROJECT = "1-default" +"""The project reference the tests put resources in.""" + +PROJECT_ID = ProjectId(1) +"""The ID of `PROJECT`, as the client methods take it.""" + +OTHER_PROJECT = Project.model_validate( + { + "id": 2, + "reference": "2-banking", + "name": "Banking", + "description": "", + "created": "2026-10-01T10:00:00Z", + "updated": "2026-10-01T10:00:00Z", + } +) +"""A second project, for tests that pass a `Project` to a client method.""" + + +def active_projects_page(count: int) -> dict[str, object]: + """Builds the `GET /api/projects/?limit=1` response of a server with `count` active projects, `PROJECT` first.""" + project = { + "id": PROJECT_ID, + "reference": PROJECT, + "name": "Default", + "description": "", + "created": "2026-10-01T10:00:00Z", + "updated": "2026-10-01T10:00:00Z", + } + return {"count": count, "next": None, "previous": None, "results": [project] if count else []} + def parse_multipart_form(request) -> dict: # noqa: C901 """ @@ -69,6 +101,7 @@ def parse_multipart_form(request) -> dict: # noqa: C901 def database_connection_config(): return DatabaseConnectionConfig( + project=PROJECT, name=fake.word(), user=fake.user_name(), password=fake.password(), @@ -82,6 +115,7 @@ def database_connection_config(): def file_connection_config(): return S3ConnectionConfig( + project=PROJECT, name=fake.word(), base_directory=fake.uri_path(), bucket=fake.uri_page(), @@ -94,6 +128,7 @@ def sample_mounted_share_connection_json(*, id, name): return { "name": name, "id": id, + "project": PROJECT, "mask_type": "file", "type": "mounted_share_connection", "base_directory": "", @@ -118,6 +153,7 @@ def make_ok_response() -> Response: def snowflake_connection_config(): return SnowflakeConnectionConfig( + project=PROJECT, name="snowflake", database="test_db", user="snowflake_user", diff --git a/tests/test_connections.py b/tests/test_connections.py index cbc3f7d..9f22271 100644 --- a/tests/test_connections.py +++ b/tests/test_connections.py @@ -1,10 +1,12 @@ """Tests for `ConnectionClient` (CRUD + Snowflake-specific behaviour).""" from datetime import datetime, timezone +from typing import Union import pytest import requests_mock +from datamasque.client import DataMasqueClient, Project, ProjectId from datamasque.client.exceptions import DataMasqueApiError, DataMasqueException from datamasque.client.models.connection import ( AzureConnectionConfig, @@ -31,6 +33,9 @@ validate_connection, ) from tests.helpers import ( + OTHER_PROJECT, + PROJECT, + PROJECT_ID, sample_mounted_share_connection_json, snowflake_connection_config, ) @@ -96,6 +101,69 @@ def test_create_or_update_connection_create_fail(client, connection_config, exis client.create_or_update_connection(connection_config) +def test_create_or_update_connection_sends_the_project( + client: DataMasqueClient, connection_config: ConnectionConfig +) -> None: + """The lookup for an existing connection is filtered to the connection's project, and the body carries it.""" + with requests_mock.Mocker() as m: + m.get("http://test-server/api/connections/", json=[], status_code=200) + m.post("http://test-server/api/connections/", json={"id": "2"}, status_code=201) + + client.create_or_update_connection(connection_config) + + assert m.request_history[0].qs == {"project": [PROJECT]} + assert m.last_request.json()["project"] == PROJECT + + +def test_create_or_update_connection_ignores_same_name_in_another_project( + client: DataMasqueClient, connection_config: ConnectionConfig +) -> None: + """A same-named connection in another project is not updated, because the listing is filtered by project.""" + other_project_connection = sample_mounted_share_connection_json(name=connection_config.name, id="1") | { + "project": "2-banking" + } + with requests_mock.Mocker() as m: + m.get("http://test-server/api/connections/", json=[other_project_connection], status_code=200) + m.get( + f"http://test-server/api/connections/?project={PROJECT}", + json=[], + status_code=200, + complete_qs=True, + ) + m.post("http://test-server/api/connections/", json={"id": "2"}, status_code=201) + + result = client.create_or_update_connection(connection_config) + + assert result.id == ConnectionId("2") + assert [request.method for request in m.request_history] == ["GET", "POST"] + + +@pytest.mark.parametrize("project", [ProjectId(2), OTHER_PROJECT]) +def test_list_connections_filters_by_project(client: DataMasqueClient, project: Union[ProjectId, Project]) -> None: + with requests_mock.Mocker() as m: + m.get("http://test-server/api/connections/", json=[], status_code=200) + client.list_connections(project) + + assert m.last_request.qs == {"project": ["2"]} + + +def test_list_connections_keeps_same_name_in_different_projects(client: DataMasqueClient) -> None: + """Connection names are unique per project, so an unfiltered listing can hold two connections with one name.""" + connections = [ + sample_mounted_share_connection_json(name="shared", id="1"), + sample_mounted_share_connection_json(name="shared", id="2") | {"project": OTHER_PROJECT.reference}, + ] + with requests_mock.Mocker() as m: + m.get("http://test-server/api/connections/", json=connections, status_code=200) + result = client.list_connections() + + assert m.last_request.qs == {} + assert [(connection.id, connection.project) for connection in result] == [ + ("1", PROJECT), + ("2", OTHER_PROJECT.reference), + ] + + def test_list_connnections(client): with requests_mock.Mocker() as m: m.get( @@ -104,6 +172,7 @@ def test_list_connnections(client): # Realistic API data, hence why it's so verbose { "name": "s3", + "project": PROJECT, "bucket": "my-s3-bucket", "base_directory": "", "type": "s3_connection", @@ -117,6 +186,7 @@ def test_list_connnections(client): }, { "name": "azure", + "project": PROJECT, "type": "azure_blob_connection", "base_directory": "", "mask_type": "file", @@ -131,6 +201,7 @@ def test_list_connnections(client): }, { "name": "mounted_share_dest", + "project": PROJECT, "type": "mounted_share_connection", "base_directory": "dest", "mask_type": "file", @@ -149,6 +220,7 @@ def test_list_connnections(client): "db_type": "mysql", "database": "mydatabase", "name": "mysql", + "project": PROJECT, "schema": "", "is_read_only": False, "password_encrypted": "some_base64_here", @@ -163,6 +235,7 @@ def test_list_connnections(client): "version": "1.0", "mask_type": "database", "name": "db_dynamo", + "project": PROJECT, "s3_bucket_name": "my-dynamo-staging-bucket", "dynamo_append_datetime": False, "dynamo_append_suffix": "-masked", @@ -193,6 +266,7 @@ def test_list_connnections(client): "db_type": "mssql_linked", "database": "database_name", "name": "mssql-linked", + "project": PROJECT, "schema": "", "is_read_only": False, "password_encrypted": "some_base64_here", @@ -208,6 +282,7 @@ def test_list_connnections(client): "version": "1.0", "mask_type": "database", "name": "db_dynamo_2", + "project": PROJECT, "s3_bucket_name": "my-dynamo-staging-bucket-2", "dynamo_append_datetime": False, "dynamo_append_suffix": "-masked", @@ -232,6 +307,7 @@ def test_list_connnections(client): "db_type": "snowflake", "database": "icicle", "name": "snowflake", + "project": PROJECT, "schema": "snowball", "snowflake_role": "snowballs do indeed roll", "snowflake_account_id": "ABCDEF-123456", @@ -253,6 +329,7 @@ def test_list_connnections(client): "db_type": "snowflake", "database": "igloo", "name": "snowflake_minimal_with_key", + "project": PROJECT, "snowflake_account_id": "ACCOUNT-1234", "snowflake_warehouse": "clothing_store", "snowflake_private_key": "2831289a-4398-abcd-4112-fe09a1239f89", @@ -381,10 +458,11 @@ def test_delete_connection_by_name(client): status_code=200, ) - client.delete_connection_by_name_if_exists(connection_name) + client.delete_connection_by_name_if_exists(connection_name, PROJECT_ID) assert m.call_count == 3 assert m.request_history[0].method == "GET" + assert m.request_history[0].qs == {"project": [str(PROJECT_ID)]} assert m.request_history[1].method == "DELETE" assert m.request_history[2].method == "DELETE" @@ -405,7 +483,7 @@ def test_delete_connection_that_does_not_exist(client): ], status_code=200, ) - client.delete_connection_by_name_if_exists("my_connection") + client.delete_connection_by_name_if_exists("my_connection", PROJECT_ID) assert m.call_count == 1 assert m.request_history[0].method == "GET" @@ -467,6 +545,7 @@ def test_list_snowflake_connection_from_older_server_drops_external_stage(client "db_type": "snowflake", "database": "s3_db", "name": "snowflake_s3", + "project": PROJECT, "snowflake_account_id": "S3-ACCOUNT", "snowflake_warehouse": "s3_warehouse", "snowflake_storage_integration_name": "s3_integration", @@ -494,6 +573,7 @@ def test_s3_connection_model_validate(): payload = { "id": "88dabb63-aca5-4cc4-8f76-f78736a42f39", "name": "s3", + "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "data/", @@ -519,6 +599,7 @@ def test_connection_model_validate_parses_license_lock() -> None: payload = { "id": "88dabb63-aca5-4cc4-8f76-f78736a42f39", "name": "s3", + "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "data/", @@ -542,6 +623,7 @@ def test_connection_model_validate_null_license_lock() -> None: payload = { "id": "id-1", "name": "s3", + "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "", @@ -559,6 +641,7 @@ def test_connection_model_validate_parses_freeze_fields() -> None: payload = { "id": "db-1", "name": "pg", + "project": PROJECT, "mask_type": "database", "db_type": "postgres", "host": "h", @@ -582,6 +665,7 @@ def test_connection_freeze_fields_default_when_absent() -> None: payload = { "id": "db-1", "name": "pg", + "project": PROJECT, "mask_type": "database", "db_type": "postgres", "host": "h", @@ -610,6 +694,7 @@ def test_connection_freeze_fields_default_when_absent() -> None: { "id": "id-1", "name": "s3", + "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "", @@ -624,6 +709,7 @@ def test_connection_freeze_fields_default_when_absent() -> None: { "id": "db-1", "name": "pg", + "project": PROJECT, "mask_type": "database", "db_type": "postgres", "host": "h", @@ -639,6 +725,7 @@ def test_connection_freeze_fields_default_when_absent() -> None: { "id": "dyn-1", "name": "dyn", + "project": PROJECT, "mask_type": "database", "db_type": "dynamo_db", "s3_bucket_name": "bucket", @@ -653,6 +740,7 @@ def test_connection_freeze_fields_default_when_absent() -> None: { "id": "mongo-1", "name": "mongo", + "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -671,6 +759,7 @@ def test_connection_freeze_fields_default_when_absent() -> None: { "id": "sf-1", "name": "snowflake", + "project": PROJECT, "mask_type": "database", "db_type": "snowflake", "user": "snowman", @@ -713,6 +802,7 @@ def test_s3_connection_model_validate_no_iam_role(): payload = { "id": "id-1", "name": "s3", + "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "", @@ -729,6 +819,7 @@ def test_databricks_connection_model_validate(): payload = { "id": "db-id-1", "name": "databricks", + "project": PROJECT, "mask_type": "database", "db_type": "databricks", "server_hostname": "adb-1234.azuredatabricks.net", @@ -755,6 +846,7 @@ def test_databricks_connection_model_validate_blanks_encrypted_token(): payload = { "id": "db-id-2", "name": "databricks", + "project": PROJECT, "mask_type": "database", "db_type": "databricks", "server_hostname": "adb-1234.azuredatabricks.net", @@ -774,6 +866,7 @@ def test_validate_connection_dispatches_databricks(): payload = { "id": "db-id-3", "name": "databricks", + "project": PROJECT, "mask_type": "database", "db_type": "databricks", "server_hostname": "adb-1234.azuredatabricks.net", @@ -792,6 +885,7 @@ def test_azure_connection_model_validate_blanks_encrypted_connection_string(): payload = { "id": "490502e5-5bf6-4abb-b67b-c6091d40ecf0", "name": "azure", + "project": PROJECT, "mask_type": "file", "type": "azure_blob_connection", "base_directory": "", @@ -814,6 +908,7 @@ def test_gcs_connection_model_validate_blanks_encrypted_service_account_key(): payload = { "id": "0f9f5d2e-8d1a-4a1f-9f57-3f2c1b0c9a11", "name": "gcs", + "project": PROJECT, "mask_type": "file", "type": "gcs_connection", "base_directory": "inbound", @@ -835,6 +930,7 @@ def test_gcs_connection_model_validate_blanks_encrypted_service_account_key(): def test_gcs_connection_serializes_the_key_for_create(): conn = GcsConnectionConfig( + project=PROJECT, name="gcs", bucket="my-gcs-bucket", service_account_key='{"type": "service_account"}', @@ -864,6 +960,7 @@ def test_database_connection_model_validate_drops_schema_for_mysql(): payload = { "id": "f0557fb3-1c9a-4cb1-bcf4-9699cf496bf7", "name": "mysql", + "project": PROJECT, "mask_type": "database", "db_type": "mysql", "host": "my-mysql-host", @@ -886,6 +983,7 @@ def test_database_connection_model_validate_keeps_schema_for_postgres(): payload = { "id": "abc", "name": "pg", + "project": PROJECT, "mask_type": "database", "db_type": "postgres", "host": "pg-host", @@ -904,6 +1002,7 @@ def test_database_connection_model_validate_databricks_lakebase(): payload = { "id": "abc-lakebase", "name": "lakebase", + "project": PROJECT, "mask_type": "database", "db_type": "databricks_lakebase", "host": "lakebase-host", @@ -925,6 +1024,7 @@ def test_mssql_linked_connection_model_validate_includes_linked_server(): payload = { "id": "48a7af45-f63f-4e05-bf9f-7b1cc3a0e89d", "name": "mssql-linked", + "project": PROJECT, "mask_type": "database", "db_type": "mssql_linked", "host": "mssql-linked-host", @@ -947,6 +1047,7 @@ def test_dynamo_connection_model_validate_with_sse(): payload = { "id": "d7257552-0485-4806-b0fb-d72b4d268073", "name": "db_dynamo", + "project": PROJECT, "mask_type": "database", "db_type": "dynamo_db", "s3_bucket_name": "my-dynamo-staging-bucket", @@ -975,6 +1076,7 @@ def test_dynamo_connection_model_validate_without_sse_uses_default(): payload = { "id": "id-2", "name": "db_dynamo", + "project": PROJECT, "mask_type": "database", "db_type": "dynamo_db", "s3_bucket_name": "bucket", @@ -996,6 +1098,7 @@ def test_snowflake_connection_model_validate_drops_external_stage_location(stage payload = { "id": "f0557fb3-1c9a-4cb1-bcf4-9699cf496bf7", "name": "snowflake", + "project": PROJECT, "mask_type": "database", "db_type": "snowflake", "user": "snowman", @@ -1029,6 +1132,7 @@ def test_snowflake_connection_model_validate_minimal_payload(): payload = { "id": "id-3", "name": "snowflake", + "project": PROJECT, "mask_type": "database", "db_type": "snowflake", "user": "frosty", @@ -1049,6 +1153,7 @@ def test_connection_config_dispatch_picks_subclass(): s3_payload = { "id": "id-s3", "name": "s3", + "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "", @@ -1059,6 +1164,7 @@ def test_connection_config_dispatch_picks_subclass(): db_payload = { "id": "id-pg", "name": "pg", + "project": PROJECT, "mask_type": "database", "db_type": "postgres", "host": "h", @@ -1088,6 +1194,7 @@ def test_dynamo_connection_round_trip_with_iam_role_and_prefix(): payload = { "id": "id-dynamo-1", "name": "db_dynamo", + "project": PROJECT, "mask_type": "database", "db_type": "dynamo_db", "s3_bucket_name": "staging-bucket", @@ -1111,6 +1218,7 @@ def test_dynamo_connection_round_trip_with_iam_role_and_prefix(): def test_dynamo_connection_model_dump_omits_unset_iam_role_and_prefix(): """When the new optional fields are unset, `model_dump` must omit them entirely (not send `null`).""" conn = DynamoConnectionConfig( + project=PROJECT, name="db_dynamo", s3_bucket_name="bucket", dynamo_append_datetime=False, @@ -1129,6 +1237,7 @@ def test_dynamo_model_validate_defaults_to_none_when_fields_absent(): payload = { "id": "id-dynamo-2", "name": "db_dynamo", + "project": PROJECT, "mask_type": "database", "db_type": "dynamo_db", "s3_bucket_name": "bucket", @@ -1146,6 +1255,7 @@ def test_dynamo_model_validate_defaults_to_none_when_fields_absent(): def test_mongo_connection_model_dump_minimal(): """An unauthenticated, plain TCP connection sends only the required keys plus the defaulted booleans.""" conn = MongoConnectionConfig( + project=PROJECT, name="mongo", host="mongo.example", database="people", @@ -1164,6 +1274,7 @@ def test_mongo_connection_model_dump_minimal(): def test_mongo_connection_model_dump_full(): conn = MongoConnectionConfig( + project=PROJECT, name="mongo", host="mongo.example", port=27018, @@ -1195,6 +1306,7 @@ def test_mongo_connection_model_dump_full(): def test_mongo_connection_model_dump_omits_falsy_optional_flags(): """`tls`, `direct_connection`, `user`, `password`, and `replica_set` are only sent when truthy.""" conn = MongoConnectionConfig( + project=PROJECT, name="mongo", host="mongo.example", database="people", @@ -1212,7 +1324,7 @@ def test_mongo_connection_model_dump_omits_falsy_optional_flags(): def test_mongo_connection_retry_writes_defaults_true_and_is_omitted(): """retry_writes defaults to True (matching the server default), so it is left out of the payload.""" - conn = MongoConnectionConfig(name="mongo", host="mongo.example", database="people") + conn = MongoConnectionConfig(project=PROJECT, name="mongo", host="mongo.example", database="people") assert conn.retry_writes is True api_dict = conn.model_dump(exclude_none=True, by_alias=True, mode="json") assert "retry_writes" not in api_dict @@ -1220,14 +1332,18 @@ def test_mongo_connection_retry_writes_defaults_true_and_is_omitted(): def test_mongo_connection_sends_retry_writes_when_disabled(): """retry_writes is sent only when explicitly disabled (e.g. AWS DocumentDB rejects retryable writes).""" - conn = MongoConnectionConfig(name="mongo", host="mongo.example", database="people", retry_writes=False) + conn = MongoConnectionConfig( + project=PROJECT, name="mongo", host="mongo.example", database="people", retry_writes=False + ) api_dict = conn.model_dump(exclude_none=True, by_alias=True, mode="json") assert api_dict["retry_writes"] is False def test_documentdb_connection_sends_retry_writes_when_disabled(): """A DocumentDB connection inherits retry_writes and sends it when disabled.""" - conn = DocumentDbConnectionConfig(name="docdb", host="docdb.example", database="people", retry_writes=False) + conn = DocumentDbConnectionConfig( + project=PROJECT, name="docdb", host="docdb.example", database="people", retry_writes=False + ) api_dict = conn.model_dump(exclude_none=True, by_alias=True, mode="json") assert api_dict["db_type"] == "documentdb" assert api_dict["retry_writes"] is False @@ -1237,6 +1353,7 @@ def test_mongo_connection_model_validate_blanks_encrypted_password(): payload = { "id": "mongo-id-1", "name": "mongo", + "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -1268,6 +1385,7 @@ def test_mongo_connection_model_validate_defaults_when_optional_fields_missing() payload = { "id": "mongo-id-2", "name": "mongo-min", + "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -1288,6 +1406,7 @@ def test_connection_config_dispatch_picks_mongo_subclass(): payload = { "id": "mongo-id-3", "name": "mongo", + "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -1298,6 +1417,7 @@ def test_connection_config_dispatch_picks_mongo_subclass(): def test_mongo_tls_ca_fields_emitted_when_tls_enabled(): conn = MongoConnectionConfig( + project=PROJECT, name="mongo", host="mongo.example", database="people", @@ -1312,7 +1432,7 @@ def test_mongo_tls_ca_fields_emitted_when_tls_enabled(): def test_mongo_tls_ca_fields_omitted_when_unset_but_tls_enabled(): - conn = MongoConnectionConfig(name="mongo", host="mongo.example", database="people", tls=True) + conn = MongoConnectionConfig(project=PROJECT, name="mongo", host="mongo.example", database="people", tls=True) d = conn.model_dump(exclude_none=True, by_alias=True, mode="json") assert d["tls"] is True assert "tls_ca_file" not in d @@ -1322,6 +1442,7 @@ def test_mongo_tls_ca_fields_omitted_when_unset_but_tls_enabled(): def test_mongo_tls_ca_fields_omitted_when_tls_disabled(): """Even when set, the TLS-dependent fields are not sent while TLS is disabled.""" conn = MongoConnectionConfig( + project=PROJECT, name="mongo", host="mongo.example", database="people", @@ -1338,6 +1459,7 @@ def test_mongo_tls_ca_fields_roundtrip(): payload = { "id": "mongo-tls-1", "name": "mongo", + "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -1356,6 +1478,7 @@ def test_mongo_tls_ca_fields_default_when_missing(): payload = { "id": "mongo-tls-2", "name": "mongo", + "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -1368,6 +1491,7 @@ def test_mongo_tls_ca_fields_default_when_missing(): def test_documentdb_connection_model_dump(): conn = DocumentDbConnectionConfig( + project=PROJECT, name="docdb", host="dtq-documentdb.cluster.example", database="people", @@ -1391,6 +1515,7 @@ def test_connection_config_dispatch_picks_documentdb_subclass(): payload = { "id": "docdb-id-1", "name": "docdb", + "project": PROJECT, "mask_type": "database", "db_type": "documentdb", "host": "dtq-documentdb.cluster.example", @@ -1404,6 +1529,7 @@ def test_connection_config_dispatch_picks_documentdb_subclass(): def _cosmosdb_connection(**overrides) -> CosmosDbConnectionConfig: """Build a Cosmos DB config, leaving the port to the connection type's own default.""" return CosmosDbConnectionConfig( + project=PROJECT, name="cosmos", host="dtq-cosmos.mongo.cosmos.azure.com", database="people", @@ -1449,6 +1575,7 @@ def test_connection_config_dispatch_picks_cosmosdb_subclass(): payload = { "id": "cosmos-id-1", "name": "cosmos", + "project": PROJECT, "mask_type": "database", "db_type": "cosmosdb", "host": "dtq-cosmos.mongo.cosmos.azure.com", @@ -1473,6 +1600,7 @@ def test_database_connection_config_rejects_document_store_database_types( """`DatabaseConnectionConfig` is for SQL engines; each document store has its own class.""" with pytest.raises(ValueError, match=message): DatabaseConnectionConfig( + project=PROJECT, name="doc-store", host="doc-store.example", port=27017, @@ -1486,6 +1614,7 @@ def test_database_connection_config_rejects_document_store_database_types( def test_create_or_update_mongo_connection(client): """End-to-end: a Mongo connection round-trips through `create_or_update_connection`.""" conn = MongoConnectionConfig( + project=PROJECT, name="mongo", host="mongo.example", database="people", @@ -1513,6 +1642,7 @@ def test_create_or_update_mongo_connection(client): def _database_connection(database_type: DatabaseType, **extra) -> DatabaseConnectionConfig: return DatabaseConnectionConfig( + project=PROJECT, name="conn", host="mydb.abc123.ap-southeast-2.rds.amazonaws.com", port=5432, @@ -1607,6 +1737,7 @@ def test_database_connection_config_rejects_engines_with_their_own_class( ) -> None: with pytest.raises(ValueError, match=message): DatabaseConnectionConfig( + project=PROJECT, name="own-class", host="db.example", port=1, @@ -1619,6 +1750,7 @@ def test_database_connection_config_rejects_engines_with_their_own_class( def test_cassandra_connection_serializes_password_as_dbpassword(): conn = CassandraConnectionConfig( + project=PROJECT, name="cassandra", host="cassandra.example", database="people", @@ -1643,9 +1775,9 @@ def test_cassandra_connection_serializes_password_as_dbpassword(): def test_cassandra_connection_without_credentials_omits_user_and_password(): """Cassandra allows unauthenticated access; the server rejects a username sent without a password.""" - d = CassandraConnectionConfig(name="cassandra", host="cassandra.example", database="people").model_dump( - exclude_none=True, by_alias=True, mode="json" - ) + d = CassandraConnectionConfig( + project=PROJECT, name="cassandra", host="cassandra.example", database="people" + ).model_dump(exclude_none=True, by_alias=True, mode="json") assert "user" not in d assert "dbpassword" not in d @@ -1655,6 +1787,7 @@ def test_connection_config_dispatch_picks_cassandra_subclass(): payload = { "id": "cassandra-id-1", "name": "cassandra", + "project": PROJECT, "mask_type": "database", "db_type": "cassandra", "host": "cassandra.example", @@ -1678,6 +1811,7 @@ def test_connection_config_dispatch_picks_cassandra_subclass(): def test_sap_ase_connection_serializes_without_a_schema(): """ASE has no schema setting: the schema is the table owner.""" conn = SapAseConnectionConfig( + project=PROJECT, name="sap_ase", host="ase.example", database="people", @@ -1703,9 +1837,9 @@ def test_sap_ase_connection_serializes_without_a_schema(): def test_sap_ase_connection_omits_unset_tls_options(): - d = SapAseConnectionConfig(name="sap_ase", host="ase.example", database="people", user="sa").model_dump( - exclude_none=True, by_alias=True, mode="json" - ) + d = SapAseConnectionConfig( + project=PROJECT, name="sap_ase", host="ase.example", database="people", user="sa" + ).model_dump(exclude_none=True, by_alias=True, mode="json") assert "tls_server_name" not in d assert "connect_timeout" not in d @@ -1716,6 +1850,7 @@ def test_connection_config_dispatch_picks_sap_ase_subclass(): payload = { "id": "sap-ase-id-1", "name": "sap_ase", + "project": PROJECT, "mask_type": "database", "db_type": "sap_ase", "host": "ase.example", @@ -1734,6 +1869,7 @@ def test_connection_config_dispatch_picks_sap_ase_subclass(): def test_salesforce_connection_serializes_for_create(): conn = SalesforceConnectionConfig( + project=PROJECT, name="salesforce", instance_url="https://example--uat.sandbox.my.salesforce.com", login_url="https://example--uat.sandbox.my.salesforce.com", @@ -1760,6 +1896,7 @@ def test_salesforce_connection_serializes_for_create(): def test_salesforce_connection_without_passphrase_omits_it(): """An unencrypted private key needs no passphrase.""" d = SalesforceConnectionConfig( + project=PROJECT, name="salesforce", instance_url="https://example.my.salesforce.com", login_url="https://example.my.salesforce.com", @@ -1775,6 +1912,7 @@ def test_connection_config_dispatch_picks_salesforce_subclass(): payload = { "id": "salesforce-id-1", "name": "salesforce", + "project": PROJECT, "mask_type": "database", "db_type": "salesforce", "instance_url": "https://example.my.salesforce.com", diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 83f9348..a42a396 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -46,10 +46,12 @@ from datamasque.client.models.connection import ConnectionId, DatabaseConnectionConfig, DatabaseType from datamasque.client.models.data_selection import SelectedColumns, SelectedFileData, UserSelection from datamasque.client.models.status import AsyncRulesetGenerationTaskStatus -from tests.helpers import parse_multipart_form +from tests.helpers import PROJECT, parse_multipart_form, sample_mounted_share_connection_json DISCOVERY_CONFIG_ID = "aaaaaaaa-1111-2222-3333-444444444444" RG_CONFIG_ID = "bbbbbbbb-1111-2222-3333-444444444444" +CONNECTION_PROJECT = "2-banking" +"""The project of the connection the generated rulesets come from.""" def test_generate_ruleset(client): @@ -159,6 +161,14 @@ def test_poll_async_ruleset_generation(client): assert status is AsyncRulesetGenerationTaskStatus.failed +def mock_source_connection(m: requests_mock.Mocker, connection_id: ConnectionId) -> None: + """Mocks the connection lookup that `get_generated_rulesets` makes to find the rulesets' project.""" + m.get( + f"http://test-server/api/connections/{connection_id}/", + json=sample_mounted_share_connection_json(id=connection_id, name="source") | {"project": CONNECTION_PROJECT}, + ) + + def test_get_generated_rulesets_success(client): connection_id = ConnectionId("1") yaml_content_1 = b""" @@ -185,6 +195,7 @@ def test_get_generated_rulesets_success(client): """ with requests_mock.Mocker() as m: + mock_source_connection(m, connection_id) m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", json={"status": "finished"}, @@ -211,6 +222,7 @@ def test_get_generated_rulesets_success(client): assert rulesets[0].yaml == yaml_content_1.decode("utf-8") assert rulesets[1].name == "ruleset2" assert rulesets[1].yaml == yaml_content_2.decode("utf-8") + assert [ruleset.project for ruleset in rulesets] == [CONNECTION_PROJECT, CONNECTION_PROJECT] def test_get_generated_rulesets_finished_with_warnings_success(client): @@ -229,6 +241,7 @@ def test_get_generated_rulesets_finished_with_warnings_success(client): """ with requests_mock.Mocker() as m: + mock_source_connection(m, connection_id) m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", json={"status": "finished_with_warnings"}, @@ -259,6 +272,7 @@ def test_get_generated_rulesets_empty_archive_raises(client): connection_id = ConnectionId("1") with requests_mock.Mocker() as m: + mock_source_connection(m, connection_id) m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", json={"status": "finished"}, @@ -287,6 +301,7 @@ def test_get_generated_rulesets_from_selection_success(client): generated_yaml = 'version: "1.0"\ntasks:\n- type: mask_table\n table: users\n key: id\n rules: []\n' with requests_mock.Mocker() as m: + mock_source_connection(m, connection_id) # Status check and the redirect target are the same URL; both resolve to the same JSON. m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", @@ -305,12 +320,14 @@ def test_get_generated_rulesets_from_selection_success(client): assert rulesets[0].yaml == generated_yaml # The server doesn't return a name — callers set one before create_or_update_ruleset. assert rulesets[0].name == "generated_ruleset" + assert rulesets[0].project == CONNECTION_PROJECT def test_get_generated_rulesets_from_selection_empty_ruleset_raises(client): """A finished task with no `generated_ruleset` in the JSON body raises a clear error.""" connection_id = ConnectionId("1") with requests_mock.Mocker() as m: + mock_source_connection(m, connection_id) m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", json={"status": "finished", "generated_ruleset": None}, @@ -376,6 +393,7 @@ def test_get_generated_rulesets_download_fail(client): connection_id = ConnectionId("1") with requests_mock.Mocker() as m: + mock_source_connection(m, connection_id) m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", json={"status": "finished"}, @@ -660,6 +678,7 @@ def test_schema_discovery_request_model_dump_includes_set_fields(): def test_discovery_requests_accept_connection_config_objects(): """All three discovery request models accept a full `ConnectionConfig` and extract its `id`.""" connection = DatabaseConnectionConfig( + project=PROJECT, id=ConnectionId("conn-uuid"), name="prod_db", db_type=DatabaseType.postgres, @@ -1242,14 +1261,16 @@ def test_schema_discovery_from_config_request_accepts_discovery_config_id(): def test_schema_discovery_from_config_request_unwraps_discovery_config_model(): """Passing a full `DiscoveryConfig` object substitutes its `id` for the wire payload.""" - config = DiscoveryConfig(name="my_cfg", config_type="database", id=DiscoveryConfigId(DISCOVERY_CONFIG_ID)) + config = DiscoveryConfig( + project=PROJECT, name="my_cfg", config_type="database", id=DiscoveryConfigId(DISCOVERY_CONFIG_ID) + ) req = SchemaDiscoveryFromConfigRequest(connection="conn-1", discovery_config=config) assert req.model_dump(exclude_none=True, mode="json")["discovery_config"] == DISCOVERY_CONFIG_ID def test_schema_discovery_from_config_request_rejects_unsaved_discovery_config(): """A `DiscoveryConfig` without an `id` cannot be used yet — raises immediately.""" - config = DiscoveryConfig(name="my_cfg", config_type="database") + config = DiscoveryConfig(project=PROJECT, name="my_cfg", config_type="database") with pytest.raises(ValueError, match="id is None"): SchemaDiscoveryFromConfigRequest(connection="conn-1", discovery_config=config) @@ -1352,7 +1373,9 @@ def test_start_schema_discovery_run_from_config_sends_schemas(client): def test_start_file_data_discovery_run_from_config_sends_discovery_config(client): """`start_file_data_discovery_run_from_config` posts only the connection and discovery_config id.""" - config = DiscoveryConfig(name="my_cfg", config_type="file", id=DiscoveryConfigId(DISCOVERY_CONFIG_ID)) + config = DiscoveryConfig( + project=PROJECT, name="my_cfg", config_type="file", id=DiscoveryConfigId(DISCOVERY_CONFIG_ID) + ) req = FileDataDiscoveryFromConfigRequest(connection="conn-1", discovery_config=config) with requests_mock.Mocker() as m: m.post("http://test-server/api/run-file-data-discovery/v2/", json={"id": 99}, status_code=201) @@ -1618,7 +1641,7 @@ def test_generate_ruleset_with_rg_config_none_sends_null(client): def test_ruleset_generation_with_rg_config_request_unwraps_rg_config_model(): """Passing a full `RGConfig` object substitutes its `id` for the wire payload.""" - config = RGConfig(name="my_rgc", id=RGConfigId(RG_CONFIG_ID)) + config = RGConfig(project=PROJECT, name="my_rgc", id=RGConfigId(RG_CONFIG_ID)) req = RulesetGenerationWithRGConfigRequest( connection="conn-1", selected_columns={"public": {"users": ["email"]}}, @@ -1629,7 +1652,7 @@ def test_ruleset_generation_with_rg_config_request_unwraps_rg_config_model(): def test_ruleset_generation_with_rg_config_request_rejects_unsaved_rg_config(): """An `RGConfig` without an `id` cannot be used yet — raises immediately.""" - config = RGConfig(name="my_rgc") + config = RGConfig(project=PROJECT, name="my_rgc") with pytest.raises(ValueError, match="id is None"): RulesetGenerationWithRGConfigRequest( connection="conn-1", @@ -1759,7 +1782,7 @@ def test_start_async_ruleset_generation_with_rg_config_unwraps_rg_config_model(c """A full `RGConfig` object is unwrapped to its id in the request body.""" connection_id = ConnectionId("1") selected_columns = SelectedColumns(columns={"public": {"users": ["col1"]}}) - config = RGConfig(name="my_rgc", id=RGConfigId(RG_CONFIG_ID)) + config = RGConfig(project=PROJECT, name="my_rgc", id=RGConfigId(RG_CONFIG_ID)) with requests_mock.Mocker() as m: m.post(f"http://test-server/api/async-generate-ruleset/v2/{connection_id}/", status_code=201) diff --git a/tests/test_discovery_config_libraries.py b/tests/test_discovery_config_libraries.py index 20a0d4d..c73f76b 100644 --- a/tests/test_discovery_config_libraries.py +++ b/tests/test_discovery_config_libraries.py @@ -13,6 +13,7 @@ ) from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.status import ValidationStatus +from tests.helpers import PROJECT, PROJECT_ID LIBRARY_ID_1 = "aaaaaaaa-1111-2222-3333-444444444444" LIBRARY_ID_2 = "bbbbbbbb-1111-2222-3333-444444444444" @@ -25,7 +26,7 @@ def sample_library_list_response() -> list[dict[str, Any]]: { "id": LIBRARY_ID_1, "name": "my_library", - "namespace": "org", + "project": PROJECT, "is_valid": "valid", "validation_error": None, "usage_count": 0, @@ -35,7 +36,7 @@ def sample_library_list_response() -> list[dict[str, Any]]: { "id": LIBRARY_ID_2, "name": "another_library", - "namespace": "", + "project": PROJECT, "is_valid": "invalid", "validation_error": "bad yaml", "usage_count": 2, @@ -51,7 +52,7 @@ def sample_library_detail_response() -> dict[str, Any]: return { "id": LIBRARY_ID_1, "name": "my_library", - "namespace": "org", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "valid", "validation_error": None, @@ -65,14 +66,14 @@ def sample_library_detail_response() -> dict[str, Any]: def config_library() -> DiscoveryConfigLibrary: return DiscoveryConfigLibrary( name="test_library", - namespace="test_ns", + project=PROJECT, yaml="labels: []\nmetadata_rules: []\nidd_rules: []\n", ) def test_library_is_untyped() -> None: """A library carries no `config_type`; the same library serves both config types.""" - library = DiscoveryConfigLibrary(name="test_library", yaml="labels: []\n") + library = DiscoveryConfigLibrary(name="test_library", project=PROJECT, yaml="labels: []\n") assert "config_type" not in library.model_dump(by_alias=True) @@ -90,7 +91,7 @@ def test_list_discovery_config_libraries( assert len(libraries) == 2 assert libraries[0].id == DiscoveryConfigLibraryId(LIBRARY_ID_1) assert libraries[0].name == "my_library" - assert libraries[0].namespace == "org" + assert libraries[0].project == PROJECT assert libraries[0].yaml is None assert libraries[0].is_valid is ValidationStatus.valid assert libraries[0].usage_count == 0 @@ -124,7 +125,7 @@ def test_get_discovery_config_library(client: DataMasqueClient, sample_library_d assert library.id == DiscoveryConfigLibraryId(LIBRARY_ID_1) assert library.name == "my_library" - assert library.namespace == "org" + assert library.project == PROJECT assert library.yaml == "labels: []\nmetadata_rules: []\nidd_rules: []\n" assert library.is_valid is ValidationStatus.valid assert library.usage_count == 0 @@ -146,67 +147,12 @@ def test_get_discovery_config_library_by_name_found( json=sample_library_detail_response, status_code=200, ) - library = client.get_discovery_config_library_by_name("my_library", "org") + library = client.get_discovery_config_library_by_name("my_library", PROJECT_ID) assert library is not None assert library.name == "my_library" assert library.yaml == "labels: []\nmetadata_rules: []\nidd_rules: []\n" - assert "name_exact=my_library" in m.request_history[0].url - - -def test_get_discovery_config_library_by_name_matches_namespace_client_side( - client: DataMasqueClient, sample_library_detail_response: dict[str, Any] -) -> None: - """The server's `namespace_exact` filter ignores empty strings, so namespace matching happens client-side.""" - same_name_two_namespaces = [ - { - "id": LIBRARY_ID_1, - "name": "my_library", - "namespace": "org", - "is_valid": "valid", - "created": "2025-01-01T12:00:00Z", - "modified": "2025-01-02T12:00:00Z", - }, - { - "id": LIBRARY_ID_2, - "name": "my_library", - "namespace": "", - "is_valid": "valid", - "created": "2025-01-01T12:00:00Z", - "modified": "2025-01-02T12:00:00Z", - }, - ] - - with requests_mock.Mocker() as m: - m.get( - "http://test-server/api/discovery/config-libraries/", - json=same_name_two_namespaces, - status_code=200, - ) - m.get( - f"http://test-server/api/discovery/config-libraries/{LIBRARY_ID_1}/", - json=sample_library_detail_response, - status_code=200, - ) - library = client.get_discovery_config_library_by_name("my_library", "org") - - assert library is not None - assert library.id == DiscoveryConfigLibraryId(LIBRARY_ID_1) - - -def test_get_discovery_config_library_by_name_default_namespace_does_not_match_other_namespaces( - client: DataMasqueClient, sample_library_list_response: list[dict[str, Any]] -) -> None: - with requests_mock.Mocker() as m: - m.get( - "http://test-server/api/discovery/config-libraries/", - json=[sample_library_list_response[0]], - status_code=200, - ) - library = client.get_discovery_config_library_by_name("my_library") - - assert library is None - assert m.call_count == 1 + assert m.request_history[0].qs == {"name_exact": ["my_library"], "project": [str(PROJECT_ID)]} def test_get_discovery_config_library_by_name_not_found(client: DataMasqueClient) -> None: @@ -216,7 +162,7 @@ def test_get_discovery_config_library_by_name_not_found(client: DataMasqueClient json=[], status_code=200, ) - library = client.get_discovery_config_library_by_name("nonexistent") + library = client.get_discovery_config_library_by_name("nonexistent", PROJECT_ID) assert library is None @@ -226,7 +172,7 @@ def test_get_discovery_config_library_by_name_raises_when_server_omits_id(client list_response_without_id = [ { "name": "my_library", - "namespace": "org", + "project": PROJECT, "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -240,14 +186,14 @@ def test_get_discovery_config_library_by_name_raises_when_server_omits_id(client status_code=200, ) with pytest.raises(DataMasqueApiError, match="without an `id`"): - client.get_discovery_config_library_by_name("my_library", "org") + client.get_discovery_config_library_by_name("my_library", PROJECT_ID) def test_create_discovery_config_library(client: DataMasqueClient, config_library: DiscoveryConfigLibrary) -> None: create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "valid", "validation_error": None, @@ -274,7 +220,7 @@ def test_create_discovery_config_library(client: DataMasqueClient, config_librar request_body = m.last_request.json() assert request_body == { "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", } @@ -285,7 +231,7 @@ def test_create_discovery_config_library_reports_validation_error( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "invalid", "validation_error": "metadata_rules is not a list", @@ -312,7 +258,7 @@ def test_update_discovery_config_library(client: DataMasqueClient, config_librar update_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "valid", "validation_error": None, @@ -338,7 +284,7 @@ def test_update_discovery_config_library(client: DataMasqueClient, config_librar request_body = m.last_request.json() assert request_body == { "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", } @@ -366,7 +312,7 @@ def test_create_or_update_discovery_config_library_create( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "valid", "validation_error": None, @@ -389,8 +335,9 @@ def test_create_or_update_discovery_config_library_create( assert result.id == DiscoveryConfigLibraryId(LIBRARY_ID_1) assert m.request_history[0].method == "GET" - assert "name_exact=test_library" in m.request_history[0].url + assert m.request_history[0].qs == {"name_exact": ["test_library"], "project": [PROJECT]} assert m.request_history[1].method == "POST" + assert m.request_history[1].json()["project"] == PROJECT def test_create_or_update_discovery_config_library_update( @@ -400,7 +347,7 @@ def test_create_or_update_discovery_config_library_update( { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "is_valid": "valid", "created": "2025-06-01T10:00:00Z", "modified": "2025-06-01T10:00:00Z", @@ -409,7 +356,7 @@ def test_create_or_update_discovery_config_library_update( update_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "valid", "validation_error": None, @@ -480,10 +427,10 @@ def test_delete_discovery_config_library_by_name( f"http://test-server/api/discovery/config-libraries/{LIBRARY_ID_1}/", status_code=204, ) - client.delete_discovery_config_library_by_name_if_exists("my_library", "org") + client.delete_discovery_config_library_by_name_if_exists("my_library", PROJECT_ID) assert m.request_history[0].method == "GET" - assert "name_exact=my_library" in m.request_history[0].url + assert m.request_history[0].qs == {"name_exact": ["my_library"], "project": [str(PROJECT_ID)]} assert m.request_history[1].method == "DELETE" @@ -496,17 +443,45 @@ def test_delete_discovery_config_library_by_name_not_found( json=sample_library_list_response, status_code=200, ) - client.delete_discovery_config_library_by_name_if_exists("nonexistent") + client.delete_discovery_config_library_by_name_if_exists("nonexistent", PROJECT_ID) assert m.call_count == 1 assert m.request_history[0].method == "GET" def test_create_discovery_config_library_with_empty_yaml_raises(client: DataMasqueClient) -> None: - library = DiscoveryConfigLibrary(name="test_library", yaml="") + library = DiscoveryConfigLibrary(name="test_library", project=PROJECT, yaml="") with requests_mock.Mocker() as m: with pytest.raises(ValueError, match="without YAML content"): client.create_discovery_config_library(library) assert not any(request.method == "POST" for request in m.request_history) + + +def test_namespace_in_a_response_is_dropped_and_not_sent_back( + client: DataMasqueClient, sample_library_detail_response: dict[str, Any] +) -> None: + """The server lets a sent `namespace` override `project`, so the client never echoes one back.""" + response = {**sample_library_detail_response, "namespace": "org"} + + with requests_mock.Mocker() as m: + m.get( + f"http://test-server/api/discovery/config-libraries/{LIBRARY_ID_1}/", + json=response, + status_code=200, + ) + m.put( + f"http://test-server/api/discovery/config-libraries/{LIBRARY_ID_1}/", + json=response, + status_code=200, + ) + library = client.get_discovery_config_library(DiscoveryConfigLibraryId(LIBRARY_ID_1)) + client.update_discovery_config_library(library) + + assert "namespace" not in library.model_dump() + assert m.last_request.json() == { + "name": "my_library", + "project": PROJECT, + "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", + } diff --git a/tests/test_discovery_configs.py b/tests/test_discovery_configs.py index d34c47d..a700251 100644 --- a/tests/test_discovery_configs.py +++ b/tests/test_discovery_configs.py @@ -8,7 +8,7 @@ from pydantic import JsonValue, ValidationError from datamasque.client import DataMasqueClient -from datamasque.client.exceptions import DataMasqueApiError, DataMasqueException +from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.discovery_config import ( DiscoveryConfig, DiscoveryConfigId, @@ -16,6 +16,7 @@ unwrap_discovery_config_id, ) from datamasque.client.models.status import ValidationStatus +from tests.helpers import PROJECT, PROJECT_ID CONFIG_ID_1 = "aaaaaaaa-1111-2222-3333-444444444444" CONFIG_ID_2 = "bbbbbbbb-1111-2222-3333-444444444444" @@ -31,6 +32,7 @@ def sample_config_list_response() -> dict[str, Any]: { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -39,6 +41,7 @@ def sample_config_list_response() -> dict[str, Any]: { "id": CONFIG_ID_2, "name": "another_config", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-02-01T12:00:00Z", @@ -53,6 +56,7 @@ def sample_config_detail_response() -> dict[str, Any]: return { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "config_type": "database", "archived": False, @@ -65,6 +69,7 @@ def sample_config_detail_response() -> dict[str, Any]: def discovery_config() -> DiscoveryConfig: return DiscoveryConfig( name="test_config", + project=PROJECT, yaml="labels: []\nmetadata_rules: []\nidd_rules: []\n", config_type="database", ) @@ -95,6 +100,7 @@ def test_list_discovery_configs_pagination(client: DataMasqueClient) -> None: { "id": CONFIG_ID_1, "name": "c1", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -103,6 +109,7 @@ def test_list_discovery_configs_pagination(client: DataMasqueClient) -> None: { "id": CONFIG_ID_2, "name": "c2", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -118,6 +125,7 @@ def test_list_discovery_configs_pagination(client: DataMasqueClient) -> None: { "id": "cccccccc-1111-2222-3333-444444444444", "name": "c3", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -171,6 +179,7 @@ def test_get_discovery_config_by_name_found( { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -190,12 +199,16 @@ def test_get_discovery_config_by_name_found( json=sample_config_detail_response, status_code=200, ) - config = client.get_discovery_config_by_name("my_config", DiscoveryConfigType.database) + config = client.get_discovery_config_by_name("my_config", DiscoveryConfigType.database, PROJECT_ID) assert config is not None assert config.name == "my_config" - assert "name_exact=my_config" in m.request_history[0].url - assert "config_type=database" in m.request_history[0].url + assert m.request_history[0].qs == { + "name_exact": ["my_config"], + "config_type": ["database"], + "project": [str(PROJECT_ID)], + "limit": ["1"], + } def test_get_discovery_config_by_name_not_found(client: DataMasqueClient) -> None: @@ -207,7 +220,7 @@ def test_get_discovery_config_by_name_not_found(client: DataMasqueClient) -> Non json=empty_response, status_code=200, ) - config = client.get_discovery_config_by_name("nonexistent", DiscoveryConfigType.database) + config = client.get_discovery_config_by_name("nonexistent", DiscoveryConfigType.database, PROJECT_ID) assert config is None @@ -220,6 +233,7 @@ def test_get_discovery_config_by_name_raises_when_server_omits_id(client: DataMa "results": [ { "name": "my_config", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -235,13 +249,14 @@ def test_get_discovery_config_by_name_raises_when_server_omits_id(client: DataMa status_code=200, ) with pytest.raises(DataMasqueApiError, match="without an `id`"): - client.get_discovery_config_by_name("my_config", DiscoveryConfigType.database) + client.get_discovery_config_by_name("my_config", DiscoveryConfigType.database, PROJECT_ID) def test_create_discovery_config(client: DataMasqueClient, discovery_config: DiscoveryConfig) -> None: create_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "config_type": "database", "is_valid": "in_progress", @@ -268,6 +283,7 @@ def test_create_discovery_config(client: DataMasqueClient, discovery_config: Dis assert request_body["name"] == "test_config" assert request_body["config_yaml"] == "labels: []\nmetadata_rules: []\nidd_rules: []\n" assert request_body["config_type"] == "database" + assert request_body["project"] == PROJECT # Server-managed read-only fields must not be echoed back in the request body. assert "id" not in request_body assert "is_valid" not in request_body @@ -281,6 +297,7 @@ def test_update_discovery_config(client: DataMasqueClient, discovery_config: Dis update_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "config_type": "database", "archived": False, @@ -326,6 +343,7 @@ def test_create_or_update_discovery_config_create(client: DataMasqueClient, disc create_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "config_type": "database", "archived": False, @@ -340,7 +358,9 @@ def test_create_or_update_discovery_config_create(client: DataMasqueClient, disc assert result.id == DiscoveryConfigId(CONFIG_ID_1) assert m.request_history[0].method == "GET" + assert m.request_history[0].qs["project"] == [PROJECT] assert m.request_history[1].method == "POST" + assert m.request_history[1].json()["project"] == PROJECT def test_create_or_update_discovery_config_update(client: DataMasqueClient, discovery_config: DiscoveryConfig) -> None: @@ -352,6 +372,7 @@ def test_create_or_update_discovery_config_update(client: DataMasqueClient, disc { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-06-01T10:00:00Z", @@ -362,6 +383,7 @@ def test_create_or_update_discovery_config_update(client: DataMasqueClient, disc update_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "config_type": "database", "archived": False, @@ -399,20 +421,28 @@ def test_delete_discovery_config_by_id_not_found(client: DataMasqueClient) -> No def test_delete_discovery_config_by_name(client: DataMasqueClient, sample_config_list_response: dict[str, Any]) -> None: with requests_mock.Mocker() as m: - m.get("http://test-server/api/discovery/configs/", json=sample_config_list_response, status_code=200) + # The server filters by name, so only the matching config is listed. + only_match = {**sample_config_list_response, "count": 1, "results": sample_config_list_response["results"][:1]} + m.get("http://test-server/api/discovery/configs/", json=only_match, status_code=200) m.delete(f"http://test-server/api/discovery/configs/{CONFIG_ID_1}/", status_code=204) - client.delete_discovery_config_by_name_if_exists("my_config", DiscoveryConfigType.database) + client.delete_discovery_config_by_name_if_exists("my_config", DiscoveryConfigType.database, PROJECT_ID) assert m.request_history[0].method == "GET" + assert m.request_history[0].qs == { + "name_exact": ["my_config"], + "config_type": ["database"], + "project": [str(PROJECT_ID)], + "limit": ["1"], + } assert m.request_history[1].method == "DELETE" -def test_delete_discovery_config_by_name_not_found( - client: DataMasqueClient, sample_config_list_response: dict[str, Any] -) -> None: +def test_delete_discovery_config_by_name_not_found(client: DataMasqueClient) -> None: + empty_response = {"count": 0, "next": None, "previous": None, "results": []} + with requests_mock.Mocker() as m: - m.get("http://test-server/api/discovery/configs/", json=sample_config_list_response, status_code=200) - client.delete_discovery_config_by_name_if_exists("nonexistent", DiscoveryConfigType.database) + m.get("http://test-server/api/discovery/configs/", json=empty_response, status_code=200) + client.delete_discovery_config_by_name_if_exists("nonexistent", DiscoveryConfigType.database, PROJECT_ID) assert m.call_count == 1 assert m.request_history[0].method == "GET" @@ -426,6 +456,7 @@ def test_delete_discovery_config_by_name_raises_when_server_omits_id(client: Dat "results": [ { "name": "my_config", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -440,29 +471,8 @@ def test_delete_discovery_config_by_name_raises_when_server_omits_id(client: Dat json=list_response_without_id, status_code=200, ) - with pytest.raises(DataMasqueException, match="without an `id`"): - client.delete_discovery_config_by_name_if_exists("my_config", DiscoveryConfigType.database) - - -def test_delete_discovery_config_by_name_only_deletes_matching_type(client: DataMasqueClient) -> None: - shared_name_response = { - "count": 2, - "next": None, - "previous": None, - "results": [ - {"id": CONFIG_ID_1, "name": "shared", "config_type": "database", "archived": False}, - {"id": CONFIG_ID_2, "name": "shared", "config_type": "file", "archived": False}, - ], - } - - with requests_mock.Mocker() as m: - m.get("http://test-server/api/discovery/configs/", json=shared_name_response, status_code=200) - m.delete(f"http://test-server/api/discovery/configs/{CONFIG_ID_2}/", status_code=204) - client.delete_discovery_config_by_name_if_exists("shared", DiscoveryConfigType.file) - - assert m.request_history[-1].method == "DELETE" - assert CONFIG_ID_2 in m.request_history[-1].url - assert CONFIG_ID_1 not in m.request_history[-1].url + with pytest.raises(DataMasqueApiError, match="without an `id`"): + client.delete_discovery_config_by_name_if_exists("my_config", DiscoveryConfigType.database, PROJECT_ID) def test_get_default_discovery_config_yaml(client: DataMasqueClient) -> None: @@ -486,6 +496,7 @@ def test_discovery_config_parses_validation_fields() -> None: { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "config_type": "database", "config_yaml": "labels: []", "is_valid": "invalid", @@ -505,6 +516,7 @@ def test_discovery_config_validation_fields_optional() -> None: { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "config_type": "database", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -521,12 +533,12 @@ def test_unwrap_discovery_config_id_passes_through_strings() -> None: def test_unwrap_discovery_config_id_extracts_id_from_model() -> None: - config = DiscoveryConfig(name="x", config_type="database", id=DiscoveryConfigId(CONFIG_ID_1)) + config = DiscoveryConfig(name="x", project=PROJECT, config_type="database", id=DiscoveryConfigId(CONFIG_ID_1)) assert unwrap_discovery_config_id(config) == CONFIG_ID_1 def test_unwrap_discovery_config_id_raises_without_id() -> None: - config = DiscoveryConfig(name="x", config_type="database") + config = DiscoveryConfig(name="x", project=PROJECT, config_type="database") with pytest.raises(ValueError, match="id is None"): unwrap_discovery_config_id(config) @@ -535,6 +547,7 @@ def _build_config_response(is_valid: str, **extra: JsonValue) -> dict[str, JsonV return { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_type": "database", "is_valid": is_valid, "validation_error": None, @@ -595,7 +608,7 @@ def test_discovery_config_rejects_malformed_error_entry() -> None: def test_create_discovery_config_with_empty_yaml_raises(client: DataMasqueClient) -> None: - config = DiscoveryConfig(name="test_config", yaml="", config_type="database") + config = DiscoveryConfig(name="test_config", project=PROJECT, yaml="", config_type="database") with requests_mock.Mocker() as m: with pytest.raises(ValueError, match="without YAML content"): @@ -605,7 +618,7 @@ def test_create_discovery_config_with_empty_yaml_raises(client: DataMasqueClient def test_update_discovery_config_with_empty_yaml_raises(client: DataMasqueClient) -> None: - config = DiscoveryConfig(id=CONFIG_ID_1, name="test_config", yaml="", config_type="database") + config = DiscoveryConfig(id=CONFIG_ID_1, name="test_config", project=PROJECT, yaml="", config_type="database") with requests_mock.Mocker() as m: with pytest.raises(ValueError, match="without YAML content"): diff --git a/tests/test_files.py b/tests/test_files.py index 3b8f99b..37a2913 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -10,8 +10,8 @@ import requests_mock from datamasque.client import DataMasqueClient -from datamasque.client.models.files import OracleWalletFile, SeedFile, SnowflakeKeyFile, SslZipFile -from tests.helpers import fake, parse_multipart_form +from datamasque.client.models.files import DataMasqueFile, OracleWalletFile, SeedFile, SnowflakeKeyFile, SslZipFile +from tests.helpers import OTHER_PROJECT, PROJECT, PROJECT_ID, fake, parse_multipart_form @pytest.mark.parametrize( @@ -49,7 +49,7 @@ def test_upload_file(client, file_type, source_factory): "modified_date": mock_return_modified_date.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), }, ) - file = client.upload_file(file_type, name_of_file, source) + file = client.upload_file(file_type, name_of_file, source, PROJECT_ID) if isinstance(source, (str, Path)): m_open.assert_called_once_with(source, "rb") @@ -141,7 +141,7 @@ def test_get_files_by_type_and_name(client, file_type): }, ], ) - result = client.get_file_of_type_by_name(file_type, "file2") + result = client.get_file_of_type_by_name(file_type, "file2", PROJECT_ID) assert isinstance(result, file_type) assert result.id == id_2 @@ -202,7 +202,7 @@ def test_upload_file_if_not_exists_skips_when_same_name_exists(client, file_type }, ], ) - result = client.upload_file_if_not_exists(file_type, "already_here.csv") + result = client.upload_file_if_not_exists(file_type, "already_here.csv", PROJECT_ID) assert result is None # Only the list-by-type GET should have fired — no POST. @@ -231,7 +231,7 @@ def test_upload_file_if_not_exists_uploads_when_missing(client, file_type): "modified_date": "2024-01-01T00:00:00.000000Z", }, ) - result = client.upload_file_if_not_exists(file_type, Path("new_file.csv")) + result = client.upload_file_if_not_exists(file_type, Path("new_file.csv"), PROJECT_ID) assert isinstance(result, file_type) assert result.id == new_id @@ -264,7 +264,7 @@ def test_upload_file_retries_on_401(config): }, ], ) - client.upload_file(SeedFile, "seed.csv", file_content) + client.upload_file(SeedFile, "seed.csv", file_content, PROJECT_ID) assert m_request.call_count == 2 first_form = parse_multipart_form(m_request.request_history[0]) @@ -286,7 +286,7 @@ def test_upload_ssl_zip_file_sends_mysql_database_type(client): "modified_date": "2024-01-01T00:00:00.000000Z", }, ) - client.upload_file(SslZipFile, "certs.zip", b"zip bytes") + client.upload_file(SslZipFile, "certs.zip", b"zip bytes", PROJECT_ID) form = parse_multipart_form(m_request.request_history[0]) assert form["database_type"] == "mysql" @@ -306,7 +306,77 @@ def test_upload_non_ssl_zip_file_omits_database_type(client, file_type): "modified_date": "2024-01-01T00:00:00.000000Z", }, ) - client.upload_file(file_type, "file.bin", b"content") + client.upload_file(file_type, "file.bin", b"content", PROJECT_ID) form = parse_multipart_form(m_request.request_history[0]) assert "database_type" not in form + + +@pytest.mark.parametrize("file_type", [SeedFile, OracleWalletFile, SslZipFile, SnowflakeKeyFile]) +def test_upload_file_sends_project(client: DataMasqueClient, file_type: type[DataMasqueFile]) -> None: + with requests_mock.Mocker() as m_request: + m_request.post( + f"http://test-server/{file_type.get_url()}", + status_code=201, + json={ + "id": str(uuid.uuid4()), + "name": "file.bin", + "project": PROJECT, + "created_date": "2024-01-01T00:00:00.000000Z", + "modified_date": "2024-01-01T00:00:00.000000Z", + }, + ) + file = client.upload_file(file_type, "file.bin", b"content", PROJECT_ID) + + form = parse_multipart_form(m_request.request_history[0]) + assert form["project"] == str(PROJECT_ID) + assert file.project == PROJECT + + +@pytest.mark.parametrize("file_type", [SeedFile, OracleWalletFile, SslZipFile, SnowflakeKeyFile]) +def test_list_files_of_type_filters_by_project(client: DataMasqueClient, file_type: type[DataMasqueFile]) -> None: + with requests_mock.Mocker() as m_request: + m_request.get(f"http://test-server/{file_type.get_url()}?project={PROJECT_ID}", json=[], complete_qs=True) + assert client.list_files_of_type(file_type, PROJECT_ID) == [] + + +@pytest.mark.parametrize("file_type", [SeedFile, OracleWalletFile, SslZipFile, SnowflakeKeyFile]) +def test_get_file_of_type_by_name_is_scoped_to_the_project( + client: DataMasqueClient, file_type: type[DataMasqueFile] +) -> None: + """A file with the same name in another project is not matched, because the listing is filtered by project.""" + other_project_file = { + "id": str(uuid.uuid4()), + "name": "seed.csv", + "project": "2-other", + "created_date": "2024-01-01T00:00:00.000000Z", + } + with requests_mock.Mocker() as m_request: + m_request.get(f"http://test-server/{file_type.get_url()}", json=[other_project_file]) + m_request.get(f"http://test-server/{file_type.get_url()}?project={PROJECT_ID}", json=[], complete_qs=True) + result = client.get_file_of_type_by_name(file_type, "seed.csv", PROJECT_ID) + + assert result is None + assert m_request.last_request.qs == {"project": [str(PROJECT_ID)]} + + +def test_upload_file_sends_the_id_of_a_project(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m_request: + m_request.post( + f"http://test-server/{SeedFile.get_url()}", + status_code=201, + json={ + "id": str(uuid.uuid4()), + "name": "seed.csv", + "project": "2-banking", + "created_date": "2024-01-01T00:00:00.000000Z", + }, + ) + client.upload_file( + SeedFile, + "seed.csv", + b"content", + OTHER_PROJECT, + ) + + assert parse_multipart_form(m_request.request_history[0])["project"] == "2" diff --git a/tests/test_model_project.py b/tests/test_model_project.py index 112b18a..17251d0 100644 --- a/tests/test_model_project.py +++ b/tests/test_model_project.py @@ -13,6 +13,7 @@ from datamasque.client.models.ruleset_library import RulesetLibrary from datamasque.client.models.runs import RunInfo, UnfinishedRun from datamasque.client.models.table_reference import TableReference +from tests.helpers import PROJECT ResourceModel = Union[ ConnectionConfig, Ruleset, RulesetLibrary, DiscoveryConfig, DiscoveryConfigLibrary, RGConfig, TableReference @@ -52,17 +53,29 @@ @pytest.mark.parametrize(("parse", "payload"), _RESOURCE_CASES) -def test_resource_parses_project_and_excludes_it_when_serializing(parse: ParseResource, payload: Payload) -> None: - model = parse({**payload, "project": "1-default"}) +def test_resource_parses_project_and_sends_it_when_serializing(parse: ParseResource, payload: Payload) -> None: + model = parse({**payload, "project": PROJECT}) - assert model.project == "1-default" + assert model.project == PROJECT assert "project" not in (model.model_extra or {}) - assert "project" not in model.model_dump(by_alias=True, mode="json") + assert model.model_dump(by_alias=True, mode="json")["project"] == PROJECT @pytest.mark.parametrize(("parse", "payload"), _RESOURCE_CASES) -def test_resource_project_defaults_to_none_when_absent(parse: ParseResource, payload: Payload) -> None: - assert parse(payload).project is None +def test_resource_project_is_optional_and_not_sent_when_unset(parse: ParseResource, payload: Payload) -> None: + model = parse(payload) + + assert model.project is None + assert "project" not in model.model_dump(by_alias=True, exclude_none=True, mode="json") + + +@pytest.mark.parametrize("library_class", [RulesetLibrary, DiscoveryConfigLibrary]) +def test_library_drops_namespace(library_class: type[Union[RulesetLibrary, DiscoveryConfigLibrary]]) -> None: + """The server still returns the deprecated `namespace`, which would override `project` if it were sent back.""" + library = library_class.model_validate({"id": "l-1", "name": "l", "project": PROJECT, "namespace": "org"}) + + assert "namespace" not in (library.model_extra or {}) + assert "namespace" not in library.model_dump(by_alias=True, mode="json") _RUN_PAYLOAD: Payload = { @@ -77,7 +90,7 @@ def test_resource_project_defaults_to_none_when_absent(parse: ParseResource, pay @pytest.mark.parametrize("run_class", [RunInfo, UnfinishedRun]) def test_run_parses_project(run_class: type[Union[RunInfo, UnfinishedRun]]) -> None: - run = run_class.model_validate({**_RUN_PAYLOAD, "project": "1-default"}) + run = run_class.model_validate({**_RUN_PAYLOAD, "project": PROJECT}) - assert run.project == "1-default" + assert run.project == PROJECT assert "project" not in (run.model_extra or {}) diff --git a/tests/test_projects.py b/tests/test_projects.py index 38b1556..0830293 100644 --- a/tests/test_projects.py +++ b/tests/test_projects.py @@ -1,11 +1,11 @@ """Tests for project support in the DataMasque client.""" from datetime import datetime -from typing import Union +from typing import Callable, Union import pytest import requests_mock -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from datamasque.client import ( DataMasqueApiError, @@ -17,6 +17,17 @@ ProjectDeleteConflictReason, ProjectId, ) +from datamasque.client.exceptions import DataMasqueException +from datamasque.client.models.connection import ConnectionId, S3ConnectionConfig +from datamasque.client.models.discovery_config import DiscoveryConfig +from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary +from datamasque.client.models.rg_config import RGConfig +from datamasque.client.models.ruleset import Ruleset, RulesetType +from datamasque.client.models.ruleset_library import RulesetLibrary +from datamasque.client.models.table_reference import TableReference +from tests.helpers import PROJECT, PROJECT_ID, active_projects_page + +CREATED_ID = "aaaaaaaa-1111-2222-3333-444444444444" BANKING_PROJECT = { "id": 7, @@ -266,3 +277,141 @@ def test_get_missing_project_raises(client: DataMasqueClient) -> None: client.get_project(ProjectId(99)) assert exc_info.value.response.status_code == 404 + + +EMPTY_PAGE = {"count": 0, "next": None, "previous": None, "results": []} + +CREATE_OR_UPDATE_WITHOUT_A_PROJECT = [ + pytest.param( + lambda: S3ConnectionConfig(name="s3", bucket="b", base_directory=""), + "create_or_update_connection", + "/api/connections/", + [], + id="connection", + ), + pytest.param( + lambda: RGConfig(name="rg", yaml="labels: []"), + "create_or_update_rg_config", + "/api/ruleset-generation-configs/", + EMPTY_PAGE, + id="rg_config", + ), + pytest.param( + lambda: RulesetLibrary(name="lib", yaml="version: '1.0'"), + "create_or_update_ruleset_library", + "/api/ruleset-libraries/", + EMPTY_PAGE, + id="ruleset_library", + ), + pytest.param( + lambda: DiscoveryConfig(name="dc", yaml="labels: []", config_type="database"), + "create_or_update_discovery_config", + "/api/discovery/configs/", + EMPTY_PAGE, + id="discovery_config", + ), + pytest.param( + lambda: DiscoveryConfigLibrary(name="dcl", yaml="labels: []"), + "create_or_update_discovery_config_library", + "/api/discovery/config-libraries/", + [], + id="discovery_config_library", + ), + pytest.param( + lambda: TableReference(name="tr", connection=ConnectionId("1"), source="customers.csv"), + "create_or_update_table_reference", + "/api/table-references/", + [], + id="table_reference", + ), +] +""" +Each `create_or_update_*` that looks up the existing resource itself: +a factory for a resource with no `project`, the method, the path it looks up and creates on, +and that path's empty listing. +""" + +RULESET_WITHOUT_A_PROJECT = pytest.param( + lambda: Ruleset(name="rs", yaml="version: '1.0'\ntasks: []", ruleset_type=RulesetType.database), + "create_or_update_ruleset", + "/api/rulesets/", + None, + id="ruleset", +) +"""`create_or_update_ruleset`, which leaves finding the existing ruleset to the server's upsert.""" + + +@pytest.mark.parametrize("make_resource, method, path, empty_listing", CREATE_OR_UPDATE_WITHOUT_A_PROJECT) +def test_create_or_update_without_a_project_uses_the_only_active_project( + client: DataMasqueClient, + make_resource: Callable[[], BaseModel], + method: str, + path: str, + empty_listing: object, +) -> None: + """The lookup is in the only active project, and the create leaves the choice of project to the server.""" + resource = make_resource() + created = resource.model_dump(mode="json", by_alias=True) | {"id": CREATED_ID, "project": PROJECT} + + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/", json=active_projects_page(1), status_code=200) + m.get(f"http://test-server{path}", json=empty_listing, status_code=200) + m.post(f"http://test-server{path}", json=created, status_code=201) + getattr(client, method)(resource) + + assert [(request.method, request.path) for request in m.request_history] == [ + ("GET", "/api/projects/"), + ("GET", path), + ("POST", path), + ] + assert m.request_history[0].qs == {"limit": ["1"]} + assert m.request_history[1].qs["project"] == [str(PROJECT_ID)] + assert "project" not in m.last_request.json() + assert resource.model_dump()["project"] == PROJECT + + +def test_create_or_update_ruleset_without_a_project_uses_the_only_active_project(client: DataMasqueClient) -> None: + """The upsert leaves the choice of project to the server, after checking that exactly one project is active.""" + ruleset = Ruleset(name="rs", yaml="version: '1.0'\ntasks: []", ruleset_type=RulesetType.database) + + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/", json=active_projects_page(1), status_code=200) + m.post( + "http://test-server/api/rulesets/?upsert=true", + json={"id": CREATED_ID, "name": "rs", "project": PROJECT}, + status_code=201, + ) + client.create_or_update_ruleset(ruleset) + + assert [(request.method, request.path) for request in m.request_history] == [ + ("GET", "/api/projects/"), + ("POST", "/api/rulesets/"), + ] + assert "project" not in m.last_request.json() + assert ruleset.project == PROJECT + + +@pytest.mark.parametrize("active_projects", [0, 2]) +@pytest.mark.parametrize( + "make_resource, method, path, empty_listing", [*CREATE_OR_UPDATE_WITHOUT_A_PROJECT, RULESET_WITHOUT_A_PROJECT] +) +def test_create_or_update_without_a_project_needs_exactly_one_active_project( + client: DataMasqueClient, + make_resource: Callable[[], BaseModel], + method: str, + path: str, + empty_listing: object, + active_projects: int, +) -> None: + """ + With no project, the server only accepts a create when exactly one project is active. + + So the client raises before looking up or writing anything, + rather than updating a same-named resource in a project the server would not have picked. + """ + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/", json=active_projects_page(active_projects), status_code=200) + with pytest.raises(DataMasqueException, match=f"has {active_projects} active projects"): + getattr(client, method)(make_resource()) + + assert [request.path for request in m.request_history] == ["/api/projects/"] diff --git a/tests/test_rg_configs.py b/tests/test_rg_configs.py index d393b84..82381b0 100644 --- a/tests/test_rg_configs.py +++ b/tests/test_rg_configs.py @@ -14,6 +14,7 @@ unwrap_rg_config_id, ) from datamasque.client.models.status import ValidationErrorType, ValidationStatus +from tests.helpers import PROJECT, PROJECT_ID CONFIG_ID_1 = "aaaaaaaa-1111-2222-3333-444444444444" CONFIG_ID_2 = "bbbbbbbb-1111-2222-3333-444444444444" @@ -31,6 +32,7 @@ def sample_config_list_response() -> dict[str, Any]: { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -38,6 +40,7 @@ def sample_config_list_response() -> dict[str, Any]: { "id": CONFIG_ID_2, "name": "another_config", + "project": PROJECT, "archived": False, "created": "2025-02-01T12:00:00Z", "modified": "2025-02-02T12:00:00Z", @@ -51,6 +54,7 @@ def sample_config_detail_response() -> dict[str, Any]: return { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "config_yaml": CONFIG_YAML, "archived": False, "created": "2025-01-01T12:00:00Z", @@ -60,7 +64,7 @@ def sample_config_detail_response() -> dict[str, Any]: @pytest.fixture def rg_config() -> RGConfig: - return RGConfig(name="test_config", yaml=CONFIG_YAML) + return RGConfig(name="test_config", project=PROJECT, yaml=CONFIG_YAML) def test_list_rg_configs(client: DataMasqueClient, sample_config_list_response: dict[str, Any]) -> None: @@ -88,6 +92,7 @@ def test_list_rg_configs_pagination(client: DataMasqueClient) -> None: { "id": CONFIG_ID_1, "name": "c1", + "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -95,6 +100,7 @@ def test_list_rg_configs_pagination(client: DataMasqueClient) -> None: { "id": CONFIG_ID_2, "name": "c2", + "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -109,6 +115,7 @@ def test_list_rg_configs_pagination(client: DataMasqueClient) -> None: { "id": "cccccccc-1111-2222-3333-444444444444", "name": "c3", + "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -159,6 +166,7 @@ def test_get_rg_config_by_name_found(client: DataMasqueClient, sample_config_det { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -177,11 +185,11 @@ def test_get_rg_config_by_name_found(client: DataMasqueClient, sample_config_det json=sample_config_detail_response, status_code=200, ) - config = client.get_rg_config_by_name("my_config") + config = client.get_rg_config_by_name("my_config", PROJECT_ID) assert config is not None assert config.name == "my_config" - assert "name_exact=my_config" in m.request_history[0].url + assert m.request_history[0].qs == {"name_exact": ["my_config"], "project": [str(PROJECT_ID)], "limit": ["1"]} def test_get_rg_config_by_name_not_found(client: DataMasqueClient) -> None: @@ -193,7 +201,7 @@ def test_get_rg_config_by_name_not_found(client: DataMasqueClient) -> None: json=empty_response, status_code=200, ) - config = client.get_rg_config_by_name("nonexistent") + config = client.get_rg_config_by_name("nonexistent", PROJECT_ID) assert config is None @@ -206,6 +214,7 @@ def test_get_rg_config_by_name_raises_when_server_omits_id(client: DataMasqueCli "results": [ { "name": "my_config", + "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -220,13 +229,14 @@ def test_get_rg_config_by_name_raises_when_server_omits_id(client: DataMasqueCli status_code=200, ) with pytest.raises(DataMasqueApiError, match="without an `id`"): - client.get_rg_config_by_name("my_config") + client.get_rg_config_by_name("my_config", PROJECT_ID) def test_create_rg_config(client: DataMasqueClient, rg_config: RGConfig) -> None: create_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": CONFIG_YAML, "is_valid": "valid", "archived": False, @@ -250,6 +260,7 @@ def test_create_rg_config(client: DataMasqueClient, rg_config: RGConfig) -> None request_body = m.last_request.json() assert request_body["name"] == "test_config" + assert request_body["project"] == PROJECT assert request_body["config_yaml"] == CONFIG_YAML # Server-managed read-only fields must not be echoed back in the request body. assert "id" not in request_body @@ -264,6 +275,7 @@ def test_create_and_update_rg_config_populate_validation_errors(client: DataMasq invalid_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": CONFIG_YAML, "is_valid": "invalid", "validation_errors": [ @@ -312,6 +324,7 @@ def test_update_rg_config(client: DataMasqueClient, rg_config: RGConfig) -> None update_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": CONFIG_YAML, "archived": False, "created": "2025-06-01T10:00:00Z", @@ -344,7 +357,7 @@ def test_update_rg_config_no_id_raises(client: DataMasqueClient, rg_config: RGCo def test_update_rg_config_no_yaml_raises(client: DataMasqueClient) -> None: """A config from a list response carries no YAML, so a PUT would silently blank the server's copy.""" - listed = RGConfig(name="test_config") + listed = RGConfig(name="test_config", project=PROJECT) listed.id = RGConfigId(CONFIG_ID_1) with pytest.raises(ValueError, match="yaml is None"): @@ -356,6 +369,7 @@ def test_create_or_update_rg_config_create(client: DataMasqueClient, rg_config: create_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": CONFIG_YAML, "archived": False, "created": "2025-06-01T10:00:00Z", @@ -369,7 +383,9 @@ def test_create_or_update_rg_config_create(client: DataMasqueClient, rg_config: assert result.id == RGConfigId(CONFIG_ID_1) assert m.request_history[0].method == "GET" + assert m.request_history[0].qs["project"] == [PROJECT] assert m.request_history[1].method == "POST" + assert m.request_history[1].json()["project"] == PROJECT def test_create_or_update_rg_config_update(client: DataMasqueClient, rg_config: RGConfig) -> None: @@ -381,6 +397,7 @@ def test_create_or_update_rg_config_update(client: DataMasqueClient, rg_config: { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "archived": False, "created": "2025-06-01T10:00:00Z", "modified": "2025-06-01T10:00:00Z", @@ -390,6 +407,7 @@ def test_create_or_update_rg_config_update(client: DataMasqueClient, rg_config: update_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": CONFIG_YAML, "archived": False, "created": "2025-06-01T10:00:00Z", @@ -433,6 +451,7 @@ def test_delete_rg_config_by_name(client: DataMasqueClient) -> None: { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -443,10 +462,10 @@ def test_delete_rg_config_by_name(client: DataMasqueClient) -> None: with requests_mock.Mocker() as m: m.get("http://test-server/api/ruleset-generation-configs/", json=list_response, status_code=200) m.delete(f"http://test-server/api/ruleset-generation-configs/{CONFIG_ID_1}/", status_code=204) - client.delete_rg_config_by_name_if_exists("my_config") + client.delete_rg_config_by_name_if_exists("my_config", PROJECT_ID) assert [r.method for r in m.request_history] == ["GET", "DELETE"] - assert "name_exact=my_config" in m.request_history[0].url + assert m.request_history[0].qs == {"name_exact": ["my_config"], "project": [str(PROJECT_ID)], "limit": ["1"]} def test_delete_rg_config_by_name_not_found(client: DataMasqueClient) -> None: @@ -454,7 +473,7 @@ def test_delete_rg_config_by_name_not_found(client: DataMasqueClient) -> None: with requests_mock.Mocker() as m: m.get("http://test-server/api/ruleset-generation-configs/", json=empty_response, status_code=200) - client.delete_rg_config_by_name_if_exists("nonexistent") + client.delete_rg_config_by_name_if_exists("nonexistent", PROJECT_ID) assert m.call_count == 1 assert m.request_history[0].method == "GET" @@ -479,6 +498,7 @@ def test_rg_config_parses_validation_fields() -> None: { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "config_yaml": "labels: []", "is_valid": "invalid", "validation_errors": [{"message": "bad shape on line 3", "line_number": 3, "column_number": 1}], @@ -497,6 +517,7 @@ def test_rg_config_validation_fields_optional() -> None: { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", } @@ -512,11 +533,11 @@ def test_unwrap_rg_config_id_passes_through_strings() -> None: def test_unwrap_rg_config_id_extracts_id_from_model() -> None: - config = RGConfig(name="x", id=RGConfigId(CONFIG_ID_1)) + config = RGConfig(name="x", project=PROJECT, id=RGConfigId(CONFIG_ID_1)) assert unwrap_rg_config_id(config) == CONFIG_ID_1 def test_unwrap_rg_config_id_raises_without_id() -> None: - config = RGConfig(name="x") + config = RGConfig(name="x", project=PROJECT) with pytest.raises(ValueError, match="id is None"): unwrap_rg_config_id(config) diff --git a/tests/test_ruleset_library.py b/tests/test_ruleset_library.py index b3fec81..29b558a 100644 --- a/tests/test_ruleset_library.py +++ b/tests/test_ruleset_library.py @@ -15,6 +15,7 @@ ValidationStatus, ) from datamasque.client.models.status import ValidationErrorDetails +from tests.helpers import PROJECT, PROJECT_ID LIBRARY_ID_1 = "aaaaaaaa-1111-2222-3333-444444444444" LIBRARY_ID_2 = "bbbbbbbb-1111-2222-3333-444444444444" @@ -31,7 +32,7 @@ def sample_library_list_response(): { "id": LIBRARY_ID_1, "name": "my_library", - "namespace": "org", + "project": PROJECT, "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -39,7 +40,7 @@ def sample_library_list_response(): { "id": LIBRARY_ID_2, "name": "another_library", - "namespace": "", + "project": PROJECT, "is_valid": "invalid", "created": "2025-02-01T12:00:00Z", "modified": "2025-02-02T12:00:00Z", @@ -54,7 +55,7 @@ def sample_library_detail_response(): return { "id": LIBRARY_ID_1, "name": "my_library", - "namespace": "org", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions:\n - name: my_func", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", @@ -66,7 +67,7 @@ def sample_library_detail_response(): def ruleset_library(): return RulesetLibrary( name="test_library", - namespace="test_ns", + project=PROJECT, yaml="version: '1.0'\nfunctions: []", ) @@ -83,7 +84,7 @@ def test_list_ruleset_libraries(client: DataMasqueClient, sample_library_list_re assert len(libraries) == 2 assert libraries[0].id == RulesetLibraryId(LIBRARY_ID_1) assert libraries[0].name == "my_library" - assert libraries[0].namespace == "org" + assert libraries[0].project == PROJECT assert libraries[0].yaml is None assert libraries[0].is_valid is ValidationStatus.valid assert libraries[1].id == RulesetLibraryId(LIBRARY_ID_2) @@ -100,7 +101,7 @@ def test_list_ruleset_libraries_pagination(client: DataMasqueClient) -> None: { "id": LIBRARY_ID_1, "name": "lib1", - "namespace": "", + "project": PROJECT, "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -108,7 +109,7 @@ def test_list_ruleset_libraries_pagination(client: DataMasqueClient) -> None: { "id": LIBRARY_ID_2, "name": "lib2", - "namespace": "", + "project": PROJECT, "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -123,7 +124,7 @@ def test_list_ruleset_libraries_pagination(client: DataMasqueClient) -> None: { "id": "cccccccc-1111-2222-3333-444444444444", "name": "lib3", - "namespace": "", + "project": PROJECT, "is_valid": "unknown", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -169,7 +170,7 @@ def test_get_ruleset_library(client: DataMasqueClient, sample_library_detail_res assert library.id == RulesetLibraryId(LIBRARY_ID_1) assert library.name == "my_library" - assert library.namespace == "org" + assert library.project == PROJECT assert library.yaml == "version: '1.0'\nfunctions:\n - name: my_func" assert library.is_valid is ValidationStatus.valid @@ -185,7 +186,7 @@ def test_get_ruleset_library_by_name_found( { "id": LIBRARY_ID_1, "name": "my_library", - "namespace": "org", + "project": PROJECT, "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -204,13 +205,12 @@ def test_get_ruleset_library_by_name_found( json=sample_library_detail_response, status_code=200, ) - library = client.get_ruleset_library_by_name("my_library", "org") + library = client.get_ruleset_library_by_name("my_library", PROJECT_ID) assert library is not None assert library.name == "my_library" assert library.yaml == "version: '1.0'\nfunctions:\n - name: my_func" - assert "name_exact=my_library" in m.request_history[0].url - assert "namespace_exact=org" in m.request_history[0].url + assert m.request_history[0].qs == {"name_exact": ["my_library"], "project": [str(PROJECT_ID)], "limit": ["1"]} def test_get_ruleset_library_by_name_raises_when_server_omits_id(client: DataMasqueClient) -> None: @@ -222,7 +222,7 @@ def test_get_ruleset_library_by_name_raises_when_server_omits_id(client: DataMas "results": [ { "name": "my_library", - "namespace": "org", + "project": PROJECT, "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -237,7 +237,7 @@ def test_get_ruleset_library_by_name_raises_when_server_omits_id(client: DataMas status_code=200, ) with pytest.raises(DataMasqueApiError, match="without an `id`"): - client.get_ruleset_library_by_name("my_library", "org") + client.get_ruleset_library_by_name("my_library", PROJECT_ID) def test_get_ruleset_library_by_name_not_found(client: DataMasqueClient) -> None: @@ -249,7 +249,7 @@ def test_get_ruleset_library_by_name_not_found(client: DataMasqueClient) -> None json=empty_response, status_code=200, ) - library = client.get_ruleset_library_by_name("nonexistent") + library = client.get_ruleset_library_by_name("nonexistent", PROJECT_ID) assert library is None @@ -258,7 +258,7 @@ def test_create_ruleset_library(client: DataMasqueClient, ruleset_library: Rules create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "unknown", "created": "2025-06-01T10:00:00Z", @@ -281,17 +281,35 @@ def test_create_ruleset_library(client: DataMasqueClient, ruleset_library: Rules request_body = m.last_request.json() assert request_body["name"] == "test_library" - assert request_body["namespace"] == "test_ns" + assert request_body["project"] == PROJECT assert request_body["config_yaml"] == "version: '1.0'\nfunctions: []" +def test_update_ruleset_library_does_not_send_namespace_back( + client: DataMasqueClient, sample_library_detail_response: dict[str, Any] +) -> None: + """A `namespace` the server returns is not sent back, because the server lets it override `project`.""" + with requests_mock.Mocker() as m: + m.get( + f"http://test-server/api/ruleset-libraries/{LIBRARY_ID_1}/", + json={**sample_library_detail_response, "namespace": "org"}, + ) + m.put(f"http://test-server/api/ruleset-libraries/{LIBRARY_ID_1}/", json=sample_library_detail_response) + library = client.get_ruleset_library(RulesetLibraryId(LIBRARY_ID_1)) + client.update_ruleset_library(library) + + request_body = m.last_request.json() + assert "namespace" not in request_body + assert request_body["project"] == PROJECT + + def test_update_ruleset_library(client: DataMasqueClient, ruleset_library: RulesetLibrary) -> None: ruleset_library.id = RulesetLibraryId(LIBRARY_ID_1) update_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "unknown", "created": "2025-06-01T10:00:00Z", @@ -329,7 +347,7 @@ def test_create_or_update_ruleset_library_create( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "unknown", "created": "2025-06-01T10:00:00Z", @@ -368,26 +386,17 @@ def test_create_or_update_ruleset_library_update( { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "is_valid": "valid", "created": "2025-06-01T10:00:00Z", "modified": "2025-06-01T10:00:00Z", }, ], } - detail_response = { - "id": LIBRARY_ID_1, - "name": "test_library", - "namespace": "test_ns", - "config_yaml": "version: '1.0'\nfunctions: []", - "is_valid": "valid", - "created": "2025-06-01T10:00:00Z", - "modified": "2025-06-01T10:00:00Z", - } update_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "unknown", "created": "2025-06-01T10:00:00Z", @@ -400,11 +409,6 @@ def test_create_or_update_ruleset_library_update( json=list_response, status_code=200, ) - m.get( - f"http://test-server/api/ruleset-libraries/{LIBRARY_ID_1}/", - json=detail_response, - status_code=200, - ) m.put( f"http://test-server/api/ruleset-libraries/{LIBRARY_ID_1}/", json=update_response, @@ -413,10 +417,8 @@ def test_create_or_update_ruleset_library_update( result = client.create_or_update_ruleset_library(ruleset_library) assert result.id == RulesetLibraryId(LIBRARY_ID_1) - # Should have called GET (list), GET (detail), then PUT (update) - assert m.request_history[0].method == "GET" - assert m.request_history[1].method == "GET" - assert m.request_history[2].method == "PUT" + assert [request.method for request in m.request_history] == ["GET", "PUT"] + assert m.request_history[0].qs == {"name_exact": ["test_library"], "project": [PROJECT], "limit": ["1"]} def test_delete_ruleset_library_by_id(client: DataMasqueClient) -> None: @@ -453,31 +455,35 @@ def test_delete_ruleset_library_by_id_force(client: DataMasqueClient) -> None: def test_delete_ruleset_library_by_name(client: DataMasqueClient, sample_library_list_response: dict[str, Any]) -> None: with requests_mock.Mocker() as m: + # The server filters by name, so only the matching library is listed. + only_match = { + **sample_library_list_response, + "count": 1, + "results": sample_library_list_response["results"][:1], + } m.get( "http://test-server/api/ruleset-libraries/", - json=sample_library_list_response, + json=only_match, status_code=200, ) m.delete( f"http://test-server/api/ruleset-libraries/{LIBRARY_ID_1}/", status_code=204, ) - client.delete_ruleset_library_by_name_if_exists("my_library", "org") + client.delete_ruleset_library_by_name_if_exists("my_library", PROJECT_ID) - assert m.request_history[0].method == "GET" - assert m.request_history[1].method == "DELETE" + assert m.request_history[0].qs["project"] == [str(PROJECT_ID)] + assert [request.method for request in m.request_history] == ["GET", "DELETE"] -def test_delete_ruleset_library_by_name_not_found( - client: DataMasqueClient, sample_library_list_response: dict[str, Any] -) -> None: +def test_delete_ruleset_library_by_name_not_found(client: DataMasqueClient) -> None: with requests_mock.Mocker() as m: m.get( "http://test-server/api/ruleset-libraries/", - json=sample_library_list_response, + json={"count": 0, "next": None, "previous": None, "results": []}, status_code=200, ) - client.delete_ruleset_library_by_name_if_exists("nonexistent") + client.delete_ruleset_library_by_name_if_exists("nonexistent", PROJECT_ID) # Only the list call should have been made, no DELETE assert m.call_count == 1 @@ -488,7 +494,7 @@ def test_validate_ruleset_library(client: DataMasqueClient) -> None: validate_response = { "id": LIBRARY_ID_1, "name": "my_library", - "namespace": "org", + "project": PROJECT, "config_yaml": "version: '1.0'", "is_valid": "unknown", "created": "2025-01-01T12:00:00Z", @@ -517,6 +523,7 @@ def test_list_rulesets_using_library(client: DataMasqueClient) -> None: { "id": "eeeeeeee-1111-2222-3333-444444444444", "name": "ruleset_a", + "project": PROJECT, "mask_type": "database", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", @@ -525,6 +532,7 @@ def test_list_rulesets_using_library(client: DataMasqueClient) -> None: { "id": "ffffffff-1111-2222-3333-444444444444", "name": "ruleset_b", + "project": PROJECT, "mask_type": "file", "is_valid": "unknown", "created": "2025-02-01T12:00:00Z", @@ -575,6 +583,7 @@ def test_list_rulesets_using_library_pagination(client: DataMasqueClient) -> Non { "id": "aaa", "name": "r1", + "project": PROJECT, "mask_type": "database", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", @@ -583,6 +592,7 @@ def test_list_rulesets_using_library_pagination(client: DataMasqueClient) -> Non { "id": "bbb", "name": "r2", + "project": PROJECT, "mask_type": "database", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", @@ -598,6 +608,7 @@ def test_list_rulesets_using_library_pagination(client: DataMasqueClient) -> Non { "id": "ccc", "name": "r3", + "project": PROJECT, "mask_type": "file", "is_valid": "unknown", "created": "2025-01-01T12:00:00Z", @@ -626,7 +637,7 @@ def test_create_ruleset_library_populates_validation_errors( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "invalid", "validation_errors": [{"message": "Invalid function definition"}], @@ -654,7 +665,7 @@ def test_update_ruleset_library_populates_validation_errors( update_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "invalid", "validation_errors": [{"message": "Invalid function definition"}], @@ -684,7 +695,7 @@ def test_create_ruleset_library_does_not_send_read_only_fields( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "invalid", "validation_errors": [{"message": "Invalid function definition"}], @@ -715,7 +726,7 @@ def test_create_ruleset_library_collapses_git_snapshot( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "valid", "git_branch": "main", @@ -752,7 +763,7 @@ def test_create_and_update_ruleset_library_populate_unlicensed_feature_warnings( response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "valid", "unlicensed_feature_warnings": returned, diff --git a/tests/test_rulesets.py b/tests/test_rulesets.py index f304439..b0a989d 100644 --- a/tests/test_rulesets.py +++ b/tests/test_rulesets.py @@ -8,6 +8,7 @@ from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.ruleset import RulesetType from datamasque.client.models.status import ValidationErrorType, ValidationStatus +from tests.helpers import PROJECT, PROJECT_ID def test_list_rulesets(client, existing_rulesets_json): @@ -34,7 +35,7 @@ def test_create_or_update_ruleset_create(client, ruleset): # Test creating a new ruleset with upsert m.post( "http://test-server/api/rulesets/?upsert=true", - json={"id": "2", "name": "test_ruleset", "is_valid": "in_progress"}, + json={"id": "2", "name": "test_ruleset", "project": PROJECT, "is_valid": "in_progress"}, status_code=201, ) @@ -46,6 +47,7 @@ def test_create_or_update_ruleset_create(client, ruleset): sent = m.last_request.json() assert sent["config_yaml"] == "version: '1.0'\ntasks: []" assert sent["mask_type"] == "database" + assert sent["project"] == PROJECT def test_create_or_update_ruleset_create_fail(client, ruleset): @@ -64,7 +66,7 @@ def test_create_or_update_ruleset_update(client, ruleset): # Test updating an existing ruleset with upsert (returns 200 status for update) m.post( "http://test-server/api/rulesets/?upsert=true", - json={"id": "1", "name": "test_ruleset", "is_valid": "valid"}, + json={"id": "1", "name": "test_ruleset", "project": PROJECT, "is_valid": "valid"}, status_code=200, ) @@ -94,41 +96,31 @@ def test_delete_ruleset_by_id(client): def test_delete_ruleset_by_name(client, existing_rulesets_json): + """The server filters by name, type and project, so a same-named ruleset elsewhere is never returned.""" with requests_mock.Mocker() as m: m.get( - "http://test-server/api/v2/rulesets/", - json=existing_rulesets_json, + f"http://test-server/api/v2/rulesets/?name_exact=file_masking_ruleset&mask_type=file&project={PROJECT_ID}", + json=[existing_rulesets_json[1]], status_code=200, + complete_qs=True, ) m.delete("http://test-server/api/rulesets/2/", status_code=204) - client.delete_ruleset_by_name_if_exists("file_masking_ruleset", RulesetType.file) + client.delete_ruleset_by_name_if_exists("file_masking_ruleset", RulesetType.file, PROJECT_ID) assert m.call_count == 2 assert m.request_history[0].method == "GET" assert m.request_history[1].method == "DELETE" -def test_delete_ruleset_by_name_skips_other_type(client, existing_rulesets_json): - with requests_mock.Mocker() as m: - m.get( - "http://test-server/api/v2/rulesets/", - json=existing_rulesets_json, - status_code=200, - ) - client.delete_ruleset_by_name_if_exists("file_masking_ruleset", RulesetType.database) - - assert m.call_count == 1 - assert m.request_history[0].method == "GET" - - -def test_delete_ruleset_that_does_not_exist(client, existing_rulesets_json): +def test_delete_ruleset_that_does_not_exist(client): with requests_mock.Mocker() as m: m.get( - "http://test-server/api/v2/rulesets/", - json=existing_rulesets_json, + f"http://test-server/api/v2/rulesets/?name_exact=not_a_ruleset&mask_type=database&project={PROJECT_ID}", + json=[], status_code=200, + complete_qs=True, ) - client.delete_ruleset_by_name_if_exists("not_a_ruleset", RulesetType.database) + client.delete_ruleset_by_name_if_exists("not_a_ruleset", RulesetType.database, PROJECT_ID) assert m.call_count == 1 assert m.request_history[0].method == "GET" @@ -142,6 +134,7 @@ def test_create_or_update_ruleset_populates_validation_errors(client, ruleset): json={ "id": "2", "name": "test_ruleset", + "project": PROJECT, "is_valid": "invalid", "validation_errors": [ { @@ -169,7 +162,7 @@ def test_create_or_update_ruleset_validation_errors_empty_when_valid(client, rul with requests_mock.Mocker() as m: m.post( "http://test-server/api/rulesets/?upsert=true", - json={"id": "2", "name": "test_ruleset", "is_valid": "valid"}, + json={"id": "2", "name": "test_ruleset", "project": PROJECT, "is_valid": "valid"}, status_code=201, ) result = client.create_or_update_ruleset(ruleset) @@ -195,7 +188,13 @@ def test_create_or_update_ruleset_populates_unlicensed_feature_warnings(client, with requests_mock.Mocker() as m: m.post( "http://test-server/api/rulesets/?upsert=true", - json={"id": "2", "name": "test_ruleset", "is_valid": "valid", "unlicensed_feature_warnings": returned}, + json={ + "id": "2", + "name": "test_ruleset", + "project": PROJECT, + "is_valid": "valid", + "unlicensed_feature_warnings": returned, + }, status_code=201, ) result = client.create_or_update_ruleset(ruleset) @@ -208,7 +207,7 @@ def test_create_or_update_ruleset_unlicensed_feature_warnings_default_to_none(cl with requests_mock.Mocker() as m: m.post( "http://test-server/api/rulesets/?upsert=true", - json={"id": "2", "name": "test_ruleset", "is_valid": "in_progress"}, + json={"id": "2", "name": "test_ruleset", "project": PROJECT, "is_valid": "in_progress"}, status_code=201, ) result = client.create_or_update_ruleset(ruleset) @@ -225,6 +224,7 @@ def test_list_rulesets_unlicensed_feature_warnings(client): { "id": "1", "name": "unstructured_ruleset", + "project": PROJECT, "mask_type": "database", "is_valid": "valid", "unlicensed_feature_warnings": [ @@ -234,6 +234,7 @@ def test_list_rulesets_unlicensed_feature_warnings(client): { "id": "2", "name": "licensed_ruleset", + "project": PROJECT, "mask_type": "database", "is_valid": "valid", "unlicensed_feature_warnings": [], @@ -241,6 +242,7 @@ def test_list_rulesets_unlicensed_feature_warnings(client): { "id": "3", "name": "unvalidated_ruleset", + "project": PROJECT, "mask_type": "database", "is_valid": "in_progress", "unlicensed_feature_warnings": None, @@ -265,6 +267,7 @@ def test_create_or_update_ruleset_does_not_send_read_only_fields(client, ruleset json={ "id": "2", "name": "test_ruleset", + "project": PROJECT, "is_valid": "invalid", "validation_errors": [{"message": "bad", "validation_error_type": "ruleset"}], "unlicensed_feature_warnings": [ @@ -283,6 +286,7 @@ def test_create_or_update_ruleset_does_not_send_read_only_fields(client, ruleset assert read_only_field not in body # Input fields are still present. assert body["config_yaml"] == "version: '1.0'\ntasks: []" + assert body["project"] == PROJECT def test_create_or_update_ruleset_collapses_git_snapshot(client, ruleset): @@ -293,6 +297,7 @@ def test_create_or_update_ruleset_collapses_git_snapshot(client, ruleset): json={ "id": "2", "name": "test_ruleset", + "project": PROJECT, "is_valid": "valid", "git_branch": "main", "git_commit_sha": "abc123", @@ -318,6 +323,7 @@ def test_create_or_update_ruleset_git_none_when_not_synced(client, ruleset): json={ "id": "2", "name": "test_ruleset", + "project": PROJECT, "is_valid": "valid", "git_branch": None, "git_commit_sha": None, @@ -340,6 +346,7 @@ def test_list_rulesets_collapses_git_snapshot(client): { "id": "1", "name": "synced_ruleset", + "project": PROJECT, "mask_type": "database", "config_yaml": "version: '1.0'", "is_valid": "valid", @@ -351,6 +358,7 @@ def test_list_rulesets_collapses_git_snapshot(client): { "id": "2", "name": "local_ruleset", + "project": PROJECT, "mask_type": "database", "config_yaml": "version: '1.0'", "is_valid": "valid", diff --git a/tests/test_runs.py b/tests/test_runs.py index b31aa7b..85f1aff 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -23,7 +23,7 @@ UnfinishedRun, ) from datamasque.client.models.status import MaskingRunStatus -from tests.helpers import fake +from tests.helpers import PROJECT, fake def test_get_run_log(client): @@ -122,7 +122,7 @@ def unfinished_run_json( "destination_connection": destination[0] or None, "destination_connection_name": destination[1], "ruleset_name": f"ruleset_{run_id}", - "project": "1-default", + "project": PROJECT, "status": status, } @@ -385,6 +385,7 @@ def test_masking_run_request_requires_ruleset(): def test_masking_run_request_accepts_connection_config_and_ruleset_objects(): """Callers may pass full `ConnectionConfig` / `Ruleset` objects; their IDs are extracted at construction.""" connection = DatabaseConnectionConfig( + project=PROJECT, id=ConnectionId("conn-uuid"), name="prod_db", db_type=DatabaseType.postgres, @@ -394,6 +395,7 @@ def test_masking_run_request_accepts_connection_config_and_ruleset_objects(): user="masker", ) dest = DatabaseConnectionConfig( + project=PROJECT, id=ConnectionId("dest-uuid"), name="staging_db", db_type=DatabaseType.postgres, @@ -403,6 +405,7 @@ def test_masking_run_request_accepts_connection_config_and_ruleset_objects(): user="masker", ) ruleset = Ruleset( + project=PROJECT, id=RulesetId("rs-uuid"), name="my_ruleset", yaml="version: '1.0'", @@ -420,6 +423,7 @@ def test_masking_run_request_accepts_connection_config_and_ruleset_objects(): def test_masking_run_request_rejects_unpersisted_connection(): """A `ConnectionConfig` without an `id` means the caller hasn't created it yet — raise at construction.""" connection = DatabaseConnectionConfig( + project=PROJECT, name="not_yet_created", db_type=DatabaseType.postgres, host="localhost", @@ -433,7 +437,7 @@ def test_masking_run_request_rejects_unpersisted_connection(): def test_masking_run_request_rejects_unpersisted_ruleset(): """Same check on the ruleset side.""" - ruleset = Ruleset(name="fresh_ruleset", yaml="version: '1.0'", ruleset_type=RulesetType.database) + ruleset = Ruleset(project=PROJECT, name="fresh_ruleset", yaml="version: '1.0'", ruleset_type=RulesetType.database) with pytest.raises(ValueError, match="has not been created yet"): MaskingRunRequest(connection="conn-1", ruleset=ruleset) diff --git a/tests/test_table_references.py b/tests/test_table_references.py index 1a7968f..a2af76d 100644 --- a/tests/test_table_references.py +++ b/tests/test_table_references.py @@ -15,7 +15,7 @@ TableReferenceOptions, ) from datamasque.client.exceptions import DataMasqueException -from tests.helpers import file_connection_config +from tests.helpers import PROJECT, PROJECT_ID, file_connection_config TABLE_REFERENCE_ID_1 = "aaaaaaaa-1111-2222-3333-444444444444" TABLE_REFERENCE_ID_2 = "bbbbbbbb-1111-2222-3333-444444444444" @@ -38,6 +38,7 @@ def sample_table_reference_list_response() -> list[dict[str, object]]: { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -47,6 +48,7 @@ def sample_table_reference_list_response() -> list[dict[str, object]]: { "id": TABLE_REFERENCE_ID_2, "name": "account_identities", + "project": PROJECT, "connection": CONNECTION_ID_2, "source": "OPS.ACCOUNTS", "options": { @@ -66,6 +68,7 @@ def sample_table_reference_list_response() -> list[dict[str, object]]: def table_reference() -> TableReference: return TableReference( name="customer_identities", + project=PROJECT, connection=ConnectionId(CONNECTION_ID_1), source="identities/customers.csv", ) @@ -73,7 +76,7 @@ def table_reference() -> TableReference: def test_name_connection_and_source_are_required_on_the_model() -> None: with pytest.raises(ValidationError, match="source"): - TableReference(name="customer_identities", connection=ConnectionId(CONNECTION_ID_1)) + TableReference(name="customer_identities", project=PROJECT, connection=ConnectionId(CONNECTION_ID_1)) def test_options_default_to_the_servers_defaults() -> None: @@ -90,7 +93,9 @@ def test_connection_accepts_a_connection_config() -> None: connection = file_connection_config() connection.id = ConnectionId(CONNECTION_ID_1) - reference = TableReference(name="customer_identities", connection=connection, source="customers.csv") + reference = TableReference( + name="customer_identities", project=PROJECT, connection=connection, source="customers.csv" + ) assert reference.connection == ConnectionId(CONNECTION_ID_1) assert reference.connection_id == ConnectionId(CONNECTION_ID_1) @@ -98,7 +103,9 @@ def test_connection_accepts_a_connection_config() -> None: def test_connection_config_without_an_id_is_rejected() -> None: with pytest.raises(ValidationError, match="id is None"): - TableReference(name="customer_identities", connection=file_connection_config(), source="customers.csv") + TableReference( + name="customer_identities", project=PROJECT, connection=file_connection_config(), source="customers.csv" + ) def test_list_table_references( @@ -130,6 +137,21 @@ def test_list_table_references( assert references[1].options.null_string == "NULL" +def test_list_table_references_in_a_project( + client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, object]] +) -> None: + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/table-references/", + json=sample_table_reference_list_response, + status_code=200, + ) + references = client.list_table_references(PROJECT_ID) + + assert m.last_request.qs == {"project": [str(PROJECT_ID)]} + assert [reference.project for reference in references] == [PROJECT, PROJECT] + + def test_list_table_references_empty(client: DataMasqueClient) -> None: with requests_mock.Mocker() as m: m.get( @@ -168,11 +190,12 @@ def test_get_table_reference_by_name_found( json=sample_table_reference_list_response, status_code=200, ) - reference = client.get_table_reference_by_name("account_identities") + reference = client.get_table_reference_by_name("account_identities", PROJECT_ID) assert reference is not None assert reference.id == TableReferenceId(TABLE_REFERENCE_ID_2) assert m.call_count == 1 + assert m.last_request.qs == {"project": [str(PROJECT_ID)]} def test_get_table_reference_by_name_not_found( @@ -184,7 +207,7 @@ def test_get_table_reference_by_name_not_found( json=sample_table_reference_list_response, status_code=200, ) - reference = client.get_table_reference_by_name("nonexistent") + reference = client.get_table_reference_by_name("nonexistent", PROJECT_ID) assert reference is None @@ -193,6 +216,7 @@ def test_create_table_reference(client: DataMasqueClient, table_reference: Table create_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -218,6 +242,7 @@ def test_create_table_reference(client: DataMasqueClient, table_reference: Table request_body = m.last_request.json() assert request_body["name"] == "customer_identities" + assert request_body["project"] == PROJECT assert request_body["connection"] == CONNECTION_ID_1 assert request_body["source"] == "identities/customers.csv" assert "options" not in request_body @@ -225,9 +250,32 @@ def test_create_table_reference(client: DataMasqueClient, table_reference: Table assert not read_only_fields & request_body.keys() +def test_create_table_reference_without_a_project(client: DataMasqueClient) -> None: + """No `project` is sent, so the server uses its default project, which is read back.""" + reference = TableReference( + name="customer_identities", connection=ConnectionId(CONNECTION_ID_1), source="identities/customers.csv" + ) + create_response = { + "id": TABLE_REFERENCE_ID_1, + "name": "customer_identities", + "project": PROJECT, + "connection": CONNECTION_ID_1, + "source": "identities/customers.csv", + "options": DEFAULT_OPTIONS_JSON, + } + + with requests_mock.Mocker() as m: + m.post("http://test-server/api/table-references/", json=create_response, status_code=201) + client.create_table_reference(reference) + + assert "project" not in m.last_request.json() + assert reference.project == PROJECT + + def test_create_table_reference_sends_options(client: DataMasqueClient) -> None: reference = TableReference( name="customer_identities", + project=PROJECT, connection=ConnectionId(CONNECTION_ID_1), source="identities/customers.csv", options=TableReferenceOptions(format=TableReferenceFormat.parquet, delimiter=";", null_string="NULL"), @@ -235,6 +283,7 @@ def test_create_table_reference_sends_options(client: DataMasqueClient) -> None: create_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": { @@ -266,6 +315,7 @@ def test_create_table_reference_sends_null_string_default(client: DataMasqueClie """`null_string`'s default of `None` must be sent explicitly, not dropped from the payload.""" reference = TableReference( name="customer_identities", + project=PROJECT, connection=ConnectionId(CONNECTION_ID_1), source="identities/customers.csv", options=TableReferenceOptions(), @@ -273,6 +323,7 @@ def test_create_table_reference_sends_null_string_default(client: DataMasqueClie create_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -300,6 +351,7 @@ def test_update_table_reference(client: DataMasqueClient, table_reference: Table update_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers_v2.csv", "options": DEFAULT_OPTIONS_JSON, @@ -333,6 +385,7 @@ def test_update_table_reference_without_options_sends_none( update_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": {**DEFAULT_OPTIONS_JSON, "delimiter": ";"}, @@ -363,6 +416,7 @@ def test_create_or_update_table_reference_create(client: DataMasqueClient, table create_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -385,7 +439,9 @@ def test_create_or_update_table_reference_create(client: DataMasqueClient, table assert result.id == TableReferenceId(TABLE_REFERENCE_ID_1) assert m.request_history[0].method == "GET" + assert m.request_history[0].qs == {"project": [PROJECT]} assert m.request_history[1].method == "POST" + assert m.request_history[1].json()["project"] == PROJECT def test_create_or_update_table_reference_update( @@ -396,6 +452,7 @@ def test_create_or_update_table_reference_update( update_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -428,6 +485,7 @@ def test_create_or_update_table_reference_raises_when_server_omits_id( list_response_without_id = [ { "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -479,10 +537,11 @@ def test_delete_table_reference_by_name( f"http://test-server/api/table-references/{TABLE_REFERENCE_ID_2}/", status_code=204, ) - client.delete_table_reference_by_name_if_exists("account_identities") + client.delete_table_reference_by_name_if_exists("account_identities", PROJECT_ID) assert m.call_count == 2 assert m.request_history[0].method == "GET" + assert m.request_history[0].qs == {"project": [str(PROJECT_ID)]} assert m.request_history[1].method == "DELETE" assert TABLE_REFERENCE_ID_2 in m.request_history[1].url @@ -496,7 +555,7 @@ def test_delete_table_reference_by_name_not_found( json=sample_table_reference_list_response, status_code=200, ) - client.delete_table_reference_by_name_if_exists("nonexistent") + client.delete_table_reference_by_name_if_exists("nonexistent", PROJECT_ID) assert m.call_count == 1 assert m.request_history[0].method == "GET" From f2a2c10356921a4e7810c819396287eb03e64395 Mon Sep 17 00:00:00 2001 From: jiatolentino Date: Thu, 22 Oct 2026 12:01:55 +0800 Subject: [PATCH 09/22] feat: DM-4388: add a client-wide default_project for creates `DataMasqueInstanceConfig.default_project` names the project a create goes into when its model has no `project`. Every create of a connection, ruleset, discovery config, RG config, table reference or library sends it, and `create_or_update_*` looks for an existing resource in it, before falling back to the server's only active project. Updating a connection leaves it in its project. --- HISTORY.rst | 5 + datamasque/client/base.py | 19 ++- datamasque/client/connections.py | 3 +- .../client/discovery_config_libraries.py | 6 +- datamasque/client/discovery_configs.py | 6 +- datamasque/client/models/connection.py | 2 +- datamasque/client/models/discovery_config.py | 2 +- .../client/models/discovery_config_library.py | 2 +- datamasque/client/models/dm_instance.py | 6 + datamasque/client/models/rg_config.py | 2 +- datamasque/client/models/ruleset.py | 2 +- datamasque/client/models/ruleset_library.py | 2 +- datamasque/client/models/table_reference.py | 2 +- datamasque/client/rg_configs.py | 6 +- datamasque/client/ruleset_libraries.py | 6 +- datamasque/client/rulesets.py | 6 +- datamasque/client/table_references.py | 3 +- tests/test_default_project.py | 137 ++++++++++++++++++ 18 files changed, 196 insertions(+), 21 deletions(-) create mode 100644 tests/test_default_project.py diff --git a/HISTORY.rst b/HISTORY.rst index 7a7a4ae..9194c17 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,11 @@ History * **Breaking:** ``get_unfinished_runs`` returns its runs keyed by ``ConnectionId`` instead of connection name, because two connections in different projects can share a name. * ``list_connections``, ``list_table_references`` and ``list_files_of_type`` take an optional ``project`` filter. +* Added ``default_project`` to ``DataMasqueInstanceConfig``. + A create whose model has no ``project`` goes into ``default_project``, + and ``create_or_update_*`` looks for an existing resource there, + before falling back to the server's only active project. + Updating a connection leaves it in its project. * ``DataMasqueFile`` gains a read-only ``project`` field, which is ``None`` for built-in seeds. * ``get_generated_rulesets`` sets each ruleset's ``project`` to the connection's project. * Added project models and management APIs. diff --git a/datamasque/client/base.py b/datamasque/client/base.py index 68db356..d45ebb7 100644 --- a/datamasque/client/base.py +++ b/datamasque/client/base.py @@ -133,6 +133,7 @@ class BaseClient: password: Optional[str] verify_ssl: bool token_source: Optional[Callable[[], str]] + default_project: Optional[str] def __init__(self, connection_config: DataMasqueInstanceConfig) -> None: self.base_url = connection_config.base_url @@ -140,6 +141,7 @@ def __init__(self, connection_config: DataMasqueInstanceConfig) -> None: self.password = connection_config.password self.verify_ssl = connection_config.verify_ssl self.token_source = connection_config.token_source + self.default_project = connection_config.default_project self._session = _build_session(self.verify_ssl) self._timeout = (connection_config.connect_timeout_seconds, connection_config.read_timeout_seconds) if connection_config.spcs_pat: @@ -318,15 +320,17 @@ def _delete_if_exists(self, path: str, *, params: Optional[dict] = None) -> None def _resolve_project(self, project: Optional[Union[ProjectId, str]]) -> Union[ProjectId, str]: """ - Returns `project`, or when it is `None`, the ID of the only active project. + Returns `project`, else the client's `default_project`, else the ID of the only active project. - That is the project the server puts a create into when it is sent no `project`. + The last is the project the server puts a create into when it is sent no `project`. With no active project, or more than one, the server rejects such a create, so this raises `DataMasqueException` instead. """ if project is not None: return project + if self.default_project is not None: + return self.default_project response = self.make_request("GET", "/api/projects/", params={"limit": 1}) page = Page[Project].model_validate(response.json()) @@ -337,6 +341,17 @@ def _resolve_project(self, project: Optional[Union[ProjectId, str]]) -> Union[Pr return page.results[0].id + def _default_project_field(self, project: Optional[Union[ProjectId, str]]) -> dict[str, str]: + """ + Returns the `project` field a create sends when the resource names no `project`: the client's `default_project`. + + Empty when the resource names one, which its own fields carry, or when there is no `default_project`. + """ + + if project is None and self.default_project is not None: + return {"project": self.default_project} + return {} + def _iter_paginated( self, path: str, diff --git a/datamasque/client/connections.py b/datamasque/client/connections.py index ddf4570..20e383d 100644 --- a/datamasque/client/connections.py +++ b/datamasque/client/connections.py @@ -35,7 +35,7 @@ def create_or_update_connection(self, connection_config: ConnectionConfig) -> Co Creates or updates the connection in DM, and sets the `id` field on the given `connection_config`. An existing connection with the same name in the same project is updated. - With no `project` set, the server's only active project is used, + With no `project` set, the client's `default_project` is used, else the server's only active project, and `DataMasqueException` is raised when it has no active project or more than one. """ @@ -53,6 +53,7 @@ def create_or_update_connection(self, connection_config: ConnectionConfig) -> Co "version": "1.0", } | connection_config.model_dump(exclude_none=True, by_alias=True, mode="json") if connection_id is None: + data |= self._default_project_field(connection_config.project) response = self.make_request("POST", "/api/connections/", data=data) else: response = self.make_request("PUT", f"/api/connections/{connection_id}/", data=data) diff --git a/datamasque/client/discovery_config_libraries.py b/datamasque/client/discovery_config_libraries.py index 781d277..d43dc9c 100644 --- a/datamasque/client/discovery_config_libraries.py +++ b/datamasque/client/discovery_config_libraries.py @@ -81,7 +81,9 @@ def create_discovery_config_library(self, library: DiscoveryConfigLibrary) -> Di if not library.yaml: raise ValueError("Cannot create a discovery config library without YAML content (yaml is empty)") - data = library.model_dump(exclude_none=True, by_alias=True, mode="json") + data = library.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( + library.project + ) response = self.make_request("POST", "/api/discovery/config-libraries/", data=data) created = DiscoveryConfigLibrary.model_validate(response.json()) library.id = created.id @@ -125,7 +127,7 @@ def create_or_update_discovery_config_library(self, library: DiscoveryConfigLibr """ Creates the library, or updates the existing one with the same name in the same project. - With no `project` set, the server's only active project is used, + With no `project` set, the client's `default_project` is used, else the server's only active project, and `DataMasqueException` is raised when it has no active project or more than one. Sets the library's `id` property. """ diff --git a/datamasque/client/discovery_configs.py b/datamasque/client/discovery_configs.py index cea748f..f0eb76e 100644 --- a/datamasque/client/discovery_configs.py +++ b/datamasque/client/discovery_configs.py @@ -86,7 +86,9 @@ def create_discovery_config(self, config: DiscoveryConfig) -> DiscoveryConfig: if not config.yaml: raise ValueError("Cannot create a discovery config without YAML content (yaml is empty)") - data = config.model_dump(exclude_none=True, by_alias=True, mode="json") + data = config.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( + config.project + ) response = self.make_request("POST", "/api/discovery/configs/", data=data) created = DiscoveryConfig.model_validate(response.json()) config.id = created.id @@ -130,7 +132,7 @@ def create_or_update_discovery_config(self, config: DiscoveryConfig) -> Discover """ Creates the config, or updates the existing one with the same name and type in the same project. - With no `project` set, the server's only active project is used, + With no `project` set, the client's `default_project` is used, else the server's only active project, and `DataMasqueException` is raised when it has no active project or more than one. Sets the config's `id` property. """ diff --git a/datamasque/client/models/connection.py b/datamasque/client/models/connection.py index 086b166..cde6bd0 100644 --- a/datamasque/client/models/connection.py +++ b/datamasque/client/models/connection.py @@ -125,7 +125,7 @@ class ConnectionConfig(BaseModel): """ The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, or the project's ID as a string. - Left as `None`, a create goes into the server's default project. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. """ id: Optional[ConnectionId] = None # Server-populated and read-only, so it is excluded when serializing a config back to the API. diff --git a/datamasque/client/models/discovery_config.py b/datamasque/client/models/discovery_config.py index 6a5a909..3d767bb 100644 --- a/datamasque/client/models/discovery_config.py +++ b/datamasque/client/models/discovery_config.py @@ -44,7 +44,7 @@ class DiscoveryConfig(BaseModel): """ The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, or the project's ID as a string. - Left as `None`, a create goes into the server's default project. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. """ yaml: Optional[str] = Field(default=None, alias="config_yaml") config_type: DiscoveryConfigType diff --git a/datamasque/client/models/discovery_config_library.py b/datamasque/client/models/discovery_config_library.py index c4ef39d..06058bb 100644 --- a/datamasque/client/models/discovery_config_library.py +++ b/datamasque/client/models/discovery_config_library.py @@ -24,7 +24,7 @@ class DiscoveryConfigLibrary(BaseModel): """ The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, or the project's ID as a string. - Left as `None`, a create goes into the server's default project. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. """ yaml: Optional[str] = Field(default=None, alias="config_yaml") # Server-populated read-only fields, excluded from request bodies. diff --git a/datamasque/client/models/dm_instance.py b/datamasque/client/models/dm_instance.py index 7b30a98..0f82152 100644 --- a/datamasque/client/models/dm_instance.py +++ b/datamasque/client/models/dm_instance.py @@ -49,6 +49,12 @@ class DataMasqueInstanceConfig(BaseModel): """How long a request may take to connect, including the TLS handshake.""" read_timeout_seconds: float = DEFAULT_READ_TIMEOUT_SECONDS """How long a request may wait for the server between bytes of its response.""" + default_project: Optional[str] = None + """The project a create goes into when the resource names none, as a `-` reference (e.g. `1-default`) + or a bare project id. + + Applies to connections, rulesets, discovery configs, RG configs, table references and libraries. + Leave unset to use the server's only active project.""" @model_validator(mode="after") def _validate_auth_source(self) -> "DataMasqueInstanceConfig": diff --git a/datamasque/client/models/rg_config.py b/datamasque/client/models/rg_config.py index f5de6ac..441306b 100644 --- a/datamasque/client/models/rg_config.py +++ b/datamasque/client/models/rg_config.py @@ -43,7 +43,7 @@ class RGConfig(BaseModel): """ The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, or the project's ID as a string. - Left as `None`, a create goes into the server's default project. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. """ yaml: Optional[str] = Field(default=None, alias="config_yaml") diff --git a/datamasque/client/models/ruleset.py b/datamasque/client/models/ruleset.py index 2c6275d..a42be76 100644 --- a/datamasque/client/models/ruleset.py +++ b/datamasque/client/models/ruleset.py @@ -42,7 +42,7 @@ class Ruleset(GitTrackedEntity): """ The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, or the project's ID as a string. - Left as `None`, a create goes into the server's default project. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. """ yaml: str = Field(default="", alias="config_yaml") ruleset_type: RulesetType = Field(default=RulesetType.database, alias="mask_type") diff --git a/datamasque/client/models/ruleset_library.py b/datamasque/client/models/ruleset_library.py index d0b75d9..dd54b20 100644 --- a/datamasque/client/models/ruleset_library.py +++ b/datamasque/client/models/ruleset_library.py @@ -18,7 +18,7 @@ class RulesetLibrary(GitTrackedEntity): """ The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, or the project's ID as a string. - Left as `None`, a create goes into the server's default project. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. """ yaml: Optional[str] = Field(default=None, alias="config_yaml") diff --git a/datamasque/client/models/table_reference.py b/datamasque/client/models/table_reference.py index 3bcac90..5b6c9b5 100644 --- a/datamasque/client/models/table_reference.py +++ b/datamasque/client/models/table_reference.py @@ -77,7 +77,7 @@ class TableReference(BaseModel): """ The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, or the project's ID as a string. - Left as `None`, a create goes into the server's default project. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. """ connection: Union[ConnectionId, ConnectionConfig] """The connection the data lives in. Accepts a `ConnectionId` or a `ConnectionConfig` (its `id` is extracted).""" diff --git a/datamasque/client/rg_configs.py b/datamasque/client/rg_configs.py index 4ba96fe..ea94237 100644 --- a/datamasque/client/rg_configs.py +++ b/datamasque/client/rg_configs.py @@ -78,7 +78,9 @@ def create_rg_config(self, config: RGConfig) -> RGConfig: (`id`, `is_valid`, `validation_errors`, `created`, `modified`) and returns the config. """ - data = config.model_dump(exclude_none=True, by_alias=True, mode="json") + data = config.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( + config.project + ) response = self.make_request("POST", "/api/ruleset-generation-configs/", data=data) created = RGConfig.model_validate(response.json()) config.id = created.id @@ -121,7 +123,7 @@ def create_or_update_rg_config(self, config: RGConfig) -> RGConfig: """ Creates the config, or updates the existing one with the same name in the same project. - With no `project` set, the server's only active project is used, + With no `project` set, the client's `default_project` is used, else the server's only active project, and `DataMasqueException` is raised when it has no active project or more than one. Sets the config's `id` property. """ diff --git a/datamasque/client/ruleset_libraries.py b/datamasque/client/ruleset_libraries.py index e03b6c2..b029ead 100644 --- a/datamasque/client/ruleset_libraries.py +++ b/datamasque/client/ruleset_libraries.py @@ -79,7 +79,9 @@ def create_ruleset_library(self, library: RulesetLibrary) -> RulesetLibrary: and returns the library. """ - data = library.model_dump(exclude_none=True, by_alias=True, mode="json") + data = library.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( + library.project + ) response = self.make_request("POST", "/api/ruleset-libraries/", data=data) created_library = RulesetLibrary.model_validate(response.json()) library.id = created_library.id @@ -118,7 +120,7 @@ def create_or_update_ruleset_library(self, library: RulesetLibrary) -> RulesetLi """ Creates the library, or updates the existing one with the same name in the same project. - With no `project` set, the server's only active project is used, + With no `project` set, the client's `default_project` is used, else the server's only active project, and `DataMasqueException` is raised when it has no active project or more than one. Sets the library's `id` property. """ diff --git a/datamasque/client/rulesets.py b/datamasque/client/rulesets.py index 9a5159c..ed82ef8 100644 --- a/datamasque/client/rulesets.py +++ b/datamasque/client/rulesets.py @@ -22,7 +22,7 @@ def create_or_update_ruleset(self, ruleset: Ruleset) -> Ruleset: """ Creates the ruleset, or updates the existing one with the same name and type in the same project. - With no `project` set, the server's only active project is used, + With no `project` set, the client's `default_project` is used, else the server's only active project, and `DataMasqueException` is raised when it has no active project or more than one. Populates the given ruleset's `id`, `is_valid`, `validation_errors`, `unlicensed_feature_warnings`, and `git` fields from the server response, and returns the same ruleset instance for convenience. @@ -30,7 +30,9 @@ def create_or_update_ruleset(self, ruleset: Ruleset) -> Ruleset: # The server finds the existing ruleset itself, so this only checks that an unset project resolves. self._resolve_project(ruleset.project) - data = ruleset.model_dump(exclude_none=True, by_alias=True, mode="json") + data = ruleset.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( + ruleset.project + ) response = self.make_request("POST", "/api/rulesets/", data=data, params={"upsert": "true"}) created = Ruleset.model_validate(response.json()) ruleset.id = created.id diff --git a/datamasque/client/table_references.py b/datamasque/client/table_references.py index a3c6ff2..6d5f5b8 100644 --- a/datamasque/client/table_references.py +++ b/datamasque/client/table_references.py @@ -81,6 +81,7 @@ def create_table_reference(self, table_reference: TableReference) -> TableRefere data.pop("options", None) if data.get("project") is None: data.pop("project", None) + data |= self._default_project_field(table_reference.project) response = self.make_request("POST", "/api/table-references/", data=data) created = TableReference.model_validate(response.json()) table_reference.id = created.id @@ -122,7 +123,7 @@ def create_or_update_table_reference(self, table_reference: TableReference) -> T """ Creates the table reference, or updates the existing one with the same name in the same project. - With no `project` set, the server's only active project is used, + With no `project` set, the client's `default_project` is used, else the server's only active project, and `DataMasqueException` is raised when it has no active project or more than one. Sets the table reference's `id` property. """ diff --git a/tests/test_default_project.py b/tests/test_default_project.py new file mode 100644 index 0000000..899ad5b --- /dev/null +++ b/tests/test_default_project.py @@ -0,0 +1,137 @@ +"""Tests for the client's `default_project`, which a create uses when the resource names no project.""" + +from collections.abc import Callable +from typing import Any, Optional + +import pytest +import requests_mock + +from datamasque.client import DataMasqueClient +from datamasque.client.models.connection import ConnectionId +from datamasque.client.models.discovery_config import DiscoveryConfig +from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary +from datamasque.client.models.dm_instance import DataMasqueInstanceConfig +from datamasque.client.models.rg_config import RGConfig +from datamasque.client.models.ruleset import Ruleset, RulesetType +from datamasque.client.models.ruleset_library import RulesetLibrary +from datamasque.client.models.table_reference import TableReference +from tests.helpers import active_projects_page, database_connection_config + +BASE_URL = "http://test-server" + + +def make_client(default_project: Optional[str] = None) -> DataMasqueClient: + return DataMasqueClient( + DataMasqueInstanceConfig(base_url=BASE_URL, username="u", password="p", default_project=default_project) + ) + + +def create_ruleset(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post(f"{BASE_URL}/api/rulesets/?upsert=true", json={"id": "r-1", "name": "r", "project": "2-acme"}) + ruleset = Ruleset(name="r", yaml="version: '1.0'", ruleset_type=RulesetType.database, project=project) + return client.create_or_update_ruleset(ruleset) + + +def create_connection(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.get(f"{BASE_URL}/api/connections/", json=[]) + m.post(f"{BASE_URL}/api/connections/", json={"id": "c-1", "project": "2-acme"}) + connection = database_connection_config() + connection.project = project + return client.create_or_update_connection(connection) + + +def create_discovery_config(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post( + f"{BASE_URL}/api/discovery/configs/", + json={"id": "dc-1", "name": "dc", "config_type": "database", "project": "2-acme"}, + ) + config = DiscoveryConfig(name="dc", yaml="labels: []\n", config_type="database", project=project) + return client.create_discovery_config(config) + + +def create_rg_config(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post(f"{BASE_URL}/api/ruleset-generation-configs/", json={"id": "rg-1", "name": "rg", "project": "2-acme"}) + return client.create_rg_config(RGConfig(name="rg", yaml="x: 1\n", project=project)) + + +def create_table_reference(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post( + f"{BASE_URL}/api/table-references/", + json={"id": "tr-1", "name": "tr", "connection": "c-1", "source": "people.csv", "project": "2-acme"}, + ) + table_reference = TableReference(name="tr", connection=ConnectionId("c-1"), source="people.csv", project=project) + return client.create_table_reference(table_reference) + + +def create_ruleset_library(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post(f"{BASE_URL}/api/ruleset-libraries/", json={"id": "rl-1", "name": "rl", "project": "2-acme"}) + return client.create_ruleset_library(RulesetLibrary(name="rl", yaml="version: '1.0'\n", project=project)) + + +def create_discovery_config_library(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post(f"{BASE_URL}/api/discovery/config-libraries/", json={"id": "dl-1", "name": "dl", "project": "2-acme"}) + return client.create_discovery_config_library( + DiscoveryConfigLibrary(name="dl", yaml="labels: []\n", project=project) + ) + + +Create = Callable[[DataMasqueClient, Optional[str], requests_mock.Mocker], Any] + +CREATES = [ + pytest.param(create_ruleset, id="ruleset"), + pytest.param(create_connection, id="connection"), + pytest.param(create_discovery_config, id="discovery_config"), + pytest.param(create_rg_config, id="rg_config"), + pytest.param(create_table_reference, id="table_reference"), + pytest.param(create_ruleset_library, id="ruleset_library"), + pytest.param(create_discovery_config_library, id="discovery_config_library"), +] + + +@pytest.mark.parametrize("create", CREATES) +def test_a_create_sends_the_resources_project_over_the_default(create: Create) -> None: + with requests_mock.Mocker() as m: + create(make_client(default_project="1-default"), "2-acme", m) + + assert m.last_request.json()["project"] == "2-acme" + + +@pytest.mark.parametrize("create", CREATES) +def test_a_create_sends_the_default_project_when_the_resource_names_none(create: Create) -> None: + with requests_mock.Mocker() as m: + create(make_client(default_project="1-default"), None, m) + + assert m.last_request.json()["project"] == "1-default" + assert all(request.path != "/api/projects/" for request in m.request_history) + + +@pytest.mark.parametrize("create", CREATES) +def test_a_create_with_no_default_project_sends_no_project(create: Create) -> None: + with requests_mock.Mocker() as m: + m.get(f"{BASE_URL}/api/projects/", json=active_projects_page(1)) + create(make_client(), None, m) + + assert "project" not in m.last_request.json() + + +def test_a_connection_update_sends_no_project() -> None: + """An update leaves the connection in its own project, whatever the default.""" + client = make_client(default_project="1-default") + connection = database_connection_config() + connection.project = None + with requests_mock.Mocker() as m: + m.get(f"{BASE_URL}/api/connections/", json=[{**connection.model_dump(mode="json"), "id": "c-1"}]) + m.put(f"{BASE_URL}/api/connections/c-1/", json={"id": "c-1", "project": "2-acme"}) + + client.create_or_update_connection(connection) + + assert m.last_request.method == "PUT" + assert "project" not in m.last_request.json() + + +def test_an_upsert_looks_for_the_name_in_the_default_project() -> None: + with requests_mock.Mocker() as m: + create_connection(make_client(default_project="1-default"), None, m) + + lookup = m.request_history[0] + assert (lookup.method, lookup.path, lookup.qs) == ("GET", "/api/connections/", {"project": ["1-default"]}) From 155767ced517b83a514b79b6f5c75528c87d0d03 Mon Sep 17 00:00:00 2001 From: Colin Haywood Date: Tue, 20 Oct 2026 16:33:28 +1300 Subject: [PATCH 10/22] feat: add move_resources and get_resource_move_conflicts - `list_resources` keys its groups by `ResourceTypeName`, so its result can be passed straight to `move_resources`. - `ProjectDeleteConflict.type` is a `ResourceTypeName`. --- HISTORY.rst | 2 +- datamasque/client/__init__.py | 10 ++- datamasque/client/models/project.py | 8 +- datamasque/client/models/resource.py | 62 ++++++++++----- datamasque/client/models/resource_type.py | 19 +++++ datamasque/client/resources.py | 92 +++++++++++++++++++++-- docs/client.models.rst | 8 ++ tests/test_resources.py | 73 +++++++++++++++++- 8 files changed, 242 insertions(+), 32 deletions(-) create mode 100644 datamasque/client/models/resource_type.py diff --git a/HISTORY.rst b/HISTORY.rst index 9194c17..3357a37 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -31,7 +31,7 @@ History * ``DataMasqueFile`` gains a read-only ``project`` field, which is ``None`` for built-in seeds. * ``get_generated_rulesets`` sets each ruleset's ``project`` to the connection's project. * Added project models and management APIs. -* Added ``list_resources``, which lists resources of every type across projects. +* Added resource management APIs. 1.6.0 (unreleased) ------------------ diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index e4ba6c3..f5d7767 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -127,9 +127,13 @@ ConnectionResource, DiscoveryConfigResource, Resource, - ResourceTypeName, + ResourceMoveConflict, + ResourceMoveConflictReason, + ResourceMoveResult, + ResourcesByType, RulesetResource, ) +from datamasque.client.models.resource_type import ResourceTypeName from datamasque.client.models.rg_config import RGConfig, RGConfigId from datamasque.client.models.ruleset import Ruleset, RulesetId, RulesetType from datamasque.client.models.ruleset_library import RulesetLibrary, RulesetLibraryId @@ -297,7 +301,11 @@ "RGConfigNotFoundError", "ReferencingForeignKey", "Resource", + "ResourceMoveConflict", + "ResourceMoveConflictReason", + "ResourceMoveResult", "ResourceTypeName", + "ResourcesByType", "Ruleset", "RulesetGenerationRequest", "RulesetGenerationWithRGConfigRequest", diff --git a/datamasque/client/models/project.py b/datamasque/client/models/project.py index 758314d..727d00f 100644 --- a/datamasque/client/models/project.py +++ b/datamasque/client/models/project.py @@ -6,6 +6,8 @@ from pydantic import BaseModel, ConfigDict +from datamasque.client.models.resource_type import ResourceTypeName + ProjectId = NewType("ProjectId", int) @@ -76,10 +78,8 @@ class ProjectDeleteConflict(BaseModel): reason: ProjectDeleteConflictReason detail: str """A message that describes the conflict.""" - type: Optional[str] = None - """ - For `name_in_use` and `file_exists`: the type of the resource that would move, such as `connections` or `rulesets`. - """ + type: Optional[ResourceTypeName] = None + """For `name_in_use` and `file_exists`: the type of the resource that would move.""" id: Optional[str] = None """For `name_in_use` and `file_exists`: the ID of the resource that would move.""" name: Optional[str] = None diff --git a/datamasque/client/models/resource.py b/datamasque/client/models/resource.py index a4abec2..0ba85d4 100644 --- a/datamasque/client/models/resource.py +++ b/datamasque/client/models/resource.py @@ -1,31 +1,17 @@ """Resource listing models for the DataMasque API.""" +from collections.abc import Mapping, Sequence from datetime import datetime from enum import Enum -from typing import Optional +from typing import Optional, Union from pydantic import BaseModel, ConfigDict from datamasque.client.models.discovery_config import DiscoveryConfigType +from datamasque.client.models.resource_type import ResourceTypeName from datamasque.client.models.runs import MaskType -class ResourceTypeName(str, Enum): - """A type of resource that belongs to a project, named as the resources listing groups it.""" - - connections = "connections" - connection_filesets = "connection_filesets" - oracle_wallets = "oracle_wallets" - snowflake_keys = "snowflake_keys" - table_references = "table_references" - rulesets = "rulesets" - ruleset_libraries = "ruleset_libraries" - discovery_configs = "discovery_configs" - discovery_config_libraries = "discovery_config_libraries" - ruleset_generation_configs = "ruleset_generation_configs" - seeds = "seeds" - - class Resource(BaseModel): """ One resource in the resources listing: the summary the listing gives, not the resource's full model. @@ -69,3 +55,45 @@ class DiscoveryConfigResource(Resource): ResourceTypeName.rulesets.value: RulesetResource, ResourceTypeName.discovery_configs.value: DiscoveryConfigResource, } + +# Resources by type, each given as a listed `Resource` or its ID, in the shape `list_resources` returns. +ResourcesByType = Mapping[ResourceTypeName, Sequence[Union[Resource, str]]] + + +class ResourceMoveConflictReason(str, Enum): + """Why a resource cannot move to the destination project.""" + + already_in_destination = "already_in_destination" + """The resource is already in the destination project.""" + name_in_use = "name_in_use" + """The destination project already contains a resource of the same type and name.""" + name_in_selection = "name_in_selection" + """Another resource of the same type and name is moving too, from another project.""" + file_exists = "file_exists" + """The destination project already has a file or directory where the resource's files would move to.""" + + +class ResourceMoveConflict(BaseModel): + """A conflict that keeps a resource from moving to the destination project.""" + + model_config = ConfigDict(extra="allow") + + reason: ResourceMoveConflictReason + detail: str + """A message that describes the conflict.""" + type: ResourceTypeName + id: str + name: str + path: Optional[str] = None + """For `file_exists`: the path of the file or directory, relative to the DataMasque files directory.""" + + +class ResourceMoveResult(BaseModel): + """The outcome of `move_resources`: every resource asked for either moved or is listed in `conflicts`.""" + + model_config = ConfigDict(extra="allow") + + moved: int + """The number of resources that moved.""" + conflicts: list[ResourceMoveConflict] + """The resources that did not move, and why.""" diff --git a/datamasque/client/models/resource_type.py b/datamasque/client/models/resource_type.py new file mode 100644 index 0000000..edefcd3 --- /dev/null +++ b/datamasque/client/models/resource_type.py @@ -0,0 +1,19 @@ +"""The types of resource that belong to a project.""" + +from enum import Enum + + +class ResourceTypeName(str, Enum): + """A type of resource that belongs to a project, named as the resources listing groups it.""" + + connections = "connections" + connection_filesets = "connection_filesets" + oracle_wallets = "oracle_wallets" + snowflake_keys = "snowflake_keys" + table_references = "table_references" + rulesets = "rulesets" + ruleset_libraries = "ruleset_libraries" + discovery_configs = "discovery_configs" + discovery_config_libraries = "discovery_config_libraries" + ruleset_generation_configs = "ruleset_generation_configs" + seeds = "seeds" diff --git a/datamasque/client/resources.py b/datamasque/client/resources.py index d7215aa..49c5625 100644 --- a/datamasque/client/resources.py +++ b/datamasque/client/resources.py @@ -1,14 +1,37 @@ +import logging from collections import defaultdict from collections.abc import Sequence from typing import Optional, Union from datamasque.client.base import BaseClient from datamasque.client.models.project import Project, ProjectId, unwrap_project_id -from datamasque.client.models.resource import RESOURCE_MODEL_BY_TYPE_NAME, Resource, ResourceTypeName +from datamasque.client.models.resource import ( + RESOURCE_MODEL_BY_TYPE_NAME, + Resource, + ResourceMoveConflict, + ResourceMoveResult, + ResourcesByType, +) +from datamasque.client.models.resource_type import ResourceTypeName + +logger = logging.getLogger(__name__) + + +def _move_request(resources: ResourcesByType, destination: Union[ProjectId, Project]) -> dict[str, object]: + keys = [ + {"type": type_name.value, "id": resource.id if isinstance(resource, Resource) else resource} + for type_name, group in resources.items() + for resource in group + ] + if not keys: + raise ValueError("Specify at least one resource to move") + + # The API takes the destination as a string: an id or `-`. + return {"destination": str(unwrap_project_id(destination)), "resources": keys} class ResourceClient(BaseClient): - """Resource listing API methods. Mixed into `DataMasqueClient`.""" + """Resource listing and move API methods. Mixed into `DataMasqueClient`.""" def list_resources( self, @@ -17,9 +40,9 @@ def list_resources( types: Optional[Sequence[ResourceTypeName]] = None, search: Optional[str] = None, page_size: int = 500, - ) -> dict[str, list[Resource]]: + ) -> dict[ResourceTypeName, list[Resource]]: """ - Returns the resources the caller can manage, grouped by type name such as `rulesets`, across every page. + Returns the resources the caller can manage, grouped by type, fetched in pages of `page_size`. `projects` limits them to those projects, each given as a `Project` or its `ProjectId`; `types` to those types; and `search` to resources whose name contains every word of it, ignoring case. @@ -29,12 +52,12 @@ def list_resources( and `DiscoveryConfigResource`. Raises: - DataMasqueApiError: a project does not exist, or any other non-2xx response. + DataMasqueApiError: the server rejects the request, for example because a project does not exist. """ # `requests` leaves out a parameter whose value is an empty list, which would remove the filter. if projects is not None and not projects: - return {type_name.value: [] for type_name in types or []} + return {type_name: [] for type_name in types or []} if types is not None and not types: return {} @@ -47,10 +70,63 @@ def list_resources( if search is not None: params["search"] = search - resources: defaultdict[str, list[Resource]] = defaultdict(list) + resources: defaultdict[ResourceTypeName, list[Resource]] = defaultdict(list) for results in self._iter_pages("/api/resources/", params=params, page_size=page_size): for type_name, rows in results.items(): model = RESOURCE_MODEL_BY_TYPE_NAME.get(type_name, Resource) - resources[type_name].extend(model.model_validate(row) for row in rows) + resources[ResourceTypeName(type_name)].extend(model.model_validate(row) for row in rows) return dict(resources) + + def get_resource_move_conflicts( + self, + resources: ResourcesByType, + destination: Union[ProjectId, Project], + ) -> list[ResourceMoveConflict]: + """ + Checks whether `resources` can move to the destination project, without moving them. + + `resources` can contain up to 2,000 resources. + + Returns: + The conflicts that would keep any of `resources` from moving (if any). + + Raises: + ValueError: `resources` is empty. + DataMasqueApiError: the server rejects the request, for example because the destination does not exist + or `resources` contains more than 2,000 resources. + """ + + response = self.make_request("POST", "/api/resources/move-check/", data=_move_request(resources, destination)) + return [ResourceMoveConflict.model_validate(conflict) for conflict in response.json()["conflicts"]] + + def move_resources( + self, + resources: ResourcesByType, + destination: Union[ProjectId, Project], + ) -> ResourceMoveResult: + """ + Moves `resources` to the destination project, with their files and runs. + + `resources` can be the result of `list_resources`, or any part of it, and can contain up to 2,000 resources. + A resource that conflicts with the destination, or with another resource in `resources`, + stays where it is and is listed in the result's `conflicts`; every other resource moves. + Rulesets, discovery configs and RG configs whose library imports resolve differently after the move + are marked invalid. + + Raises: + ValueError: `resources` is empty. + DataMasqueApiError: the server rejects the move, for example because the destination does not exist, + `resources` contains more than 2,000 resources, + or the caller does not have permission to move one of the resource types. + """ + + response = self.make_request("POST", "/api/resources/move/", data=_move_request(resources, destination)) + result = ResourceMoveResult.model_validate(response.json()) + logger.info( + "Moved %d resources to project %s; %d could not move", + result.moved, + unwrap_project_id(destination), + len(result.conflicts), + ) + return result diff --git a/docs/client.models.rst b/docs/client.models.rst index bd00a65..aa1e198 100644 --- a/docs/client.models.rst +++ b/docs/client.models.rst @@ -76,6 +76,14 @@ datamasque.client.models.resource module :undoc-members: :show-inheritance: +datamasque.client.models.resource_type module +--------------------------------------------- + +.. automodule:: datamasque.client.models.resource_type + :members: + :undoc-members: + :show-inheritance: + datamasque.client.models.rg_config module ----------------------------------------- diff --git a/tests/test_resources.py b/tests/test_resources.py index 7fde2eb..ad8ca5e 100644 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -1,4 +1,4 @@ -"""Tests for listing resources with the DataMasque client.""" +"""Tests for listing and moving resources with the DataMasque client.""" from datetime import datetime from typing import Optional @@ -14,6 +14,7 @@ Project, ProjectId, Resource, + ResourceMoveConflictReason, ResourceTypeName, RulesetResource, ) @@ -150,3 +151,73 @@ def test_list_resources_raises_for_a_caller_who_cannot_list_them(client: DataMas client.list_resources() assert raised.value.response.status_code == 403 + + +# The move methods accept a listed `Resource` or a plain ID. +MOVING = { + ResourceTypeName.rulesets: [RulesetResource.model_validate(RULESET)], + ResourceTypeName.connections: [CONNECTION["id"]], +} +MOVE_REQUEST = { + "destination": "8", + "resources": [{"type": "rulesets", "id": RULESET["id"]}, {"type": "connections", "id": CONNECTION["id"]}], +} +NAME_CLASH = { + "reason": "name_in_use", + "detail": 'The destination project already contains a connection named "customers_db".', + "type": "connections", + "id": CONNECTION["id"], + "name": "customers_db", +} + + +def test_get_resource_move_conflicts_sends_the_resources_and_destination(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.post("http://test-server/api/resources/move-check/", json={"conflicts": [NAME_CLASH]}) + conflicts = client.get_resource_move_conflicts(MOVING, ProjectId(8)) + + assert m.last_request.json() == MOVE_REQUEST + assert len(conflicts) == 1 + assert conflicts[0].reason is ResourceMoveConflictReason.name_in_use + assert conflicts[0].type is ResourceTypeName.connections + assert conflicts[0].name == "customers_db" + + +def test_move_resources_returns_the_count_moved_and_the_conflicts(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.post("http://test-server/api/resources/move/", json={"moved": 1, "conflicts": [NAME_CLASH]}) + result = client.move_resources(MOVING, ProjectId(8)) + + assert m.last_request.json() == MOVE_REQUEST + assert result.moved == 1 + assert [conflict.id for conflict in result.conflicts] == [CONNECTION["id"]] + + +def test_move_resources_moves_what_list_resources_returned_to_a_project(client: DataMasqueClient) -> None: + retail = Project.model_validate( + { + "id": 8, + "reference": "8-retail", + "name": "Retail", + "created": "2026-10-01T10:00:00Z", + "updated": "2026-10-01T10:00:00Z", + } + ) + with requests_mock.Mocker() as m: + m.get("http://test-server/api/resources/", json=page({"rulesets": [RULESET], "connections": [CONNECTION]})) + m.post("http://test-server/api/resources/move/", json={"moved": 2, "conflicts": []}) + client.move_resources(client.list_resources(), retail) + + assert m.last_request.json() == MOVE_REQUEST + + +@pytest.mark.parametrize("resources", [{}, {ResourceTypeName.rulesets: []}]) +def test_moving_no_resources_raises_without_a_request(client: DataMasqueClient, resources: dict) -> None: + with requests_mock.Mocker() as m: + with pytest.raises(ValueError, match="at least one resource"): + client.move_resources(resources, ProjectId(8)) + + with pytest.raises(ValueError, match="at least one resource"): + client.get_resource_move_conflicts(resources, ProjectId(8)) + + assert m.call_count == 0 From 0be0b51c355b4cd95afb9025ae5d4b2d345dbb19 Mon Sep 17 00:00:00 2001 From: Colin Haywood Date: Tue, 20 Oct 2026 16:51:01 +1300 Subject: [PATCH 11/22] feat: accept a project's - form wherever the client takes a project - `ProjectIdentifier` names the three forms a project can be given in. --- datamasque/client/__init__.py | 2 ++ datamasque/client/connections.py | 10 +++---- .../client/discovery_config_libraries.py | 10 +++---- datamasque/client/discovery_configs.py | 10 +++---- datamasque/client/files.py | 16 +++++------ datamasque/client/models/project.py | 27 ++++++++++++++++--- datamasque/client/projects.py | 18 ++++++++----- datamasque/client/resources.py | 14 +++++----- datamasque/client/rg_configs.py | 10 +++---- datamasque/client/ruleset_libraries.py | 10 +++---- datamasque/client/rulesets.py | 7 +++-- datamasque/client/table_references.py | 14 +++++----- tests/test_projects.py | 23 +++++++++++++--- tests/test_resources.py | 9 +++++++ 14 files changed, 116 insertions(+), 64 deletions(-) diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index f5d7767..af70b7d 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -122,6 +122,7 @@ ProjectDeleteConflict, ProjectDeleteConflictReason, ProjectId, + ProjectIdentifier, ) from datamasque.client.models.resource import ( ConnectionResource, @@ -296,6 +297,7 @@ "ProjectDeleteConflictError", "ProjectDeleteConflictReason", "ProjectId", + "ProjectIdentifier", "RGConfig", "RGConfigId", "RGConfigNotFoundError", diff --git a/datamasque/client/connections.py b/datamasque/client/connections.py index 20e383d..43e7652 100644 --- a/datamasque/client/connections.py +++ b/datamasque/client/connections.py @@ -4,7 +4,7 @@ from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueException from datamasque.client.models.connection import ConnectionConfig, ConnectionId, validate_connection -from datamasque.client.models.project import Project, ProjectId, unwrap_project_id +from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id logger = logging.getLogger(__name__) @@ -12,11 +12,11 @@ class ConnectionClient(BaseClient): """Connection-related API methods. Mixed into `DataMasqueClient`.""" - def list_connections(self, project: Optional[Union[ProjectId, Project]] = None) -> list[ConnectionConfig]: + def list_connections(self, project: Optional[ProjectIdentifier] = None) -> list[ConnectionConfig]: """ Lists all configured connections, or only those in `project` when it is given. - `project` is a `Project` or its `ProjectId`. + `project` is a `Project`, its `ProjectId` or its `-` form. Note that database passwords and connection strings are returned encrypted over the API and so are `None` on the returned `ConnectionConfig` objects. """ @@ -70,11 +70,11 @@ def delete_connection_by_id_if_exists(self, connection_id: ConnectionId) -> None self._delete_if_exists(f"/api/connections/{connection_id}/") - def delete_connection_by_name_if_exists(self, connection_name: str, project: Union[ProjectId, Project]) -> None: + def delete_connection_by_name_if_exists(self, connection_name: str, project: ProjectIdentifier) -> None: """ Deletes the connection with the given name in the given project. - `project` is a `Project` or its `ProjectId`. + `project` is a `Project`, its `ProjectId` or its `-` form. No-op if the connection does not exist. """ diff --git a/datamasque/client/discovery_config_libraries.py b/datamasque/client/discovery_config_libraries.py index d43dc9c..c0e978b 100644 --- a/datamasque/client/discovery_config_libraries.py +++ b/datamasque/client/discovery_config_libraries.py @@ -4,7 +4,7 @@ from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary, DiscoveryConfigLibraryId -from datamasque.client.models.project import Project, ProjectId, unwrap_project_id +from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id logger = logging.getLogger(__name__) @@ -55,12 +55,12 @@ def _get_discovery_config_library_id_by_name( return matches[0].id def get_discovery_config_library_by_name( - self, name: str, project: Union[ProjectId, Project] + self, name: str, project: ProjectIdentifier ) -> Optional[DiscoveryConfigLibrary]: """ Looks for a discovery config library matching the given name (case-sensitive, exact match) in the given project. - `project` is a `Project` or its `ProjectId`. + `project` is a `Project`, its `ProjectId` or its `-` form. Returns it (with full YAML content) if found, otherwise `None`. """ @@ -155,12 +155,12 @@ def delete_discovery_config_library_by_id_if_exists( self._delete_if_exists(f"/api/discovery/config-libraries/{library_id}/", params=params) def delete_discovery_config_library_by_name_if_exists( - self, name: str, project: Union[ProjectId, Project], *, force: bool = False + self, name: str, project: ProjectIdentifier, *, force: bool = False ) -> None: """ Deletes the discovery config library with the given name in the given project. - `project` is a `Project` or its `ProjectId`. + `project` is a `Project`, its `ProjectId` or its `-` form. No-op if no such library exists. """ diff --git a/datamasque/client/discovery_configs.py b/datamasque/client/discovery_configs.py index f0eb76e..69f5cac 100644 --- a/datamasque/client/discovery_configs.py +++ b/datamasque/client/discovery_configs.py @@ -5,7 +5,7 @@ from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType from datamasque.client.models.pagination import Page -from datamasque.client.models.project import Project, ProjectId, unwrap_project_id +from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id logger = logging.getLogger(__name__) @@ -58,13 +58,13 @@ def _get_discovery_config_id_by_name( return config_id def get_discovery_config_by_name( - self, name: str, config_type: DiscoveryConfigType, project: Union[ProjectId, Project] + self, name: str, config_type: DiscoveryConfigType, project: ProjectIdentifier ) -> Optional[DiscoveryConfig]: """ Looks for a discovery config matching the given name (case-sensitive, exact match), type and project. Config names are unique per type and project, so both are required to identify a single config. - `project` is a `Project` or its `ProjectId`. + `project` is a `Project`, its `ProjectId` or its `-` form. Returns it if found, otherwise `None`. """ @@ -156,13 +156,13 @@ def delete_discovery_config_by_id_if_exists(self, config_id: DiscoveryConfigId) self._delete_if_exists(f"/api/discovery/configs/{config_id}/") def delete_discovery_config_by_name_if_exists( - self, name: str, config_type: DiscoveryConfigType, project: Union[ProjectId, Project] + self, name: str, config_type: DiscoveryConfigType, project: ProjectIdentifier ) -> None: """ Deletes the discovery config with the given name and type in the given project. Config names are unique per type and project, so both are required to identify a single config. - `project` is a `Project` or its `ProjectId`. + `project` is a `Project`, its `ProjectId` or its `-` form. No-op if no such config exists. """ diff --git a/datamasque/client/files.py b/datamasque/client/files.py index d7f196c..641b4bc 100644 --- a/datamasque/client/files.py +++ b/datamasque/client/files.py @@ -3,7 +3,7 @@ from datamasque.client.base import BaseClient, UploadFile, read_file_or_content from datamasque.client.models.files import DataMasqueFile -from datamasque.client.models.project import Project, ProjectId, unwrap_project_id +from datamasque.client.models.project import ProjectIdentifier, unwrap_project_id FileTypeT = TypeVar("FileTypeT", bound=DataMasqueFile) @@ -16,7 +16,7 @@ def upload_file( file_type: Type[FileTypeT], file_name: str, file_path_or_content: Union[str, bytes, Path], - project: Union[ProjectId, Project], + project: ProjectIdentifier, ) -> FileTypeT: """ Uploads a file of the given type to the DataMasque server, into the given project. @@ -24,7 +24,7 @@ def upload_file( `file_type` must be a concrete subclass of `DataMasqueFile` (`SeedFile`, `OracleWalletFile`, `SslZipFile`, `SnowflakeKeyFile`). `file_path_or_content` may be a path (as `str` or `Path`), raw `bytes`, or a file-like object. - `project` is a `Project` or its `ProjectId`. + `project` is a `Project`, its `ProjectId` or its `-` form. """ name, content = read_file_or_content(file_path_or_content, file_name) @@ -60,13 +60,13 @@ def delete_file_if_exists(self, file: DataMasqueFile) -> None: self._delete_if_exists(f"{file.get_url()}{file.id}/") def list_files_of_type( - self, file_type: Type[FileTypeT], project: Optional[Union[ProjectId, Project]] = None + self, file_type: Type[FileTypeT], project: Optional[ProjectIdentifier] = None ) -> list[FileTypeT]: """ Returns all files of the given type (a concrete subclass of `DataMasqueFile`). Only files in `project` are returned when it is given. - `project` is a `Project` or its `ProjectId`. + `project` is a `Project`, its `ProjectId` or its `-` form. """ params = {"project": unwrap_project_id(project)} if project is not None else None @@ -74,7 +74,7 @@ def list_files_of_type( return [file_type.model_validate(file) for file in response.json()] def get_file_of_type_by_name( - self, file_type: Type[FileTypeT], name: str, project: Union[ProjectId, Project] + self, file_type: Type[FileTypeT], name: str, project: ProjectIdentifier ) -> Optional[FileTypeT]: """ Looks for a file of the given type (a concrete subclass of `DataMasqueFile`) by name in the given project. @@ -86,7 +86,7 @@ def get_file_of_type_by_name( return matching_files[0] if matching_files else None def upload_file_if_not_exists( - self, file_type: Type[FileTypeT], file_path: Union[str, Path], project: Union[ProjectId, Project] + self, file_type: Type[FileTypeT], file_path: Union[str, Path], project: ProjectIdentifier ) -> Optional[FileTypeT]: """ Upload a file only if one with the same name doesn't already exist in the project. @@ -94,7 +94,7 @@ def upload_file_if_not_exists( Args: file_type: A concrete subclass of `DataMasqueFile` (e.g., SeedFile, OracleWalletFile). file_path: Path to the file to upload. - project: A `Project` or its `ProjectId`. + project: A `Project`, its `ProjectId` or its `-` form. Returns: The uploaded file object if a new file was uploaded, or None if a file diff --git a/datamasque/client/models/project.py b/datamasque/client/models/project.py index 727d00f..c925f03 100644 --- a/datamasque/client/models/project.py +++ b/datamasque/client/models/project.py @@ -39,9 +39,30 @@ class Project(BaseModel): updated: datetime -def unwrap_project_id(project: Union[ProjectId, Project]) -> ProjectId: - """Return the id of `project`, which a caller may give as a `Project` or as its `ProjectId`.""" - return project.id if isinstance(project, Project) else project +# A project, given as a `Project`, its `ProjectId`, or its `-` form such as a listed resource's `project`. +ProjectIdentifier = Union[ProjectId, Project, str] + + +def unwrap_project_id(project: ProjectIdentifier) -> ProjectId: + """ + Return the ID of `project`. + + Raises: + ValueError: `project` is a string that is not a project reference in the form `-`. + """ + if isinstance(project, Project): + return project.id + + if isinstance(project, str): + try: + return ProjectId(int(project.split("-")[0])) + except ValueError: + raise ValueError( + f'"{project}" is not a project ID or reference. ' + "Specify the ID as an integer, or a reference in the form `3-myproject`." + ) from None + + return project class ProjectCreateRequest(BaseModel): diff --git a/datamasque/client/projects.py b/datamasque/client/projects.py index 149ba0e..ce03ede 100644 --- a/datamasque/client/projects.py +++ b/datamasque/client/projects.py @@ -1,5 +1,5 @@ import logging -from typing import Iterator, Union +from typing import Iterator from requests import Response @@ -9,7 +9,7 @@ Project, ProjectCreateRequest, ProjectDeleteConflict, - ProjectId, + ProjectIdentifier, unwrap_project_id, ) @@ -42,9 +42,9 @@ def list_projects(self, *, include_archived: bool = False) -> list[Project]: return list(self.iter_projects(include_archived=include_archived)) - def get_project(self, project: Union[ProjectId, Project]) -> Project: + def get_project(self, project: ProjectIdentifier) -> Project: """ - Retrieves a single project, given as a `Project` or its `ProjectId`. + Retrieves a single project, given as a `Project`, its `ProjectId` or its `-` form. Archived projects can be retrieved too. """ @@ -68,8 +68,8 @@ def create_project(self, request: ProjectCreateRequest) -> Project: def get_project_delete_conflicts( self, - project: Union[ProjectId, Project], - destination: Union[ProjectId, Project], + project: ProjectIdentifier, + destination: ProjectIdentifier, ) -> list[ProjectDeleteConflict]: """ Returns the conflicts that would prevent `delete_project` moving the project's resources to the destination. @@ -89,7 +89,11 @@ def get_project_delete_conflicts( ) return _get_conflicts(response) - def delete_project(self, project: Union[ProjectId, Project], destination: Union[ProjectId, Project]) -> None: + def delete_project( + self, + project: ProjectIdentifier, + destination: ProjectIdentifier, + ) -> None: """ Deletes a project, after moving every resource in it to the destination project. diff --git a/datamasque/client/resources.py b/datamasque/client/resources.py index 49c5625..e2be1a1 100644 --- a/datamasque/client/resources.py +++ b/datamasque/client/resources.py @@ -1,10 +1,10 @@ import logging from collections import defaultdict from collections.abc import Sequence -from typing import Optional, Union +from typing import Optional from datamasque.client.base import BaseClient -from datamasque.client.models.project import Project, ProjectId, unwrap_project_id +from datamasque.client.models.project import ProjectIdentifier, unwrap_project_id from datamasque.client.models.resource import ( RESOURCE_MODEL_BY_TYPE_NAME, Resource, @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) -def _move_request(resources: ResourcesByType, destination: Union[ProjectId, Project]) -> dict[str, object]: +def _move_request(resources: ResourcesByType, destination: ProjectIdentifier) -> dict[str, object]: keys = [ {"type": type_name.value, "id": resource.id if isinstance(resource, Resource) else resource} for type_name, group in resources.items() @@ -36,7 +36,7 @@ class ResourceClient(BaseClient): def list_resources( self, *, - projects: Optional[Sequence[Union[ProjectId, Project]]] = None, + projects: Optional[Sequence[ProjectIdentifier]] = None, types: Optional[Sequence[ResourceTypeName]] = None, search: Optional[str] = None, page_size: int = 500, @@ -44,7 +44,7 @@ def list_resources( """ Returns the resources the caller can manage, grouped by type, fetched in pages of `page_size`. - `projects` limits them to those projects, each given as a `Project` or its `ProjectId`; + `projects` limits them to those projects, each given as a `Project`, its `ProjectId` or its `-` form; `types` to those types; and `search` to resources whose name contains every word of it, ignoring case. The groups are the requested types, or every type the server has when `types` is not given. An empty `projects` or `types` matches no resources, without a request to the server. @@ -81,7 +81,7 @@ def list_resources( def get_resource_move_conflicts( self, resources: ResourcesByType, - destination: Union[ProjectId, Project], + destination: ProjectIdentifier, ) -> list[ResourceMoveConflict]: """ Checks whether `resources` can move to the destination project, without moving them. @@ -103,7 +103,7 @@ def get_resource_move_conflicts( def move_resources( self, resources: ResourcesByType, - destination: Union[ProjectId, Project], + destination: ProjectIdentifier, ) -> ResourceMoveResult: """ Moves `resources` to the destination project, with their files and runs. diff --git a/datamasque/client/rg_configs.py b/datamasque/client/rg_configs.py index ea94237..cf5237d 100644 --- a/datamasque/client/rg_configs.py +++ b/datamasque/client/rg_configs.py @@ -4,7 +4,7 @@ from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.pagination import Page -from datamasque.client.models.project import Project, ProjectId, unwrap_project_id +from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id from datamasque.client.models.rg_config import RGConfig, RGConfigId logger = logging.getLogger(__name__) @@ -55,12 +55,12 @@ def _get_rg_config_id_by_name(self, name: str, project: Union[ProjectId, str]) - return config_id - def get_rg_config_by_name(self, name: str, project: Union[ProjectId, Project]) -> Optional[RGConfig]: + def get_rg_config_by_name(self, name: str, project: ProjectIdentifier) -> Optional[RGConfig]: """ Looks for an RG config matching the given name (case-sensitive, exact match) in the given project. RG configs are untyped and their names are unique per project. - `project` is a `Project` or its `ProjectId`. + `project` is a `Project`, its `ProjectId` or its `-` form. Returns it if found, otherwise `None`. """ @@ -144,11 +144,11 @@ def delete_rg_config_by_id_if_exists(self, config_id: RGConfigId) -> None: self._delete_if_exists(f"/api/ruleset-generation-configs/{config_id}/") - def delete_rg_config_by_name_if_exists(self, name: str, project: Union[ProjectId, Project]) -> None: + def delete_rg_config_by_name_if_exists(self, name: str, project: ProjectIdentifier) -> None: """ Deletes the RG config with the given name in the given project. - `project` is a `Project` or its `ProjectId`. + `project` is a `Project`, its `ProjectId` or its `-` form. No-op if no such config exists. """ diff --git a/datamasque/client/ruleset_libraries.py b/datamasque/client/ruleset_libraries.py index b029ead..20003c3 100644 --- a/datamasque/client/ruleset_libraries.py +++ b/datamasque/client/ruleset_libraries.py @@ -4,7 +4,7 @@ from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.pagination import Page -from datamasque.client.models.project import Project, ProjectId, unwrap_project_id +from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id from datamasque.client.models.ruleset import Ruleset from datamasque.client.models.ruleset_library import RulesetLibrary, RulesetLibraryId @@ -56,11 +56,11 @@ def _get_ruleset_library_id_by_name(self, name: str, project: Union[ProjectId, s return library_id - def get_ruleset_library_by_name(self, name: str, project: Union[ProjectId, Project]) -> Optional[RulesetLibrary]: + def get_ruleset_library_by_name(self, name: str, project: ProjectIdentifier) -> Optional[RulesetLibrary]: """ Looks for a ruleset library matching the given name (case-sensitive, exact match) in the given project. - `project` is a `Project` or its `ProjectId`. + `project` is a `Project`, its `ProjectId` or its `-` form. Returns it (with full YAML content) if found, otherwise None. """ @@ -147,12 +147,12 @@ def delete_ruleset_library_by_id_if_exists(self, library_id: RulesetLibraryId, * self._delete_if_exists(f"/api/ruleset-libraries/{library_id}/", params=params) def delete_ruleset_library_by_name_if_exists( - self, library_name: str, project: Union[ProjectId, Project], *, force: bool = False + self, library_name: str, project: ProjectIdentifier, *, force: bool = False ) -> None: """ Deletes the ruleset library with the given name in the given project. - `project` is a `Project` or its `ProjectId`. + `project` is a `Project`, its `ProjectId` or its `-` form. No-op if the library does not exist. """ diff --git a/datamasque/client/rulesets.py b/datamasque/client/rulesets.py index ed82ef8..5337c93 100644 --- a/datamasque/client/rulesets.py +++ b/datamasque/client/rulesets.py @@ -1,9 +1,8 @@ import logging -from typing import Union from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueException -from datamasque.client.models.project import Project, ProjectId, unwrap_project_id +from datamasque.client.models.project import ProjectIdentifier, unwrap_project_id from datamasque.client.models.ruleset import Ruleset, RulesetId, RulesetType logger = logging.getLogger(__name__) @@ -55,13 +54,13 @@ def delete_ruleset_by_id_if_exists(self, ruleset_id: RulesetId) -> None: self._delete_if_exists(f"/api/rulesets/{ruleset_id}/") def delete_ruleset_by_name_if_exists( - self, ruleset_name: str, ruleset_type: RulesetType, project: Union[ProjectId, Project] + self, ruleset_name: str, ruleset_type: RulesetType, project: ProjectIdentifier ) -> None: """ Deletes the ruleset with the given name and type in the given project. Ruleset names are unique per type and project, so both are required to identify a single ruleset. - `project` is a `Project` or its `ProjectId`. + `project` is a `Project`, its `ProjectId` or its `-` form. No-op if no such ruleset exists. """ diff --git a/datamasque/client/table_references.py b/datamasque/client/table_references.py index 6d5f5b8..04ec955 100644 --- a/datamasque/client/table_references.py +++ b/datamasque/client/table_references.py @@ -3,7 +3,7 @@ from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueApiError -from datamasque.client.models.project import Project, ProjectId, unwrap_project_id +from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id from datamasque.client.models.table_reference import TableReference, TableReferenceId logger = logging.getLogger(__name__) @@ -12,11 +12,11 @@ class TableReferenceClient(BaseClient): """Table reference CRUD API methods. Mixed into `DataMasqueClient`.""" - def list_table_references(self, project: Optional[Union[ProjectId, Project]] = None) -> list[TableReference]: + def list_table_references(self, project: Optional[ProjectIdentifier] = None) -> list[TableReference]: """ Lists all table references, or only those in `project` when it is given. - `project` is a `Project` or its `ProjectId`. + `project` is a `Project`, its `ProjectId` or its `-` form. Unlike most list endpoints, this one is not paginated; the server returns every table reference in a single response. @@ -33,12 +33,12 @@ def get_table_reference(self, table_reference_id: TableReferenceId) -> TableRefe response = self.make_request("GET", f"/api/table-references/{table_reference_id}/") return TableReference.model_validate(response.json()) - def get_table_reference_by_name(self, name: str, project: Union[ProjectId, Project]) -> Optional[TableReference]: + def get_table_reference_by_name(self, name: str, project: ProjectIdentifier) -> Optional[TableReference]: """ Looks for a table reference with the given name (case-sensitive, exact match) in the given project. Names are unique across a project's live table references, so at most one can match. - `project` is a `Project` or its `ProjectId`. + `project` is a `Project`, its `ProjectId` or its `-` form. Returns it if found, otherwise `None`. The endpoint takes no name filter, so the match is made client-side over the project's listing. @@ -147,11 +147,11 @@ def delete_table_reference_by_id_if_exists(self, table_reference_id: TableRefere self._delete_if_exists(f"/api/table-references/{table_reference_id}/") - def delete_table_reference_by_name_if_exists(self, name: str, project: Union[ProjectId, Project]) -> None: + def delete_table_reference_by_name_if_exists(self, name: str, project: ProjectIdentifier) -> None: """ Deletes the table reference with the given name in the given project. - `project` is a `Project` or its `ProjectId`. + `project` is a `Project`, its `ProjectId` or its `-` form. No-op if no such table reference exists. """ diff --git a/tests/test_projects.py b/tests/test_projects.py index 0830293..ff94b7a 100644 --- a/tests/test_projects.py +++ b/tests/test_projects.py @@ -1,7 +1,7 @@ """Tests for project support in the DataMasque client.""" from datetime import datetime -from typing import Callable, Union +from typing import Callable import pytest import requests_mock @@ -16,11 +16,13 @@ ProjectDeleteConflictError, ProjectDeleteConflictReason, ProjectId, + ProjectIdentifier, ) from datamasque.client.exceptions import DataMasqueException from datamasque.client.models.connection import ConnectionId, S3ConnectionConfig from datamasque.client.models.discovery_config import DiscoveryConfig from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary +from datamasque.client.models.project import unwrap_project_id from datamasque.client.models.rg_config import RGConfig from datamasque.client.models.ruleset import Ruleset, RulesetType from datamasque.client.models.ruleset_library import RulesetLibrary @@ -260,8 +262,8 @@ def test_list_projects_include_archived(client: DataMasqueClient) -> None: assert [project.id for project in projects] == [ProjectId(7), ProjectId(8)] -@pytest.mark.parametrize("project", [ProjectId(7), Project.model_validate(BANKING_PROJECT)]) -def test_get_project(client: DataMasqueClient, project: Union[ProjectId, Project]) -> None: +@pytest.mark.parametrize("project", [ProjectId(7), Project.model_validate(BANKING_PROJECT), "7-banking"]) +def test_get_project(client: DataMasqueClient, project: ProjectIdentifier) -> None: with requests_mock.Mocker() as m: m.get("http://test-server/api/projects/7/", json=BANKING_PROJECT, complete_qs=True) result = client.get_project(project) @@ -415,3 +417,18 @@ def test_create_or_update_without_a_project_needs_exactly_one_active_project( getattr(client, method)(make_resource()) assert [request.path for request in m.request_history] == ["/api/projects/"] + + +@pytest.mark.parametrize( + "project", + [ProjectId(7), Project.model_validate(BANKING_PROJECT), "7-banking", "7"], + ids=["id", "project", "reference", "id as a string"], +) +def test_unwrap_project_id_reads_the_id_from_each_form(project: ProjectIdentifier) -> None: + assert unwrap_project_id(project) == ProjectId(7) + + +@pytest.mark.parametrize("project", ["myproject", "", "-7"]) +def test_unwrap_project_id_rejects_a_string_that_does_not_start_with_an_id(project: str) -> None: + with pytest.raises(ValueError, match=f'"{project}" is not a project ID or reference'): + unwrap_project_id(project) diff --git a/tests/test_resources.py b/tests/test_resources.py index ad8ca5e..c912cec 100644 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -211,6 +211,15 @@ def test_move_resources_moves_what_list_resources_returned_to_a_project(client: assert m.last_request.json() == MOVE_REQUEST +def test_move_resources_takes_a_listed_resources_project_as_the_destination(client: DataMasqueClient) -> None: + listed = RulesetResource.model_validate(RULESET) + with requests_mock.Mocker() as m: + m.post("http://test-server/api/resources/move/", json={"moved": 2, "conflicts": []}) + client.move_resources(MOVING, listed.project) + + assert m.last_request.json()["destination"] == "7" + + @pytest.mark.parametrize("resources", [{}, {ResourceTypeName.rulesets: []}]) def test_moving_no_resources_raises_without_a_request(client: DataMasqueClient, resources: dict) -> None: with requests_mock.Mocker() as m: From ac89d89b0b145cb686e233ad6b8b9f57fa72febe Mon Sep 17 00:00:00 2001 From: Elliot Simpson Date: Mon, 2 Nov 2026 11:20:23 +1300 Subject: [PATCH 12/22] feat: DM-4950: add ordering to list_resources and expose each resource's position --- HISTORY.rst | 3 +++ datamasque/client/__init__.py | 2 ++ datamasque/client/models/resource.py | 27 +++++++++++++++++++++ datamasque/client/resources.py | 9 +++++++ tests/test_resources.py | 35 +++++++++++++++++++++++++++- 5 files changed, 75 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 3357a37..07bb3f0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -32,6 +32,9 @@ History * ``get_generated_rulesets`` sets each ruleset's ``project`` to the connection's project. * Added project models and management APIs. * Added resource management APIs. +* ``list_resources`` takes ``ordering`` and ``descending``, and each type's resources come back in that order. + Each listed ``Resource`` has a ``position``, its place in that order across every type. + Both require server version 3.26.21. 1.6.0 (unreleased) ------------------ diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index af70b7d..c91bd61 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -131,6 +131,7 @@ ResourceMoveConflict, ResourceMoveConflictReason, ResourceMoveResult, + ResourceOrdering, ResourcesByType, RulesetResource, ) @@ -306,6 +307,7 @@ "ResourceMoveConflict", "ResourceMoveConflictReason", "ResourceMoveResult", + "ResourceOrdering", "ResourceTypeName", "ResourcesByType", "Ruleset", diff --git a/datamasque/client/models/resource.py b/datamasque/client/models/resource.py index 0ba85d4..4f50baa 100644 --- a/datamasque/client/models/resource.py +++ b/datamasque/client/models/resource.py @@ -29,6 +29,33 @@ class Resource(BaseModel): """`None` for a type that does not record it, currently connections.""" modified: Optional[datetime] = None """`None` for a type that does not record it, currently connections.""" + position: Optional[int] = None + """ + Its index, counting from 0, among every resource the listing matched, in the listing's ordering. + It changes with the filters and ordering, so it does not identify the resource. + Requires server version 3.26.21, and is `None` on an older server. + """ + + +class ResourceOrdering(str, Enum): + """ + A column `list_resources` sorts by, with the columns that break ties after it. + + Ties are broken by type, then name, then project name, then ID, leaving out the column sorted by. + Types sort in the order `ResourceTypeName` declares them. + Names and project names compare ignoring case. + """ + + type = "type" + name = "name" + project = "project" + """ + Sorts by project name. + A resource in no project sorts after every project, + or before every project when `descending` is set. + """ + modified = "modified" + """A resource with no `modified` time sorts as the oldest.""" class ConnectionResource(Resource): diff --git a/datamasque/client/resources.py b/datamasque/client/resources.py index e2be1a1..2b0b5f0 100644 --- a/datamasque/client/resources.py +++ b/datamasque/client/resources.py @@ -10,6 +10,7 @@ Resource, ResourceMoveConflict, ResourceMoveResult, + ResourceOrdering, ResourcesByType, ) from datamasque.client.models.resource_type import ResourceTypeName @@ -39,6 +40,8 @@ def list_resources( projects: Optional[Sequence[ProjectIdentifier]] = None, types: Optional[Sequence[ResourceTypeName]] = None, search: Optional[str] = None, + ordering: ResourceOrdering = ResourceOrdering.type, + descending: bool = False, page_size: int = 500, ) -> dict[ResourceTypeName, list[Resource]]: """ @@ -46,6 +49,11 @@ def list_resources( `projects` limits them to those projects, each given as a `Project`, its `ProjectId` or its `-` form; `types` to those types; and `search` to resources whose name contains every word of it, ignoring case. + Each group's resources come in `ordering`, descending when `descending` is set. + Columns that break ties are always sorted ascending. + Sort the resources of every group together by `position` to get the whole listing in order. + `ordering` and `descending` require server version 3.26.21. + An older server ignores them, and each resource's `position` is `None`. The groups are the requested types, or every type the server has when `types` is not given. An empty `projects` or `types` matches no resources, without a request to the server. Connections, rulesets and discovery configs are listed as `ConnectionResource`, `RulesetResource` @@ -69,6 +77,7 @@ def list_resources( params["type"] = [type_name.value for type_name in types] if search is not None: params["search"] = search + params["ordering"] = f"-{ordering.value}" if descending else ordering.value resources: defaultdict[ResourceTypeName, list[Resource]] = defaultdict(list) for results in self._iter_pages("/api/resources/", params=params, page_size=page_size): diff --git a/tests/test_resources.py b/tests/test_resources.py index c912cec..67beae5 100644 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -1,7 +1,7 @@ """Tests for listing and moving resources with the DataMasque client.""" from datetime import datetime -from typing import Optional +from typing import Any, Optional import pytest import requests_mock @@ -15,6 +15,7 @@ ProjectId, Resource, ResourceMoveConflictReason, + ResourceOrdering, ResourceTypeName, RulesetResource, ) @@ -75,6 +76,7 @@ def test_list_resources_groups_them_by_type_name(client: DataMasqueClient) -> No (seed,) = resources["seeds"] assert type(seed) is Resource assert seed.project is None + assert seed.position is None def test_list_resources_sends_its_filters(client: DataMasqueClient) -> None: @@ -93,17 +95,48 @@ def test_list_resources_sends_its_filters(client: DataMasqueClient) -> None: projects=[ProjectId(7), retail], types=[ResourceTypeName.rulesets, ResourceTypeName.connections], search="customers", + ordering=ResourceOrdering.modified, ) assert m.last_request.qs == { "project": ["7", "8"], "type": ["rulesets", "connections"], "search": ["customers"], + "ordering": ["modified"], "limit": ["500"], "offset": ["0"], } +@pytest.mark.parametrize( + ("kwargs", "expected"), + [ + ({}, "type"), + ({"descending": True}, "-type"), + ({"ordering": ResourceOrdering.project}, "project"), + ({"ordering": ResourceOrdering.project, "descending": True}, "-project"), + ], +) +def test_list_resources_sends_its_ordering(client: DataMasqueClient, kwargs: dict[str, Any], expected: str) -> None: + with requests_mock.Mocker() as m: + m.get("http://test-server/api/resources/", json=page({})) + client.list_resources(**kwargs) + + assert m.last_request.qs["ordering"] == [expected] + + +def test_list_resources_keeps_each_resources_position(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/resources/", + json=page({"connections": [CONNECTION | {"position": 1}], "rulesets": [RULESET | {"position": 0}]}), + ) + resources = client.list_resources(ordering=ResourceOrdering.modified, descending=True) + + in_order = sorted((resource for group in resources.values() for resource in group), key=lambda r: r.position) + assert [resource.id for resource in in_order] == [RULESET["id"], CONNECTION["id"]] + + @pytest.mark.parametrize( ("projects", "types", "expected"), [ From 8bb8781102e2becd2810b03a70147a3465be3b3b Mon Sep 17 00:00:00 2001 From: Elliot Simpson Date: Fri, 30 Oct 2026 11:57:57 +1300 Subject: [PATCH 13/22] feat: DM-4677: add git push, pull, file listing and push paths --- HISTORY.rst | 7 + datamasque/client/__init__.py | 6 +- datamasque/client/dmclient.py | 2 + datamasque/client/git.py | 459 ++++++++++++++++++++++++++++++++ datamasque/client/models/git.py | 29 ++ docs/client.models.rst | 7 + docs/client.rst | 8 + tests/test_git.py | 332 +++++++++++++++++++++++ 8 files changed, 849 insertions(+), 1 deletion(-) create mode 100644 datamasque/client/git.py create mode 100644 tests/test_git.py diff --git a/HISTORY.rst b/HISTORY.rst index 07bb3f0..9e507f2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -30,6 +30,13 @@ History Updating a connection leaves it in its project. * ``DataMasqueFile`` gains a read-only ``project`` field, which is ``None`` for built-in seeds. * ``get_generated_rulesets`` sets each ruleset's ``project`` to the connection's project. +* Added Git push, pull and file listing for rulesets, ruleset libraries, discovery configs + and discovery config libraries: ``list_*_git_files``, ``pull_*_from_git``, ``push_*_to_git`` + and ``get_*_git_push_path``, with the ``GitFile``, ``GitFileCommit`` and ``GitExtension`` types. + A push writes the resource's file in its project's folder, with the ``extension`` it is given (``.yml`` or ``.yaml``). + A resource pushed before the server's upgrade to projects keeps its existing file + until it is renamed or moved to another project. + ``get_*_git_push_path`` returns the path a push writes, relative to the repository and without its extension. * Added project models and management APIs. * Added resource management APIs. * ``list_resources`` takes ``ordering`` and ``descending``, and each type's resources come back in that order. diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index c91bd61..d276425 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -27,6 +27,7 @@ RGConfigNotFoundError, RunNotCancellableError, ) +from datamasque.client.git import GitExtension from datamasque.client.ifm import DataMasqueIfmClient from datamasque.client.models.connection import ( AzureConnectionConfig, @@ -101,7 +102,7 @@ SnowflakeKeyFile, SslZipFile, ) -from datamasque.client.models.git import GitSnapshot +from datamasque.client.models.git import GitFile, GitFileCommit, GitSnapshot from datamasque.client.models.ifm import ( DataMasqueIfmInstanceConfig, IfmLog, @@ -254,6 +255,9 @@ "FirstCharsStatistics", "ForeignKeyRef", "GcsConnectionConfig", + "GitExtension", + "GitFile", + "GitFileCommit", "GitSnapshot", "HashColumnsTableConfig", "IfmAuthError", diff --git a/datamasque/client/dmclient.py b/datamasque/client/dmclient.py index 0d98dca..3591335 100644 --- a/datamasque/client/dmclient.py +++ b/datamasque/client/dmclient.py @@ -4,6 +4,7 @@ from datamasque.client.discovery_config_libraries import DiscoveryConfigLibraryClient from datamasque.client.discovery_configs import DiscoveryConfigClient from datamasque.client.files import FileClient +from datamasque.client.git import GitClient from datamasque.client.license import LicenseClient from datamasque.client.projects import ProjectClient from datamasque.client.resources import ResourceClient @@ -30,6 +31,7 @@ class DataMasqueClient( DiscoveryClient, DiscoveryConfigClient, DiscoveryConfigLibraryClient, + GitClient, TableReferenceClient, RGConfigClient, UserClient, diff --git a/datamasque/client/git.py b/datamasque/client/git.py new file mode 100644 index 0000000..af3f261 --- /dev/null +++ b/datamasque/client/git.py @@ -0,0 +1,459 @@ +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import Literal, Optional, Union + +from pydantic import BaseModel + +from datamasque.client.base import BaseClient +from datamasque.client.models.discovery_config import DiscoveryConfig +from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary +from datamasque.client.models.git import GitFile +from datamasque.client.models.ruleset import Ruleset +from datamasque.client.models.ruleset_library import RulesetLibrary + +GitExtension = Literal[".yml", ".yaml"] +"""The extensions the server accepts for a pushed file.""" + +DEFAULT_GIT_EXTENSION: GitExtension = ".yml" + + +@dataclass(frozen=True) +class _GitResourceType: + """The endpoint and request field names one resource type's git operations use.""" + + endpoint: str + name_field: str + content_field: str + folder_param: str + """The pull parameter naming the file's folder, as the file list shows it.""" + + +_RULESET = _GitResourceType("/api/ruleset-git/", "ruleset_name", "ruleset_content", "folder") +_RULESET_LIBRARY = _GitResourceType("/api/library-git/", "library_name", "library_content", "namespace") +_DISCOVERY_CONFIG = _GitResourceType("/api/discovery-config-git/", "config_name", "config_content", "folder") +_DISCOVERY_CONFIG_LIBRARY = _GitResourceType( + "/api/discovery-config-library-git/", "library_name", "library_content", "namespace" +) + + +class _GitPull(BaseModel): + """The response to a pull.""" + + config_yaml: str + + +class _GitPushPath(BaseModel): + """The response to a push-path request.""" + + path: str + + +def _branch_param(branch: Optional[str]) -> dict[str, str]: + return {} if branch is None else {"branch": branch} + + +def _require_yaml(yaml: Optional[str], description: str, getter: str) -> str: + if not yaml: + raise ValueError( + f"Cannot push a {description} to git without YAML content (yaml is empty or unset); " + f"fetch the {description} with its YAML using `{getter}` first" + ) + + return yaml + + +class GitClient(BaseClient): + """ + Git push, pull and file listing for rulesets, ruleset libraries, discovery configs and their libraries. + + Mixed into `DataMasqueClient`. + The server reads and writes the repository in its Git settings with the user's SSH key, + so both must be set up for the user the client authenticates as. + Each `branch` parameter defaults to the branch in the Git settings. + + A push writes `//` under the Git directory path, + where `` is the project's `name`, not its `reference`, + and the type folders are `rulesets/`, `libraries/`, `discovery-configs/` and `discovery-config-libraries/`. + A resource pushed before the server's upgrade to projects keeps the file it had then, + until it is renamed or moved to another project: + a ruleset at the top of the Git directory path, + a discovery config directly in `discovery-configs/`, + a library of the `Default` project directly in its type folder, + and a library of another namespace in the namespace's folder, + such as `libraries/bank/au/` for the namespace that the upgrade made the project `bank_au`. + + The methods give paths in two forms: + + - `get_*_git_push_path` returns the file's path in the repository without its extension, + starting with the Git directory path. + - `list_*_git_files` returns each file's path with its extension, + relative to the Git directory path for rulesets and to the type folder for the other types. + The matching `pull_*_from_git` method takes a path in this form. + + For example, to restore a ruleset to the commit before its latest one and push it back: + + .. code-block:: python + + banking = next(project for project in client.iter_projects() if project.name == "Banking") + file = next(f for f in client.list_ruleset_git_files() if f.path == "rulesets/Banking/customers.yml") + yaml = client.pull_ruleset_from_git(file.path, file.commits[1].commit_sha) + ruleset = client.create_or_update_ruleset(Ruleset(name="customers", project=banking.reference, yaml=yaml)) + client.push_ruleset_to_git(ruleset, "Restore the previous customers ruleset") + """ + + def list_ruleset_git_files(self, *, branch: Optional[str] = None) -> list[GitFile]: + """ + Lists the ruleset files in the git repository. + + Paths are relative to the Git directory path: + `rulesets//customers.yml` for a project's ruleset, + and `customers.yml` for one pushed before the server's upgrade to projects. + """ + + return self._list_git_files(_RULESET, branch) + + def pull_ruleset_from_git(self, path: str, commit_sha: str, *, branch: Optional[str] = None) -> str: + """ + Returns the YAML of a ruleset file at `commit_sha`. + + `path` is the file's path as `list_ruleset_git_files` lists it. + Saves nothing: set it as a ruleset's `yaml` and save that ruleset to keep it. + """ + + return self._pull_from_git(_RULESET, path, commit_sha, branch) + + def push_ruleset_to_git( + self, + ruleset: Ruleset, + commit_message: str, + *, + branch: Optional[str] = None, + extension: GitExtension = DEFAULT_GIT_EXTENSION, + ) -> None: + """ + Commits the ruleset's `yaml` to the git repository and pushes it. + + The file is the ruleset's name with `extension` (`.yml` or `.yaml`) in its project's folder, + unless the saved ruleset kept an older file (see `GitClient`). + A push with the other extension writes a second file and leaves the first. + With no `project` set, the project is the saved ruleset's when it has an `id`, + else the client's `default_project`, else the server's only active project. + The ruleset's `id`, else its name, type and project, identify the saved ruleset the push is for. + The push sends the given ruleset's `yaml`, not the saved one, so save the ruleset first. + The server records the pushed commit as the saved ruleset's git provenance when their YAML matches, + but the given ruleset is not updated. + + Raises: + ValueError: The ruleset's `yaml` is empty. + DataMasqueApiError: With status 409 when another ruleset uses or would write the file, + and with status 400 when nothing changed, the repository has a file where the path needs a folder, + or no project is set and the server does not have exactly one active project. + """ + + self._push_to_git( + _RULESET, + self._ruleset_git_target(ruleset), + _require_yaml(ruleset.yaml, "ruleset", "list_rulesets"), + commit_message, + branch=branch, + extension=extension, + ) + + def get_ruleset_git_push_path(self, ruleset: Ruleset) -> str: + """ + Returns the path `push_ruleset_to_git` would write the ruleset to. + + The path is relative to the repository and has no extension. + """ + + return self._get_git_push_path(_RULESET, self._ruleset_git_target(ruleset)) + + def list_ruleset_library_git_files(self, *, branch: Optional[str] = None) -> list[GitFile]: + """ + Lists the ruleset library files in the git repository. + + Paths are relative to `libraries/` under the Git directory path: + `/emails.yml` for a project's library, + `emails.yml` for a library of the `Default` project pushed before the server's upgrade to projects, + and `/emails.yml` for a library in a namespace whose project the upgrade named differently, + such as `bank/au/emails.yml` in the project `bank_au`. + """ + + return self._list_git_files(_RULESET_LIBRARY, branch) + + def pull_ruleset_library_from_git(self, path: str, commit_sha: str, *, branch: Optional[str] = None) -> str: + """ + Returns the YAML of a library file at `commit_sha`. + + `path` is the file's path as `list_ruleset_library_git_files` lists it. + Saves nothing: set it as a library's `yaml` and save that library to keep it. + """ + + return self._pull_from_git(_RULESET_LIBRARY, path, commit_sha, branch) + + def push_ruleset_library_to_git( + self, + library: RulesetLibrary, + commit_message: str, + *, + branch: Optional[str] = None, + extension: GitExtension = DEFAULT_GIT_EXTENSION, + ) -> None: + """ + Commits the library's `yaml` to the git repository and pushes it. + + The file is the library's name with `extension` (`.yml` or `.yaml`) in its project's folder, + unless the saved library kept an older file (see `GitClient`). + A push with the other extension writes a second file and leaves the first. + With no `project` set, the project is the saved library's when it has an `id`, + else the client's `default_project`, else the server's only active project. + The library's `id`, else its name and project, identify the saved library the push is for. + The push sends the given library's `yaml`, not the saved one, so save the library first. + The server records the pushed commit as the saved library's git provenance when their YAML matches, + but the given library is not updated. + + Raises: + ValueError: The library's `yaml` is empty or unset. + DataMasqueApiError: With status 409 when another library uses or would write the file, + and with status 400 when nothing changed, the repository has a file where the path needs a folder, + or no project is set and the server does not have exactly one active project. + """ + + self._push_to_git( + _RULESET_LIBRARY, + self._library_git_target(_RULESET_LIBRARY, library), + _require_yaml(library.yaml, "ruleset library", "get_ruleset_library"), + commit_message, + branch=branch, + extension=extension, + ) + + def get_ruleset_library_git_push_path(self, library: RulesetLibrary) -> str: + """ + Returns the path `push_ruleset_library_to_git` would write the library to. + + The path is relative to the repository and has no extension. + """ + + return self._get_git_push_path(_RULESET_LIBRARY, self._library_git_target(_RULESET_LIBRARY, library)) + + def list_discovery_config_git_files(self, *, branch: Optional[str] = None) -> list[GitFile]: + """ + Lists the discovery config files in the git repository. + + Paths are relative to `discovery-configs/` under the Git directory path: + `/customers.yml` for a project's config, + and `customers.yml` for one pushed before the server's upgrade to projects. + """ + + return self._list_git_files(_DISCOVERY_CONFIG, branch) + + def pull_discovery_config_from_git(self, path: str, commit_sha: str, *, branch: Optional[str] = None) -> str: + """ + Returns the YAML of a config file at `commit_sha`. + + `path` is the file's path as `list_discovery_config_git_files` lists it. + Saves nothing: set it as a config's `yaml` and save that config to keep it. + """ + + return self._pull_from_git(_DISCOVERY_CONFIG, path, commit_sha, branch) + + def push_discovery_config_to_git( + self, + config: DiscoveryConfig, + commit_message: str, + *, + branch: Optional[str] = None, + extension: GitExtension = DEFAULT_GIT_EXTENSION, + ) -> None: + """ + Commits the discovery config's `yaml` to the git repository and pushes it. + + The file is the config's name with `extension` (`.yml` or `.yaml`) in its project's folder, + unless the saved config kept an older file (see `GitClient`). + A push with the other extension writes a second file and leaves the first. + With no `project` set, the project is the saved config's when it has an `id`, + else the client's `default_project`, else the server's only active project. + The config's `id`, else its name, type and project, identify the saved config the push is for. + The push sends the given config's `yaml`, not the saved one, so save the config first. + The server records the pushed commit on the saved config when their YAML matches, + but `DiscoveryConfig` does not expose it. + + Raises: + ValueError: The config's `yaml` is empty or unset. + DataMasqueApiError: With status 409 when another config uses or would write the file, + and with status 400 when nothing changed, the repository has a file where the path needs a folder, + or no project is set and the server does not have exactly one active project. + """ + + self._push_to_git( + _DISCOVERY_CONFIG, + self._discovery_config_git_target(config), + _require_yaml(config.yaml, "discovery config", "get_discovery_config"), + commit_message, + branch=branch, + extension=extension, + ) + + def get_discovery_config_git_push_path(self, config: DiscoveryConfig) -> str: + """ + Returns the path `push_discovery_config_to_git` would write the config to. + + The path is relative to the repository and has no extension. + """ + + return self._get_git_push_path(_DISCOVERY_CONFIG, self._discovery_config_git_target(config)) + + def list_discovery_config_library_git_files(self, *, branch: Optional[str] = None) -> list[GitFile]: + """ + Lists the discovery config library files in the git repository. + + Paths are relative to `discovery-config-libraries/` under the Git directory path: + `/emails.yml` for a project's library, + `emails.yml` for a library of the `Default` project pushed before the server's upgrade to projects, + and `/emails.yml` for a library in a namespace whose project the upgrade named differently, + such as `bank/au/emails.yml` in the project `bank_au`. + """ + + return self._list_git_files(_DISCOVERY_CONFIG_LIBRARY, branch) + + def pull_discovery_config_library_from_git( + self, path: str, commit_sha: str, *, branch: Optional[str] = None + ) -> str: + """ + Returns the YAML of a library file at `commit_sha`. + + `path` is the file's path as `list_discovery_config_library_git_files` lists it. + Saves nothing: set it as a library's `yaml` and save that library to keep it. + """ + + return self._pull_from_git(_DISCOVERY_CONFIG_LIBRARY, path, commit_sha, branch) + + def push_discovery_config_library_to_git( + self, + library: DiscoveryConfigLibrary, + commit_message: str, + *, + branch: Optional[str] = None, + extension: GitExtension = DEFAULT_GIT_EXTENSION, + ) -> None: + """ + Commits the discovery config library's `yaml` to the git repository and pushes it. + + The file is the library's name with `extension` (`.yml` or `.yaml`) in its project's folder, + unless the saved library kept an older file (see `GitClient`). + A push with the other extension writes a second file and leaves the first. + With no `project` set, the project is the saved library's when it has an `id`, + else the client's `default_project`, else the server's only active project. + The library's `id`, else its name and project, identify the saved library the push is for. + The push sends the given library's `yaml`, not the saved one, so save the library first. + The server records the pushed commit on the saved library when their YAML matches, + but `DiscoveryConfigLibrary` does not expose it. + + Raises: + ValueError: The library's `yaml` is empty or unset. + DataMasqueApiError: With status 409 when another library uses or would write the file, + and with status 400 when nothing changed, the repository has a file where the path needs a folder, + or no project is set and the server does not have exactly one active project. + """ + + self._push_to_git( + _DISCOVERY_CONFIG_LIBRARY, + self._library_git_target(_DISCOVERY_CONFIG_LIBRARY, library), + _require_yaml(library.yaml, "discovery config library", "get_discovery_config_library"), + commit_message, + branch=branch, + extension=extension, + ) + + def get_discovery_config_library_git_push_path(self, library: DiscoveryConfigLibrary) -> str: + """ + Returns the path `push_discovery_config_library_to_git` would write the library to. + + The path is relative to the repository and has no extension. + """ + + return self._get_git_push_path( + _DISCOVERY_CONFIG_LIBRARY, + self._library_git_target(_DISCOVERY_CONFIG_LIBRARY, library), + ) + + def _ruleset_git_target(self, ruleset: Ruleset) -> dict[str, str]: + return self._git_target( + _RULESET, ruleset.name, ruleset.project, ruleset.id, mask_type=ruleset.ruleset_type.value + ) + + def _discovery_config_git_target(self, config: DiscoveryConfig) -> dict[str, str]: + return self._git_target( + _DISCOVERY_CONFIG, config.name, config.project, config.id, config_type=config.config_type.value + ) + + def _library_git_target( + self, resource_type: _GitResourceType, library: Union[RulesetLibrary, DiscoveryConfigLibrary] + ) -> dict[str, str]: + return self._git_target(resource_type, library.name, library.project, library.id) + + def _git_target( + self, + resource_type: _GitResourceType, + name: str, + project: Optional[str], + resource_id: Optional[str], + **type_fields: str, + ) -> dict[str, str]: + """ + Returns the fields that pick the file a push writes, as the push and push-path requests take them. + + An unset `project` is left for the server to take from the saved resource when there is an `id`, + and is otherwise the client's `default_project` when there is one. + """ + + fields: dict[str, str] = {resource_type.name_field: name, **type_fields} + if resource_id is not None: + fields["id"] = resource_id + if project is not None: + fields["project"] = project + elif resource_id is None: + fields |= self._default_project_field(None) + + return fields + + def _list_git_files(self, resource_type: _GitResourceType, branch: Optional[str]) -> list[GitFile]: + response = self.make_request("GET", f"{resource_type.endpoint}files/", params=_branch_param(branch)) + return [GitFile(path=path, commits=commits) for path, commits in response.json().items()] + + def _pull_from_git(self, resource_type: _GitResourceType, path: str, commit_sha: str, branch: Optional[str]) -> str: + file_path = PurePosixPath(path) + folder = file_path.parent + params = { + resource_type.name_field: file_path.stem, + "extension": file_path.suffix, + resource_type.folder_param: "" if folder == PurePosixPath(".") else folder.as_posix(), + "commit_id": commit_sha, + **_branch_param(branch), + } + response = self.make_request("GET", resource_type.endpoint, params=params) + return _GitPull.model_validate(response.json()).config_yaml + + def _push_to_git( + self, + resource_type: _GitResourceType, + target: dict[str, str], + content: str, + commit_message: str, + *, + branch: Optional[str], + extension: GitExtension, + ) -> None: + data = { + **target, + resource_type.content_field: content, + "commit_message": commit_message, + "extension": extension, + **_branch_param(branch), + } + self.make_request("POST", resource_type.endpoint, data=data) + + def _get_git_push_path(self, resource_type: _GitResourceType, target: dict[str, str]) -> str: + response = self.make_request("GET", f"{resource_type.endpoint}push-path/", params=target) + return _GitPushPath.model_validate(response.json()).path diff --git a/datamasque/client/models/git.py b/datamasque/client/models/git.py index df8e71a..c409f6c 100644 --- a/datamasque/client/models/git.py +++ b/datamasque/client/models/git.py @@ -58,3 +58,32 @@ def _collapse_git_fields(cls, data: Any) -> Any: data["git"] = snapshot return data + + +class GitFileCommit(BaseModel): + """A commit that changed a file in the git repository.""" + + model_config = ConfigDict(extra="allow", populate_by_name=True) + + commit_sha: str = Field(alias="commit") + """The commit's SHA, which the `pull_*_from_git` methods take.""" + date: datetime + """When the commit was authored.""" + message: str + """The commit message.""" + + +class GitFile(BaseModel): + """A YAML file in the git repository, as a `list_*_git_files` method lists it.""" + + path: str + """ + The file's path with its extension, + relative to the Git directory path for rulesets and to the type folder for the other types. + Pass it to the matching `pull_*_from_git` method. + """ + commits: list[GitFileCommit] + """ + The file's most recent commits, newest first (the server caps how many). + The `pull_*_from_git` methods also take the SHA of an older commit, such as one from `git log`. + """ diff --git a/docs/client.models.rst b/docs/client.models.rst index aa1e198..1ca5c5b 100644 --- a/docs/client.models.rst +++ b/docs/client.models.rst @@ -44,6 +44,13 @@ datamasque.client.models.files module :undoc-members: :show-inheritance: +datamasque.client.models.git module +----------------------------------- + +.. automodule:: datamasque.client.models.git + :members: GitSnapshot, GitFile, GitFileCommit + :show-inheritance: + datamasque.client.models.ifm module ----------------------------------- diff --git a/docs/client.rst b/docs/client.rst index 276a982..baceb9d 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -60,6 +60,14 @@ datamasque.client.files module :undoc-members: :show-inheritance: +datamasque.client.git module +---------------------------- + +.. automodule:: datamasque.client.git + :members: + :undoc-members: + :show-inheritance: + datamasque.client.spcs module ----------------------------- diff --git a/tests/test_git.py b/tests/test_git.py new file mode 100644 index 0000000..98a99d8 --- /dev/null +++ b/tests/test_git.py @@ -0,0 +1,332 @@ +"""Tests for Git push, pull and file listing with the DataMasque client.""" + +from collections.abc import Callable +from datetime import datetime, timezone + +import pytest +import requests_mock +from pydantic import ValidationError + +from datamasque.client import ( + DataMasqueApiError, + DataMasqueClient, + DiscoveryConfig, + DiscoveryConfigLibrary, + DiscoveryConfigType, + GitFile, + Ruleset, + RulesetLibrary, + RulesetType, +) +from datamasque.client.models.dm_instance import DataMasqueInstanceConfig + +BASE_URL = "http://test-server" +COMMIT_SHA = "0123456789abcdef0123456789abcdef01234567" +COMMIT = {"commit": COMMIT_SHA, "date": "2026-10-01T10:00:00+00:00", "message": "Add customers"} + +LIST_METHODS: list[tuple[Callable[[DataMasqueClient], list[GitFile]], str, str]] = [ + (lambda client: client.list_ruleset_git_files(), "/api/ruleset-git/files/", "rulesets/Banking/customers.yml"), + (lambda client: client.list_ruleset_library_git_files(), "/api/library-git/files/", "Banking/emails.yml"), + ( + lambda client: client.list_discovery_config_git_files(), + "/api/discovery-config-git/files/", + "Banking/customers.yml", + ), + ( + lambda client: client.list_discovery_config_library_git_files(), + "/api/discovery-config-library-git/files/", + "Banking/emails.yml", + ), +] + + +@pytest.mark.parametrize(("list_files", "endpoint", "project_file"), LIST_METHODS) +def test_list_git_files_returns_each_file_with_its_commits( + client: DataMasqueClient, + list_files: Callable[[DataMasqueClient], list[GitFile]], + endpoint: str, + project_file: str, +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(f"{BASE_URL}{endpoint}", json={project_file: [COMMIT], "legacy.yml": []}) + + files = list_files(client) + + assert m.last_request.qs == {} + assert [file.path for file in files] == [project_file, "legacy.yml"] + assert files[0].commits[0].commit_sha == COMMIT_SHA + assert files[0].commits[0].date == datetime(2026, 10, 1, 10, tzinfo=timezone.utc) + assert files[0].commits[0].message == "Add customers" + assert files[1].commits == [] + + +def test_list_git_files_sends_the_branch(client: DataMasqueClient) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(f"{BASE_URL}/api/ruleset-git/files/", json={}) + + assert client.list_ruleset_git_files(branch="Feature/X") == [] + + assert m.last_request.qs == {"branch": ["Feature/X"]} + + +@pytest.mark.parametrize( + ("path", "folder", "name", "extension"), + [ + ("rulesets/Banking/Customers.yml", "rulesets/Banking", "Customers", ".yml"), + ("Customers.yaml", "", "Customers", ".yaml"), + ("rulesets/Banking/v1.customers.yml", "rulesets/Banking", "v1.customers", ".yml"), + ], +) +def test_pull_ruleset_from_git_splits_the_listed_path( + client: DataMasqueClient, path: str, folder: str, name: str, extension: str +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(f"{BASE_URL}/api/ruleset-git/", json={"config_yaml": "version: '1.0'"}) + + content = client.pull_ruleset_from_git(path, COMMIT_SHA, branch="main") + + assert content == "version: '1.0'" + assert m.last_request.qs == { + "ruleset_name": [name], + "extension": [extension], + "folder": [folder], + "commit_id": [COMMIT_SHA], + "branch": ["main"], + } + + +@pytest.mark.parametrize( + ("pull", "endpoint", "name_field", "folder_param"), + [ + (DataMasqueClient.pull_ruleset_library_from_git, "/api/library-git/", "library_name", "namespace"), + (DataMasqueClient.pull_discovery_config_from_git, "/api/discovery-config-git/", "config_name", "folder"), + ( + DataMasqueClient.pull_discovery_config_library_from_git, + "/api/discovery-config-library-git/", + "library_name", + "namespace", + ), + ], +) +def test_pull_from_git_sends_each_type_s_fields( + client: DataMasqueClient, + pull: Callable[[DataMasqueClient, str, str], str], + endpoint: str, + name_field: str, + folder_param: str, +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(f"{BASE_URL}{endpoint}", json={"config_yaml": "libraries: []"}) + + assert pull(client, "Bank/AU/Emails.yml", COMMIT_SHA) == "libraries: []" + + assert m.last_request.qs == { + name_field: ["Emails"], + "extension": [".yml"], + folder_param: ["Bank/AU"], + "commit_id": [COMMIT_SHA], + } + + +def test_push_ruleset_to_git_sends_the_ruleset(client: DataMasqueClient) -> None: + ruleset = Ruleset(name="customers", project="7-banking", yaml="version: '1.0'", ruleset_type=RulesetType.file) + ruleset.id = "a-ruleset-id" + with requests_mock.Mocker(case_sensitive=True) as m: + m.post(f"{BASE_URL}/api/ruleset-git/", status_code=204) + + client.push_ruleset_to_git(ruleset, "Update customers", branch="main", extension=".yaml") + + assert m.last_request.json() == { + "ruleset_name": "customers", + "mask_type": "file", + "id": "a-ruleset-id", + "project": "7-banking", + "ruleset_content": "version: '1.0'", + "commit_message": "Update customers", + "extension": ".yaml", + "branch": "main", + } + + +def test_push_and_push_path_leave_a_saved_resource_s_unset_project_to_the_server() -> None: + client = DataMasqueClient( + DataMasqueInstanceConfig(base_url=BASE_URL, username="u", password="p", default_project="1-default") + ) + library = RulesetLibrary(name="emails", yaml="libraries: []") + library.id = "a-library-id" + with requests_mock.Mocker(case_sensitive=True) as m: + m.post(f"{BASE_URL}/api/library-git/", status_code=204) + m.get(f"{BASE_URL}/api/library-git/push-path/", json={"path": "masking/libraries/Banking/emails"}) + + client.push_ruleset_library_to_git(library, "Add emails") + assert client.get_ruleset_library_git_push_path(library) == "masking/libraries/Banking/emails" + + push, push_path = m.request_history + assert push_path.qs == {"library_name": ["emails"], "id": ["a-library-id"]} + assert push.json() == { + "library_name": "emails", + "id": "a-library-id", + "library_content": "libraries: []", + "commit_message": "Add emails", + "extension": ".yml", + } + + +def test_push_and_push_path_send_the_default_project_for_an_unsaved_resource_with_no_project() -> None: + client = DataMasqueClient( + DataMasqueInstanceConfig(base_url=BASE_URL, username="u", password="p", default_project="1-default") + ) + library = DiscoveryConfigLibrary(name="emails", yaml="x: 1") + with requests_mock.Mocker(case_sensitive=True) as m: + m.post(f"{BASE_URL}/api/discovery-config-library-git/", status_code=204) + m.get( + f"{BASE_URL}/api/discovery-config-library-git/push-path/", + json={"path": "masking/discovery-config-libraries/Default/emails"}, + ) + + client.push_discovery_config_library_to_git(library, "Add emails") + client.get_discovery_config_library_git_push_path(library) + + push, push_path = m.request_history + assert push_path.qs == {"library_name": ["emails"], "project": ["1-default"]} + assert push.json() == { + "library_name": "emails", + "project": "1-default", + "library_content": "x: 1", + "commit_message": "Add emails", + "extension": ".yml", + } + + +def test_push_and_push_path_send_no_project_when_neither_the_resource_nor_the_client_sets_one( + client: DataMasqueClient, +) -> None: + ruleset = Ruleset(name="customers", yaml="version: '1.0'") + with requests_mock.Mocker(case_sensitive=True) as m: + m.post(f"{BASE_URL}/api/ruleset-git/", status_code=204) + m.get(f"{BASE_URL}/api/ruleset-git/push-path/", json={"path": "masking/rulesets/Banking/customers"}) + + client.push_ruleset_to_git(ruleset, "Add customers") + client.get_ruleset_git_push_path(ruleset) + + push, push_path = m.request_history + assert "project" not in push.json() + assert "project" not in push_path.qs + + +def test_push_discovery_config_to_git_sends_its_type(client: DataMasqueClient) -> None: + config = DiscoveryConfig(name="customers", project="7-banking", yaml="x: 1", config_type=DiscoveryConfigType.file) + with requests_mock.Mocker(case_sensitive=True) as m: + m.post(f"{BASE_URL}/api/discovery-config-git/", status_code=204) + + client.push_discovery_config_to_git(config, "Add customers") + + assert m.last_request.json() == { + "config_name": "customers", + "config_type": "file", + "project": "7-banking", + "config_content": "x: 1", + "commit_message": "Add customers", + "extension": ".yml", + } + + +@pytest.mark.parametrize( + "push", + [ + lambda client: client.push_ruleset_to_git(Ruleset(name="customers", project="7-banking"), "Add customers"), + lambda client: client.push_ruleset_library_to_git(RulesetLibrary(name="emails", project="7-banking"), "Add"), + lambda client: client.push_discovery_config_to_git( + DiscoveryConfig(name="customers", project="7-banking", config_type=DiscoveryConfigType.database), "Add" + ), + lambda client: client.push_discovery_config_library_to_git( + DiscoveryConfigLibrary(name="emails", project="7-banking", yaml=""), "Add" + ), + ], +) +def test_push_without_yaml_raises(client: DataMasqueClient, push: Callable[[DataMasqueClient], None]) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + with pytest.raises(ValueError, match="yaml is empty or unset"): + push(client) + + assert m.request_history == [] + + +def test_push_raises_on_a_path_conflict(client: DataMasqueClient) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.post(f"{BASE_URL}/api/ruleset-git/", status_code=409, json={"detail": "Cannot push"}) + + with pytest.raises(DataMasqueApiError) as error: + client.push_ruleset_to_git( + Ruleset(name="customers", project="7-banking", yaml="version: '1.0'"), "Add customers" + ) + + assert error.value.response.status_code == 409 + + +@pytest.mark.parametrize( + ("get_push_path", "endpoint", "expected_params"), + [ + ( + lambda client: client.get_ruleset_git_push_path(Ruleset(name="customers", project="7-banking")), + "/api/ruleset-git/push-path/", + {"ruleset_name": ["customers"], "mask_type": ["database"], "project": ["7-banking"]}, + ), + ( + lambda client: client.get_ruleset_library_git_push_path(RulesetLibrary(name="emails", project="7-banking")), + "/api/library-git/push-path/", + {"library_name": ["emails"], "project": ["7-banking"]}, + ), + ( + lambda client: client.get_discovery_config_git_push_path( + DiscoveryConfig(name="customers", project="7-banking", config_type=DiscoveryConfigType.database) + ), + "/api/discovery-config-git/push-path/", + {"config_name": ["customers"], "config_type": ["database"], "project": ["7-banking"]}, + ), + ( + lambda client: client.get_discovery_config_library_git_push_path( + DiscoveryConfigLibrary(name="emails", project="7-banking") + ), + "/api/discovery-config-library-git/push-path/", + {"library_name": ["emails"], "project": ["7-banking"]}, + ), + ], +) +def test_get_git_push_path_returns_the_server_s_path( + client: DataMasqueClient, + get_push_path: Callable[[DataMasqueClient], str], + endpoint: str, + expected_params: dict[str, list[str]], +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(f"{BASE_URL}{endpoint}", json={"path": "masking/rulesets/Banking/customers"}) + + assert get_push_path(client) == "masking/rulesets/Banking/customers" + + assert m.last_request.qs == expected_params + + +@pytest.mark.parametrize( + ("call", "endpoint", "response"), + [ + ( + lambda client: client.pull_ruleset_from_git("rulesets/Banking/customers.yml", COMMIT_SHA), + "/api/ruleset-git/", + {"config_yaml": None}, + ), + ( + lambda client: client.get_ruleset_git_push_path(Ruleset(name="customers", project="7-banking")), + "/api/ruleset-git/push-path/", + {}, + ), + ], +) +def test_a_malformed_git_response_raises( + client: DataMasqueClient, call: Callable[[DataMasqueClient], str], endpoint: str, response: dict[str, object] +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(f"{BASE_URL}{endpoint}", json=response) + + with pytest.raises(ValidationError): + call(client) From 2d463e51ef89505608d4bd5ec2da69f2b8fe1787 Mon Sep 17 00:00:00 2001 From: Elliot Simpson Date: Thu, 29 Oct 2026 12:52:35 +1300 Subject: [PATCH 14/22] feat: DM-4393: add export_bundle and import_bundle for v2 bundles --- HISTORY.rst | 6 + datamasque/client/__init__.py | 16 ++ datamasque/client/bundles.py | 134 ++++++++++++++++ datamasque/client/dmclient.py | 2 + datamasque/client/models/bundle.py | 165 ++++++++++++++++++++ docs/client.models.rst | 8 + docs/client.rst | 8 + tests/test_bundles.py | 238 +++++++++++++++++++++++++++++ 8 files changed, 577 insertions(+) create mode 100644 datamasque/client/bundles.py create mode 100644 datamasque/client/models/bundle.py create mode 100644 tests/test_bundles.py diff --git a/HISTORY.rst b/HISTORY.rst index 9e507f2..d6db351 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -42,6 +42,12 @@ History * ``list_resources`` takes ``ordering`` and ``descending``, and each type's resources come back in that order. Each listed ``Resource`` has a ``position``, its place in that order across every type. Both require server version 3.26.21. +* Added ``export_bundle`` and ``import_bundle``, which export and import Version 2 bundles + that keep each ruleset, library and seed file in its project. + ``import_bundle`` also imports deprecated Version 1 bundles, + putting their items that name no project into ``fallback_project``, + else the server's earliest-created active project; ``default_project`` does not apply. + Both require server version 3.26.21. 1.6.0 (unreleased) ------------------ diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index d276425..d49e8f6 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -29,6 +29,15 @@ ) from datamasque.client.git import GitExtension from datamasque.client.ifm import DataMasqueIfmClient +from datamasque.client.models.bundle import ( + BundleImportCounts, + BundleImportItemResults, + BundleImportResult, + BundleImportStatus, + LibraryImportResult, + RulesetImportResult, + SeedFileImportResult, +) from datamasque.client.models.connection import ( AzureConnectionConfig, CassandraConnectionConfig, @@ -204,6 +213,10 @@ "AzureConnectionConfig", "BooleanPreview", "BooleanStatistics", + "BundleImportCounts", + "BundleImportItemResults", + "BundleImportResult", + "BundleImportStatus", "CassandraConnectionConfig", "ColumnKind", "ColumnPreview", @@ -275,6 +288,7 @@ "LengthEntry", "LengthUnit", "LengthsStatistics", + "LibraryImportResult", "LicenseInfo", "LicenseLock", "Locator", @@ -318,6 +332,7 @@ "RulesetGenerationRequest", "RulesetGenerationWithRGConfigRequest", "RulesetId", + "RulesetImportResult", "RulesetLibrary", "RulesetLibraryId", "RulesetPlan", @@ -342,6 +357,7 @@ "SchemaDiscoveryRequest", "SchemaDiscoveryResult", "SeedFile", + "SeedFileImportResult", "SelectedColumns", "SelectedData", "SelectedFileData", diff --git a/datamasque/client/bundles.py b/datamasque/client/bundles.py new file mode 100644 index 0000000..889629a --- /dev/null +++ b/datamasque/client/bundles.py @@ -0,0 +1,134 @@ +import logging +from collections.abc import Sequence +from typing import Optional, Union + +from datamasque.client.base import BaseClient, FileOrContent, UploadFile, read_file_or_content +from datamasque.client.models.bundle import BundleImportResult +from datamasque.client.models.project import ProjectIdentifier, unwrap_project_id +from datamasque.client.models.ruleset import Ruleset, RulesetId, unwrap_ruleset_id + +logger = logging.getLogger(__name__) + + +class BundleClient(BaseClient): + """ + Export and import bundle API methods. Mixed into `DataMasqueClient`. + + The server accepts these only with the login token that `POST /api/auth/token/login/` returns. + They fail with a 403 for a client whose `token_source` returns a user's long-lived API token, + the `api_token` that `GET /api/users/me/` gives. + """ + + def export_bundle( + self, + rulesets: Optional[Sequence[Union[Ruleset, RulesetId]]] = None, + *, + include_seed_files: bool = False, + ) -> bytes: + """ + Returns a Version 2 export bundle, a Zip archive that files every item under its project's name. + + The bundle holds `rulesets`, given as `Ruleset`s or their IDs, and the libraries they use; + or, when `rulesets` is not given, every ruleset and library. + With `include_seed_files`, it also holds the custom seed files those rulesets and libraries refer to. + Pass the bundle to `import_bundle`, on this server or another, + to put each item in the project of the same name. + For example, `Path("export.zip").write_bytes(client.export_bundle())` saves it to a file. + + Raises: + TypeError: `rulesets` is a single `Ruleset` or ID, not a sequence of them. + ValueError: `rulesets` is empty, or a `Ruleset` in it has not been created on the server. + DataMasqueApiError: the server rejects the request, + for example with a 403 when the client authenticates with a long-lived API token. + """ + + if isinstance(rulesets, (str, Ruleset)): + raise TypeError("Pass `rulesets` as a sequence, such as `[ruleset]`") + if rulesets is not None and not rulesets: + raise ValueError("Pass at least one ruleset, or omit `rulesets` to export every ruleset") + + data: dict[str, object] = {"include_seed_files": include_seed_files} + if rulesets is not None: + data["ruleset_ids"] = [unwrap_ruleset_id(ruleset) for ruleset in rulesets] + + response = self.make_request("POST", "/api/export/v2/", data=data) + return response.content + + def import_bundle( + self, + bundle: FileOrContent, + *, + overwrite_rulesets: Optional[bool] = None, + overwrite_libraries: bool = False, + overwrite_seed_files: bool = False, + fallback_project: Optional[ProjectIdentifier] = None, + ) -> BundleImportResult: + """ + Imports an export bundle, in the Version 2 or deprecated Version 1 format. + + Returns what the import did with each item. + + `bundle` is a path to the Zip archive (as `str` or `Path`), its `bytes`, + or a binary file-like object, such as a file opened with `"rb"`. + Each item goes into the project the bundle files it under, + which is created when it does not exist, if the caller is an administrator. + Items that name no project, such as a Version 1 bundle's rulesets and seed files, + go into `fallback_project`, given as a `Project`, its `ProjectId` or its `-` form, + or else the server's earliest-created active project; + the result's `fallback_project` names the project they went into. + The client's `default_project` does not apply. + + With `overwrite_rulesets` unset, a ruleset whose ID exists is left unchanged, + and a ruleset whose name is in use under another ID is imported under a new name; + `True` overwrites the existing ruleset and `False` leaves it unchanged. + Whatever the options, a ruleset or library whose ID is not on the server is left unchanged + when its name is in use by one with identical content. + `BundleImportStatus` lists every outcome. + `overwrite_libraries` and `overwrite_seed_files` overwrite existing libraries and seed files when `True`, + and leave them unchanged when `False`. + + The import can succeed for some items and fail for others: + check each type's `metadata.error`, or each item's `status`. + + Raises: + ValueError: `fallback_project` is a string that is not a project reference in the form `-`. + DataMasqueApiError: the server rejects the import, and imports nothing. + For example, a 400 when the bundle is not a valid export, `fallback_project` is not an active project, + or the bundle names a project that cannot be created with that name; + or a 403 when the bundle names a project that does not exist and the caller is not an administrator, + or when the client authenticates with a long-lived API token. + """ + + file_name, content = read_file_or_content(bundle, "bundle.zip") + content.seek(0) + + data: dict[str, object] = { + "overwrite_libraries": overwrite_libraries, + "overwrite_seed_files": overwrite_seed_files, + } + if overwrite_rulesets is not None: + data["overwrite_rulesets"] = overwrite_rulesets + if fallback_project is not None: + data["fallback_project"] = str(unwrap_project_id(fallback_project)) + + response = self.make_request( + "POST", + "/api/import/v2/", + data=data, + files=[ + UploadFile( + field_name="zip_archive", + filename=file_name, + content=content, + content_type="application/zip", + ), + ], + ) + result = BundleImportResult.model_validate(response.json()) + result.is_deprecated_bundle = response.headers.get("Deprecation", "").lower() == "true" + if result.is_deprecated_bundle: + logger.warning("Imported a deprecated Version 1 bundle; export a Version 2 bundle to keep each project") + if result.fallback_project is not None: + logger.info('Items that named no project went into project "%s"', result.fallback_project) + + return result diff --git a/datamasque/client/dmclient.py b/datamasque/client/dmclient.py index 3591335..6acaa69 100644 --- a/datamasque/client/dmclient.py +++ b/datamasque/client/dmclient.py @@ -1,4 +1,5 @@ from datamasque.client.base import FileOrContent, UploadFile +from datamasque.client.bundles import BundleClient from datamasque.client.connections import ConnectionClient from datamasque.client.discovery import DiscoveryClient from datamasque.client.discovery_config_libraries import DiscoveryConfigLibraryClient @@ -27,6 +28,7 @@ class DataMasqueClient( RulesetClient, RulesetLibraryClient, FileClient, + BundleClient, RunClient, DiscoveryClient, DiscoveryConfigClient, diff --git a/datamasque/client/models/bundle.py b/datamasque/client/models/bundle.py new file mode 100644 index 0000000..2f26271 --- /dev/null +++ b/datamasque/client/models/bundle.py @@ -0,0 +1,165 @@ +"""Export and import bundle models for the DataMasque API.""" + +from enum import Enum +from typing import Generic, Optional, TypeVar + +from pydantic import BaseModel, ConfigDict + +from datamasque.client.models.ruleset import RulesetId, RulesetType +from datamasque.client.models.ruleset_library import RulesetLibraryId + + +class BundleImportStatus(str, Enum): + """What an import did with one item of a bundle.""" + + created = "CREATED" + """No item with that ID or name existed, so it was created. Counted in `created`.""" + created_duplicate = "CREATED_DUPLICATE" + """ + A ruleset with that name existed and `overwrite_rulesets` was not set, + so it was created under a new name, given in `imported_name`. + Counted in `created`. + """ + not_created = "NOT_CREATED" + """ + The item was left unchanged, because an item with that name already has identical content, + or because a ruleset with that ID exists and `overwrite_rulesets` was not set. + Not counted in any count but `processed`. + """ + restored = "RESTORED" + """ + An item with that ID, or with that name in the same project, had been deleted, + so it was brought back with the bundle's name and content. + Counted in `restored`. + """ + updated = "UPDATED" + """ + An existing item was overwritten, because the import's `overwrite_*` option for its type was `True`. + Counted in `updated`. + """ + skipped = "SKIPPED" + """ + An existing item was left unchanged, because the import's `overwrite_*` option for its type was `False`. + Counted in `skipped`. + """ + error = "ERROR" + """The item could not be imported; `message` says why. Counted in `error`.""" + + +class BundleImportCounts(BaseModel): + """How many items of one type an import processed, and what it did with them.""" + + model_config = ConfigDict(extra="allow") + + processed: int + """The number of items of the type in the bundle.""" + created: int + """The number of `CREATED` and `CREATED_DUPLICATE` items.""" + restored: int + """The number of `RESTORED` items.""" + error: int + """The number of `ERROR` items.""" + updated: Optional[int] = None + """The number of `UPDATED` items, or `None` for rulesets imported without `overwrite_rulesets`.""" + skipped: Optional[int] = None + """The number of `SKIPPED` items, or `None` for rulesets imported without `overwrite_rulesets`.""" + + +class RulesetImportResult(BaseModel): + """What an import did with one ruleset of a bundle.""" + + model_config = ConfigDict(extra="allow") + + exported_name: str + exported_id: Optional[RulesetId] = None + """`None` for a hand-built Zip archive without the `manifest.json` that an export writes.""" + exported_project: Optional[str] = None + """The name of the project the bundle files the ruleset under, or `None` for a Version 1 bundle.""" + imported_name: Optional[str] = None + """`None` on error.""" + imported_id: Optional[RulesetId] = None + """`None` on error.""" + imported_project: Optional[str] = None + """The name of the project the ruleset is in, or `None` on error.""" + ruleset_type: RulesetType + status: BundleImportStatus + message: str + """A description of the outcome, which can change between DataMasque versions; check `status` instead.""" + + +class LibraryImportResult(BaseModel): + """What an import did with one library of a bundle.""" + + model_config = ConfigDict(extra="allow") + + exported_name: str + exported_namespace: str + """ + The folder a Version 1 bundle files the library under, which names its project; + `""` for a library in no folder, and for every library in a Version 2 bundle. + """ + exported_project: Optional[str] = None + """The name of the project the bundle files the library under, or `None` for a Version 1 bundle.""" + exported_id: Optional[RulesetLibraryId] = None + """`None` for a hand-built Zip archive without the `manifest.json` that an export writes.""" + imported_name: Optional[str] = None + """`None` on error.""" + imported_id: Optional[RulesetLibraryId] = None + """`None` on error.""" + imported_project: Optional[str] = None + """The name of the project the library is in, or `None` on error.""" + status: BundleImportStatus + message: str + """A description of the outcome, which can change between DataMasque versions; check `status` instead.""" + + +class SeedFileImportResult(BaseModel): + """What an import did with one seed file of a bundle.""" + + model_config = ConfigDict(extra="allow") + + exported_name: str + exported_project: Optional[str] = None + """The name of the project the bundle files the seed file under, or `None` for a Version 1 bundle.""" + imported_name: Optional[str] = None + """`None` on error.""" + imported_project: Optional[str] = None + """The name of the project the seed file is in, or `None` on error.""" + status: BundleImportStatus + message: str + """A description of the outcome, which can change between DataMasque versions; check `status` instead.""" + + +ItemResultT = TypeVar("ItemResultT", RulesetImportResult, LibraryImportResult, SeedFileImportResult) + + +class BundleImportItemResults(BaseModel, Generic[ItemResultT]): + """The counts and per-item results of one item type in an import.""" + + model_config = ConfigDict(extra="allow") + + metadata: BundleImportCounts + data: list[ItemResultT] + + +class BundleImportResult(BaseModel): + """The outcome of `import_bundle`, grouped by item type.""" + + model_config = ConfigDict(extra="allow") + + rulesets: BundleImportItemResults[RulesetImportResult] + libraries: Optional[BundleImportItemResults[LibraryImportResult]] = None + """`None` when the bundle has no libraries.""" + seed_files: Optional[BundleImportItemResults[SeedFileImportResult]] = None + """`None` when the bundle has no seed files.""" + fallback_project: Optional[str] = None + """ + The name of the project that took the items that named no project, such as a Version 1 bundle's rulesets. + `None` when every item went into the project it names. + """ + is_deprecated_bundle: bool = False + """ + Whether the bundle is in the deprecated Version 1 format, + or is a hand-built Zip archive without `manifest.json`, which is read as Version 1. + `import_bundle` sets it from the response's `Deprecation` header, which the response body does not carry. + """ diff --git a/docs/client.models.rst b/docs/client.models.rst index 1ca5c5b..54c271f 100644 --- a/docs/client.models.rst +++ b/docs/client.models.rst @@ -4,6 +4,14 @@ datamasque.client.models package Submodules ---------- +datamasque.client.models.bundle module +-------------------------------------- + +.. automodule:: datamasque.client.models.bundle + :members: + :undoc-members: + :show-inheritance: + datamasque.client.models.connection module ------------------------------------------ diff --git a/docs/client.rst b/docs/client.rst index baceb9d..fabcb5f 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -20,6 +20,14 @@ datamasque.client.base module :undoc-members: :show-inheritance: +datamasque.client.bundles module +-------------------------------- + +.. automodule:: datamasque.client.bundles + :members: + :undoc-members: + :show-inheritance: + datamasque.client.connections module ------------------------------------ diff --git a/tests/test_bundles.py b/tests/test_bundles.py new file mode 100644 index 0000000..3939511 --- /dev/null +++ b/tests/test_bundles.py @@ -0,0 +1,238 @@ +"""Tests for exporting and importing bundles with the DataMasque client.""" + +import json +from io import BytesIO +from pathlib import Path + +import pytest +import requests_mock + +from datamasque.client import ( + BundleImportStatus, + DataMasqueApiError, + DataMasqueClient, + Ruleset, + RulesetId, + RulesetType, +) +from tests.helpers import OTHER_PROJECT, parse_multipart_form + +BUNDLE = b"PK\x03\x04 bundle" + +V2_IMPORT_RESPONSE = { + "rulesets": { + "metadata": {"processed": 1, "created": 1, "restored": 0, "error": 0}, + "data": [ + { + "exported_name": "Customers", + "exported_project": "payments", + "exported_id": "9d641e97-adf7-4f22-9089-afc3711bf222", + "imported_name": "Customers", + "imported_id": "9d641e97-adf7-4f22-9089-afc3711bf222", + "imported_project": "payments", + "ruleset_type": "database", + "status": "CREATED", + "message": 'Ruleset named "Customers" did not exist, and was created.', + } + ], + }, + "libraries": { + "metadata": {"processed": 1, "created": 0, "restored": 0, "error": 0, "updated": 0, "skipped": 1}, + "data": [ + { + "exported_name": "pii_masks", + "exported_namespace": "", + "exported_project": "payments", + "exported_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "imported_name": "pii_masks", + "imported_namespace": "payments", + "imported_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "imported_project": "payments", + "status": "SKIPPED", + "message": 'A library with ID "a1b2c3d4-e5f6-7890-abcd-ef1234567890" already exists.', + } + ], + }, +} + +V1_IMPORT_RESPONSE = { + "rulesets": { + "metadata": {"processed": 1, "created": 1, "restored": 0, "error": 0}, + "data": [ + { + "exported_name": "Customers", + "exported_project": None, + "exported_id": None, + "imported_name": "Customers", + "imported_id": "04ea20f0-ad4c-498e-881f-b0bc79d83ba7", + "imported_project": "Banking", + "ruleset_type": "file", + "status": "CREATED", + "message": 'Ruleset named "Customers" did not exist, and was created.', + } + ], + }, + "seed_files": { + "metadata": {"processed": 1, "created": 0, "restored": 0, "error": 1, "updated": 0, "skipped": 0}, + "data": [ + { + "exported_name": "names.csv", + "exported_project": None, + "imported_name": None, + "imported_project": None, + "status": "ERROR", + "message": "The seed file could not be written.", + } + ], + }, + "fallback_project": "Banking", +} + + +def test_export_bundle_exports_everything_by_default(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.post("http://test-server/api/export/v2/", content=BUNDLE, headers={"Content-Type": "application/zip"}) + bundle = client.export_bundle() + + assert bundle == BUNDLE + assert m.last_request.json() == {"include_seed_files": False} + + +def test_export_bundle_sends_the_rulesets_and_seed_file_option(client: DataMasqueClient) -> None: + ruleset = Ruleset(name="Customers") + ruleset.id = RulesetId("9d641e97-adf7-4f22-9089-afc3711bf222") + with requests_mock.Mocker() as m: + m.post("http://test-server/api/export/v2/", content=BUNDLE) + client.export_bundle([ruleset, RulesetId("04ea20f0-ad4c-498e-881f-b0bc79d83ba7")], include_seed_files=True) + + assert m.last_request.json() == { + "include_seed_files": True, + "ruleset_ids": ["9d641e97-adf7-4f22-9089-afc3711bf222", "04ea20f0-ad4c-498e-881f-b0bc79d83ba7"], + } + + +def test_export_bundle_rejects_an_empty_sequence_of_rulesets(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + with pytest.raises(ValueError, match="at least one ruleset"): + client.export_bundle([]) + + assert not m.called + + +def test_export_bundle_rejects_a_ruleset_not_yet_created(client: DataMasqueClient) -> None: + with pytest.raises(ValueError, match="not been created"): + client.export_bundle([Ruleset(name="Customers")]) + + +@pytest.mark.parametrize("rulesets", [RulesetId("9d641e97-adf7-4f22-9089-afc3711bf222"), Ruleset(name="Customers")]) +def test_export_bundle_rejects_a_single_ruleset(client: DataMasqueClient, rulesets: object) -> None: + with pytest.raises(TypeError, match="as a sequence"): + client.export_bundle(rulesets) # type: ignore[arg-type] + + +def test_import_bundle_uploads_the_archive_with_default_options(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.post("http://test-server/api/import/v2/", json=V2_IMPORT_RESPONSE, status_code=201) + result = client.import_bundle(BUNDLE) + + form = parse_multipart_form(m.last_request) + assert form == { + "overwrite_libraries": "False", + "overwrite_seed_files": "False", + "zip_archive": {"filename": "bundle.zip", "content_type": "application/zip", "content": BUNDLE}, + } + (ruleset,) = result.rulesets.data + assert ruleset.imported_project == "payments" + assert ruleset.ruleset_type is RulesetType.database + assert ruleset.status is BundleImportStatus.created + assert result.rulesets.metadata.updated is None + assert result.libraries is not None + assert result.libraries.metadata.skipped == 1 + assert result.libraries.data[0].status is BundleImportStatus.skipped + assert result.seed_files is None + assert result.fallback_project is None + assert result.is_deprecated_bundle is False + + +@pytest.mark.parametrize(("overwrite", "form_value"), [(True, "True"), (False, "False")]) +def test_import_bundle_sends_its_options( + client: DataMasqueClient, tmp_path: Path, overwrite: bool, form_value: str +) -> None: + bundle_path = tmp_path / "export.zip" + bundle_path.write_bytes(BUNDLE) + with requests_mock.Mocker() as m: + m.post("http://test-server/api/import/v2/", json=V2_IMPORT_RESPONSE) + client.import_bundle( + bundle_path, + overwrite_rulesets=overwrite, + overwrite_libraries=overwrite, + overwrite_seed_files=overwrite, + fallback_project=OTHER_PROJECT, + ) + + form = parse_multipart_form(m.last_request) + assert form["zip_archive"]["filename"] == "export.zip" + assert form["overwrite_rulesets"] == form_value + assert form["overwrite_libraries"] == form_value + assert form["overwrite_seed_files"] == form_value + assert form["fallback_project"] == "2" + + +def test_import_bundle_uploads_all_of_a_partly_read_file(client: DataMasqueClient) -> None: + bundle = BytesIO(BUNDLE) + bundle.read(4) + with requests_mock.Mocker() as m: + m.post("http://test-server/api/import/v2/", json=V2_IMPORT_RESPONSE) + client.import_bundle(bundle) + + archive = parse_multipart_form(m.last_request)["zip_archive"] + assert archive["filename"] == "bundle.zip" + assert archive["content"] == BUNDLE + + +def test_import_bundle_rejects_a_fallback_project_that_is_not_a_reference(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + with pytest.raises(ValueError, match="not a project ID or reference"): + client.import_bundle(BUNDLE, fallback_project="banking") + + assert not m.called + + +def test_import_bundle_does_not_send_the_client_default_project(client: DataMasqueClient) -> None: + client.default_project = "1-default" + with requests_mock.Mocker() as m: + m.post("http://test-server/api/import/v2/", json=V1_IMPORT_RESPONSE) + client.import_bundle(BUNDLE) + + assert "fallback_project" not in parse_multipart_form(m.last_request) + + +def test_import_bundle_reports_a_v1_bundle_and_its_fallback_project(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.post( + "http://test-server/api/import/v2/", + json=V1_IMPORT_RESPONSE, + status_code=201, + headers={"Deprecation": "true"}, + ) + result = client.import_bundle(BUNDLE, fallback_project="2-banking") + + assert parse_multipart_form(m.last_request)["fallback_project"] == "2" + assert result.is_deprecated_bundle is True + assert result.fallback_project == "Banking" + assert result.rulesets.data[0].exported_project is None + assert result.seed_files is not None + (seed_file,) = result.seed_files.data + assert seed_file.status is BundleImportStatus.error + assert seed_file.imported_project is None + + +def test_import_bundle_raises_when_the_server_rejects_it(client: DataMasqueClient) -> None: + detail = 'The bundle names the project "payments", which does not exist. Creating it requires an administrator.' + with requests_mock.Mocker() as m: + m.post("http://test-server/api/import/v2/", status_code=403, text=json.dumps({"detail": detail})) + with pytest.raises(DataMasqueApiError) as error: + client.import_bundle(BUNDLE) + + assert error.value.response.status_code == 403 + assert error.value.response.json() == {"detail": detail} From a7059dc14fe04df41e573ccffaea529178487133 Mon Sep 17 00:00:00 2001 From: Andrew Chester Date: Tue, 27 Oct 2026 14:56:43 +1300 Subject: [PATCH 15/22] DM-4391 feat(projects): add update_project and has_git_synced_resources --- HISTORY.rst | 5 ++ datamasque/client/__init__.py | 2 + datamasque/client/models/project.py | 22 ++++++ datamasque/client/projects.py | 25 +++++++ tests/test_projects.py | 106 +++++++++++++++++++++++++++- 5 files changed, 159 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index d6db351..828fd65 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -48,6 +48,11 @@ History putting their items that name no project into ``fallback_project``, else the server's earliest-created active project; ``default_project`` does not apply. Both require server version 3.26.21. +* Added ``update_project``, which takes a ``ProjectUpdateRequest`` to rename a project or change its description, + and returns the updated ``Project``. + Only admin users can update projects, and API tokens cannot. +* ``get_project`` sets ``Project.has_git_synced_resources``, which says whether the project holds a git-synced resource. + A rename leaves a git-synced library or discovery config library at its old repository path. 1.6.0 (unreleased) ------------------ diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index d49e8f6..5399310 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -133,6 +133,7 @@ ProjectDeleteConflictReason, ProjectId, ProjectIdentifier, + ProjectUpdateRequest, ) from datamasque.client.models.resource import ( ConnectionResource, @@ -317,6 +318,7 @@ "ProjectDeleteConflictReason", "ProjectId", "ProjectIdentifier", + "ProjectUpdateRequest", "RGConfig", "RGConfigId", "RGConfigNotFoundError", diff --git a/datamasque/client/models/project.py b/datamasque/client/models/project.py index c925f03..b295b45 100644 --- a/datamasque/client/models/project.py +++ b/datamasque/client/models/project.py @@ -37,6 +37,14 @@ class Project(BaseModel): description: str = "" created: datetime updated: datetime + has_git_synced_resources: Optional[bool] = None + """ + Whether any library, discovery config library, ruleset or discovery config in the project + that is not deleted is synced with git. + + Only `get_project` sets it; other calls leave it `None`. + A rename leaves a git-synced library or discovery config library at its old repository path. + """ # A project, given as a `Project`, its `ProjectId`, or its `-` form such as a listed resource's `project`. @@ -80,6 +88,20 @@ class ProjectCreateRequest(BaseModel): description: str = "" +class ProjectUpdateRequest(BaseModel): + """ + Request body for `PATCH /api/projects/{id}/`. + + A field left as `None` is not sent, so the server keeps its current value. + A new `name` follows the rules of `ProjectCreateRequest`. + """ + + model_config = ConfigDict(extra="forbid") + + name: Optional[str] = None + description: Optional[str] = None + + class ProjectDeleteConflictReason(str, Enum): """Why a project's resources cannot move to the destination project when the project is deleted.""" diff --git a/datamasque/client/projects.py b/datamasque/client/projects.py index ce03ede..ba40839 100644 --- a/datamasque/client/projects.py +++ b/datamasque/client/projects.py @@ -10,6 +10,7 @@ ProjectCreateRequest, ProjectDeleteConflict, ProjectIdentifier, + ProjectUpdateRequest, unwrap_project_id, ) @@ -66,6 +67,30 @@ def create_project(self, request: ProjectCreateRequest) -> Project: logger.info('Creation of project "%s" successful', project.name) return project + def update_project(self, project: ProjectIdentifier, request: ProjectUpdateRequest) -> Project: + """ + Renames a project or changes its description, and returns the updated project. + + Only admin users can, and API tokens cannot. + Renaming moves the project's seed and credential files to the folder of the new name, + and invalidates rulesets, discovery configs and generator configs that import a library by its old name. + A rename that only changes the case of the name moves no files, and a file outside the project's folder stays. + Renaming the Default project invalidates imports that name its libraries without a project name. + Seed and table references in rulesets that name the old project are not rewritten. + + Raises: + DataMasqueApiError: the server rejects the name (see `ProjectCreateRequest`), + a file already exists where one of the project's files would move, + the project does not exist or is deleted, or any other non-2xx response. + """ + + project_id = unwrap_project_id(project) + data = request.model_dump(mode="json", exclude_none=True) + response = self.make_request("PATCH", f"/api/projects/{project_id}/", data=data) + updated_project = Project.model_validate(response.json()) + logger.info("Update of project %s successful", project_id) + return updated_project + def get_project_delete_conflicts( self, project: ProjectIdentifier, diff --git a/tests/test_projects.py b/tests/test_projects.py index ff94b7a..11c3a96 100644 --- a/tests/test_projects.py +++ b/tests/test_projects.py @@ -1,7 +1,7 @@ """Tests for project support in the DataMasque client.""" from datetime import datetime -from typing import Callable +from typing import Callable, Optional import pytest import requests_mock @@ -17,6 +17,7 @@ ProjectDeleteConflictReason, ProjectId, ProjectIdentifier, + ProjectUpdateRequest, ) from datamasque.client.exceptions import DataMasqueException from datamasque.client.models.connection import ConnectionId, S3ConnectionConfig @@ -49,6 +50,28 @@ "updated": "2026-10-01T10:00:00Z", } +FILES_EXIST_MESSAGE = 'Cannot rename the project, because files already exist at "seeds/retail".' + +RENAMED_PROJECT = {**BANKING_PROJECT, "reference": "7-retail", "name": "Retail"} + +UPDATE_PROJECT_CASES = [ + pytest.param(ProjectId(7), ProjectUpdateRequest(name="Retail"), {"name": "Retail"}, id="rename-by-id"), + pytest.param("7-banking", ProjectUpdateRequest(name="Retail"), {"name": "Retail"}, id="rename-by-reference"), + pytest.param( + Project.model_validate(BANKING_PROJECT), + ProjectUpdateRequest(description=""), + {"description": ""}, + id="project-object-clears-description", + ), + pytest.param( + ProjectId(7), + ProjectUpdateRequest(name="Banking", description="Customer banking data"), + {"name": "Banking", "description": "Customer banking data"}, + id="name-and-description", + ), +] + + CONFLICTS = [ { "reason": "name_in_use", @@ -432,3 +455,84 @@ def test_unwrap_project_id_reads_the_id_from_each_form(project: ProjectIdentifie def test_unwrap_project_id_rejects_a_string_that_does_not_start_with_an_id(project: str) -> None: with pytest.raises(ValueError, match=f'"{project}" is not a project ID or reference'): unwrap_project_id(project) + + +def test_update_request_rejects_unknown_fields() -> None: + with pytest.raises(ValidationError) as exc_info: + # mypy rejects the unknown field, which the test passes on purpose. + ProjectUpdateRequest(name="Banking", archived=True) # type: ignore[call-arg] + + assert [(error["loc"], error["msg"]) for error in exc_info.value.errors()] == [ + (("archived",), "Extra inputs are not permitted") + ] + + +@pytest.mark.parametrize(("project", "request_body", "expected_body"), UPDATE_PROJECT_CASES) +def test_update_project_sends_only_the_fields_set( + client: DataMasqueClient, + project: ProjectIdentifier, + request_body: ProjectUpdateRequest, + expected_body: dict[str, object], +) -> None: + with requests_mock.Mocker() as m: + m.patch("http://test-server/api/projects/7/", json=BANKING_PROJECT) + client.update_project(project, request_body) + + assert m.last_request.json() == expected_body + + +def test_update_project_returns_the_renamed_project(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.patch("http://test-server/api/projects/7/", json=RENAMED_PROJECT) + project = client.update_project(ProjectId(7), ProjectUpdateRequest(name="Retail")) + + assert project.name == "Retail" + assert project.reference == "7-retail" + + +def test_update_project_rejected_name_raises(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.patch( + "http://test-server/api/projects/7/", + json={"name": [FILES_EXIST_MESSAGE]}, + status_code=400, + ) + with pytest.raises(DataMasqueApiError) as exc_info: + client.update_project(ProjectId(7), ProjectUpdateRequest(name="Retail")) + + assert exc_info.value.response.status_code == 400 + assert exc_info.value.response.json() == {"name": [FILES_EXIST_MESSAGE]} + + +@pytest.mark.parametrize( + ("status_code", "detail"), + [ + (403, "You do not have permission to perform this action."), + (404, "Not found."), + ], +) +def test_update_project_refused_raises(client: DataMasqueClient, status_code: int, detail: str) -> None: + with requests_mock.Mocker() as m: + m.patch("http://test-server/api/projects/7/", json={"detail": detail}, status_code=status_code) + with pytest.raises(DataMasqueApiError) as exc_info: + client.update_project(ProjectId(7), ProjectUpdateRequest(name="Retail")) + + assert exc_info.value.response.status_code == status_code + + +@pytest.mark.parametrize( + ("body", "expected"), + [ + pytest.param({**BANKING_PROJECT, "has_git_synced_resources": True}, True, id="synced"), + pytest.param({**BANKING_PROJECT, "has_git_synced_resources": False}, False, id="not-synced"), + pytest.param(BANKING_PROJECT, None, id="flag-absent"), + ], +) +def test_get_project_reports_git_synced_resources( + client: DataMasqueClient, body: dict[str, object], expected: Optional[bool] +) -> None: + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/7/", json=body) + project = client.get_project(ProjectId(7)) + + assert project.has_git_synced_resources is expected From 1fe4c5f8e564e4d65a6c6788440a51850c0ce733 Mon Sep 17 00:00:00 2001 From: Andrew Chester Date: Thu, 29 Oct 2026 12:48:44 +1300 Subject: [PATCH 16/22] DM-4391 docs(projects): match the update_project and git flag docs to the server wording --- HISTORY.rst | 2 +- datamasque/client/models/project.py | 4 +++- datamasque/client/projects.py | 7 +++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 828fd65..2d1ec91 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -52,7 +52,7 @@ History and returns the updated ``Project``. Only admin users can update projects, and API tokens cannot. * ``get_project`` sets ``Project.has_git_synced_resources``, which says whether the project holds a git-synced resource. - A rename leaves a git-synced library or discovery config library at its old repository path. + After a rename, a git-synced resource's next push writes a new copy under the new name, and the old copy stays. 1.6.0 (unreleased) ------------------ diff --git a/datamasque/client/models/project.py b/datamasque/client/models/project.py index b295b45..cd450bd 100644 --- a/datamasque/client/models/project.py +++ b/datamasque/client/models/project.py @@ -42,8 +42,10 @@ class Project(BaseModel): Whether any library, discovery config library, ruleset or discovery config in the project that is not deleted is synced with git. + A resource edited since its last push is not counted. Only `get_project` sets it; other calls leave it `None`. - A rename leaves a git-synced library or discovery config library at its old repository path. + After a rename, a git-synced resource's next push writes a new copy under the new name and the old copy stays. + DataMasque does not move files in the repository. """ diff --git a/datamasque/client/projects.py b/datamasque/client/projects.py index ba40839..ef64906 100644 --- a/datamasque/client/projects.py +++ b/datamasque/client/projects.py @@ -72,10 +72,9 @@ def update_project(self, project: ProjectIdentifier, request: ProjectUpdateReque Renames a project or changes its description, and returns the updated project. Only admin users can, and API tokens cannot. - Renaming moves the project's seed and credential files to the folder of the new name, - and invalidates rulesets, discovery configs and generator configs that import a library by its old name. - A rename that only changes the case of the name moves no files, and a file outside the project's folder stays. - Renaming the Default project invalidates imports that name its libraries without a project name. + Renaming invalidates rulesets, discovery configs and generator configs that import a library of the project + by the old project name. Imports without a project name stay valid. + It also moves the project's seed and credential files, keeping connections working. Seed and table references in rulesets that name the old project are not rewritten. Raises: From 9ef92fb0cca35c3ac9b75b5368d89b29f6fb7fbe Mon Sep 17 00:00:00 2001 From: jiatolentino Date: Tue, 3 Nov 2026 10:58:37 +0800 Subject: [PATCH 17/22] docs(projects): a project name only has to differ from active projects' names --- datamasque/client/models/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datamasque/client/models/project.py b/datamasque/client/models/project.py index cd450bd..fc32cb2 100644 --- a/datamasque/client/models/project.py +++ b/datamasque/client/models/project.py @@ -80,7 +80,7 @@ class ProjectCreateRequest(BaseModel): Request body for `POST /api/projects/`. A name must not contain "/", ".." or a control character, - and must differ, ignoring case, from the name of every other project, including deleted ones. + and must differ, ignoring case, from the name of every other active project. The server trims leading and trailing whitespace from `name` and `description`. """ From fb2ab9fb61022936ae2548c9bcd3871b93d0dff8 Mon Sep 17 00:00:00 2001 From: Colin Haywood Date: Wed, 4 Nov 2026 15:52:51 +1300 Subject: [PATCH 18/22] docs: tidy the 1.7.0 changelog and drop the unreleased marker from 1.6.0 --- HISTORY.rst | 57 +++++++++++++++++------------------------------------ 1 file changed, 18 insertions(+), 39 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2d1ec91..7dea79c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,60 +2,39 @@ History ======= -1.7.0 (unreleased) +1.7.0 (2026-11-06) ------------------ +* Added project models and management APIs. +* Added resource management APIs. +* Added ``default_project`` to ``DataMasqueInstanceConfig``. * ``ConnectionConfig``, ``Ruleset``, ``RulesetLibrary``, ``DiscoveryConfig``, ``DiscoveryConfigLibrary``, ``RGConfig`` and ``TableReference`` send their ``project`` to the server when it is set. It takes a project's ``-`` reference, such as ``Project.reference``, or its ID as a string. - Left unset, a create goes into the server's default project. + Left unset, a create goes into the client's ``default_project``, else the server's default project. Each create sets the model's ``project`` to the project the server stored it in. +* **Breaking:** names are unique per project, so every by-name helper takes a required ``project``: + ``get_*_by_name``, ``delete_*_by_name_if_exists`` and ``get_file_of_type_by_name``. + These ``project`` parameters take a ``Project`` or its ``ProjectId``. + ``create_or_update_*`` updates an existing resource only when it has the same name in the same project. + If the model has no ``project``, it looks in the client's ``default_project``, else the server's only active project, + and raises ``DataMasqueException`` when there is no active project or more than one. * **Breaking:** ``upload_file`` and ``upload_file_if_not_exists`` take a required ``project``. * **Breaking:** ``namespace`` is removed from ``RulesetLibrary`` and ``DiscoveryConfigLibrary``, and the client drops it from server responses so that it is never sent back. The library by-name helpers take ``project`` in its place. -* **Breaking:** names are unique per project, so every by-name helper takes a required ``project``: - ``get_*_by_name``, ``delete_*_by_name_if_exists`` and ``get_file_of_type_by_name``. - ``create_or_update_*`` updates an existing resource only when it has the same name in the same project. - With the model's ``project`` unset, that is the server's only active project, - and ``DataMasqueException`` is raised when there is no active project or more than one. - The methods' ``project`` parameters take a ``Project`` or its ``ProjectId``. * **Breaking:** ``get_unfinished_runs`` returns its runs keyed by ``ConnectionId`` instead of connection name, because two connections in different projects can share a name. * ``list_connections``, ``list_table_references`` and ``list_files_of_type`` take an optional ``project`` filter. -* Added ``default_project`` to ``DataMasqueInstanceConfig``. - A create whose model has no ``project`` goes into ``default_project``, - and ``create_or_update_*`` looks for an existing resource there, - before falling back to the server's only active project. - Updating a connection leaves it in its project. * ``DataMasqueFile`` gains a read-only ``project`` field, which is ``None`` for built-in seeds. * ``get_generated_rulesets`` sets each ruleset's ``project`` to the connection's project. -* Added Git push, pull and file listing for rulesets, ruleset libraries, discovery configs - and discovery config libraries: ``list_*_git_files``, ``pull_*_from_git``, ``push_*_to_git`` - and ``get_*_git_push_path``, with the ``GitFile``, ``GitFileCommit`` and ``GitExtension`` types. - A push writes the resource's file in its project's folder, with the ``extension`` it is given (``.yml`` or ``.yaml``). - A resource pushed before the server's upgrade to projects keeps its existing file - until it is renamed or moved to another project. - ``get_*_git_push_path`` returns the path a push writes, relative to the repository and without its extension. -* Added project models and management APIs. -* Added resource management APIs. -* ``list_resources`` takes ``ordering`` and ``descending``, and each type's resources come back in that order. - Each listed ``Resource`` has a ``position``, its place in that order across every type. - Both require server version 3.26.21. -* Added ``export_bundle`` and ``import_bundle``, which export and import Version 2 bundles - that keep each ruleset, library and seed file in its project. - ``import_bundle`` also imports deprecated Version 1 bundles, - putting their items that name no project into ``fallback_project``, - else the server's earliest-created active project; ``default_project`` does not apply. - Both require server version 3.26.21. -* Added ``update_project``, which takes a ``ProjectUpdateRequest`` to rename a project or change its description, - and returns the updated ``Project``. - Only admin users can update projects, and API tokens cannot. -* ``get_project`` sets ``Project.has_git_synced_resources``, which says whether the project holds a git-synced resource. - After a rename, a git-synced resource's next push writes a new copy under the new name, and the old copy stays. - -1.6.0 (unreleased) ------------------- +* Added Git APIs for rulesets, ruleset libraries, discovery configs and discovery config libraries. +* Added bundle export and import APIs. + +Requires server version 3.26.21 + +1.6.0 +----- * Added a read-only ``project`` field to ``ConnectionConfig``, ``Ruleset``, ``RulesetLibrary``, ``DiscoveryConfig``, ``DiscoveryConfigLibrary``, ``RGConfig`` and ``TableReference``: From 529832f234fbc5a892b69e78eb0a5f711ec2a3ff Mon Sep 17 00:00:00 2001 From: Colin Haywood Date: Wed, 4 Nov 2026 15:52:51 +1300 Subject: [PATCH 19/22] chore: bump version to 1.7.0 --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b3183ec..7fced60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "datamasque-python" -version = "1.5.0" +version = "1.7.0" description = "Official Python client for the DataMasque data-masking API." authors = [ { name = "DataMasque Ltd" }, diff --git a/uv.lock b/uv.lock index 7164d65..c712118 100644 --- a/uv.lock +++ b/uv.lock @@ -419,7 +419,7 @@ toml = [ [[package]] name = "datamasque-python" -version = "1.5.0" +version = "1.7.0" source = { editable = "." } dependencies = [ { name = "pydantic" }, From ca39daaf6d06562d38d8318a9caeb6601a7d6741 Mon Sep 17 00:00:00 2001 From: Colin Haywood Date: Wed, 4 Nov 2026 16:17:52 +1300 Subject: [PATCH 20/22] docs: correct the git push, run project, project name and import fallback docstrings --- datamasque/client/git.py | 4 ++++ datamasque/client/models/bundle.py | 4 ++-- datamasque/client/models/project.py | 2 +- datamasque/client/models/runs.py | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/datamasque/client/git.py b/datamasque/client/git.py index af3f261..fc238d9 100644 --- a/datamasque/client/git.py +++ b/datamasque/client/git.py @@ -139,6 +139,7 @@ def push_ruleset_to_git( With no `project` set, the project is the saved ruleset's when it has an `id`, else the client's `default_project`, else the server's only active project. The ruleset's `id`, else its name, type and project, identify the saved ruleset the push is for. + With an `id`, the ruleset's name, type and project must match the saved ruleset's. The push sends the given ruleset's `yaml`, not the saved one, so save the ruleset first. The server records the pushed commit as the saved ruleset's git provenance when their YAML matches, but the given ruleset is not updated. @@ -208,6 +209,7 @@ def push_ruleset_library_to_git( With no `project` set, the project is the saved library's when it has an `id`, else the client's `default_project`, else the server's only active project. The library's `id`, else its name and project, identify the saved library the push is for. + With an `id`, the library's name and project must match the saved library's. The push sends the given library's `yaml`, not the saved one, so save the library first. The server records the pushed commit as the saved library's git provenance when their YAML matches, but the given library is not updated. @@ -275,6 +277,7 @@ def push_discovery_config_to_git( With no `project` set, the project is the saved config's when it has an `id`, else the client's `default_project`, else the server's only active project. The config's `id`, else its name, type and project, identify the saved config the push is for. + With an `id`, the config's name, type and project must match the saved config's. The push sends the given config's `yaml`, not the saved one, so save the config first. The server records the pushed commit on the saved config when their YAML matches, but `DiscoveryConfig` does not expose it. @@ -346,6 +349,7 @@ def push_discovery_config_library_to_git( With no `project` set, the project is the saved library's when it has an `id`, else the client's `default_project`, else the server's only active project. The library's `id`, else its name and project, identify the saved library the push is for. + With an `id`, the library's name and project must match the saved library's. The push sends the given library's `yaml`, not the saved one, so save the library first. The server records the pushed commit on the saved library when their YAML matches, but `DiscoveryConfigLibrary` does not expose it. diff --git a/datamasque/client/models/bundle.py b/datamasque/client/models/bundle.py index 2f26271..0504b5d 100644 --- a/datamasque/client/models/bundle.py +++ b/datamasque/client/models/bundle.py @@ -154,8 +154,8 @@ class BundleImportResult(BaseModel): """`None` when the bundle has no seed files.""" fallback_project: Optional[str] = None """ - The name of the project that took the items that named no project, such as a Version 1 bundle's rulesets. - `None` when every item went into the project it names. + The name of the project that took the resources that named no project, such as a Version 1 bundle's rulesets. + `None` when the import created, restored or updated no such resources. """ is_deprecated_bundle: bool = False """ diff --git a/datamasque/client/models/project.py b/datamasque/client/models/project.py index fc32cb2..bbe4eae 100644 --- a/datamasque/client/models/project.py +++ b/datamasque/client/models/project.py @@ -79,7 +79,7 @@ class ProjectCreateRequest(BaseModel): """ Request body for `POST /api/projects/`. - A name must not contain "/", ".." or a control character, + A name must not be ".", start with "DataMasque_", or contain "/", ".." or a control character, and must differ, ignoring case, from the name of every other active project. The server trims leading and trailing whitespace from `name` and `description`. """ diff --git a/datamasque/client/models/runs.py b/datamasque/client/models/runs.py index 09bae55..6c1bd3d 100644 --- a/datamasque/client/models/runs.py +++ b/datamasque/client/models/runs.py @@ -146,7 +146,8 @@ class RunInfo(BaseModel): """ Its project's `-` reference, e.g. `1-default`. - Taken from the ruleset's project when the run starts, and never changes after that. + The project of the run's ruleset, else its discovery config, else its source connection. + The run moves when that resource moves to another project. """ @model_validator(mode="before") From d122bf5800561c39061ec940e72295bb4821cfa1 Mon Sep 17 00:00:00 2001 From: Colin Haywood Date: Wed, 4 Nov 2026 16:28:38 +1300 Subject: [PATCH 21/22] fix: list every type as an empty group when listing resources with no projects --- datamasque/client/resources.py | 2 +- tests/test_resources.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datamasque/client/resources.py b/datamasque/client/resources.py index 2b0b5f0..380b61c 100644 --- a/datamasque/client/resources.py +++ b/datamasque/client/resources.py @@ -65,7 +65,7 @@ def list_resources( # `requests` leaves out a parameter whose value is an empty list, which would remove the filter. if projects is not None and not projects: - return {type_name: [] for type_name in types or []} + return {type_name: [] for type_name in types or list(ResourceTypeName)} if types is not None and not types: return {} diff --git a/tests/test_resources.py b/tests/test_resources.py index 67beae5..f755e6a 100644 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -140,7 +140,7 @@ def test_list_resources_keeps_each_resources_position(client: DataMasqueClient) @pytest.mark.parametrize( ("projects", "types", "expected"), [ - ([], None, {}), + ([], None, {type_name.value: [] for type_name in ResourceTypeName}), ([], [ResourceTypeName.rulesets], {"rulesets": []}), ([ProjectId(7)], [], {}), ], From efe721340643fc8dc7ee58347078cf1c6432cb7d Mon Sep 17 00:00:00 2001 From: Colin Haywood Date: Thu, 5 Nov 2026 09:45:34 +1300 Subject: [PATCH 22/22] docs: fix the Sphinx warnings that fail the docs build --- datamasque/client/models/project.py | 2 +- docs/conf.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/datamasque/client/models/project.py b/datamasque/client/models/project.py index bbe4eae..79c8c72 100644 --- a/datamasque/client/models/project.py +++ b/datamasque/client/models/project.py @@ -79,7 +79,7 @@ class ProjectCreateRequest(BaseModel): """ Request body for `POST /api/projects/`. - A name must not be ".", start with "DataMasque_", or contain "/", ".." or a control character, + A name must not be `.`, start with `DataMasque_`, or contain `/`, `..` or a control character, and must differ, ignoring case, from the name of every other active project. The server trims leading and trailing whitespace from `name` and `description`. """ diff --git a/docs/conf.py b/docs/conf.py index 3ef2bd2..db0aa2b 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -47,10 +47,11 @@ # Python identifiers that DO resolve still get cross-linked. suppress_warnings = ["ref.any"] -# Exclude pydantic's `model_config` attribute from autodoc output — -# it's plumbing, not public API, and clutters every model's docs page. +# Exclude pydantic's `model_config` attribute and the `drop_namespace` validator from autodoc output. +# Both are plumbing, not public API, and `model_config` would clutter every model's docs page. +# Documenting `drop_namespace` would also fail the -W build: its `type` annotation matches every `.type` attribute. autodoc_default_options = { - "exclude-members": "model_config", + "exclude-members": "model_config,drop_namespace", } # Add any paths that contain templates here, relative to this directory.