From d67575d81e1404516b410a5058e012e39d20de1e Mon Sep 17 00:00:00 2001 From: Carl Robben <907689+carlosfunk@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:21:56 +1300 Subject: [PATCH] feat: DM-4993 let create_or_update and run status work on a server older than projects - The VM UPGRADE shards could not seed a 3.26.17 or 3.26.19 server with this client - Every `create_or_update_*` failed: `_resolve_project` raised on the 404 from `/api/projects/` - `_resolve_project` now returns `None` on that 404, so the name lookup sends no `project` filter - `RunInfo.project` and `UnfinishedRun.project` are `None` when the server does not return `project` - A server with projects behaves as before --- HISTORY.rst | 2 ++ datamasque/client/base.py | 9 +++++-- .../client/discovery_config_libraries.py | 2 +- datamasque/client/discovery_configs.py | 2 +- datamasque/client/models/runs.py | 7 ++--- datamasque/client/rg_configs.py | 2 +- datamasque/client/ruleset_libraries.py | 4 ++- datamasque/client/table_references.py | 4 ++- tests/test_default_project.py | 27 +++++++++++++++---- tests/test_runs.py | 16 +++++++++++ 10 files changed, 60 insertions(+), 15 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7dea79c..c32a9b4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -28,6 +28,8 @@ History * ``list_connections``, ``list_table_references`` and ``list_files_of_type`` take an optional ``project`` filter. * ``DataMasqueFile`` gains a read-only ``project`` field, which is ``None`` for built-in seeds. * ``get_generated_rulesets`` sets each ruleset's ``project`` to the connection's project. +* ``create_or_update_*`` works with a server older than projects, and ``RunInfo.project`` and ``UnfinishedRun.project`` + are ``None`` from such a server. * Added Git APIs for rulesets, ruleset libraries, discovery configs and discovery config libraries. * Added bundle export and import APIs. diff --git a/datamasque/client/base.py b/datamasque/client/base.py index d45ebb7..3310e22 100644 --- a/datamasque/client/base.py +++ b/datamasque/client/base.py @@ -318,13 +318,14 @@ def _delete_if_exists(self, path: str, *, params: Optional[dict] = None) -> None self._raise_for_status(response) - def _resolve_project(self, project: Optional[Union[ProjectId, str]]) -> Union[ProjectId, str]: + def _resolve_project(self, project: Optional[Union[ProjectId, str]]) -> Optional[Union[ProjectId, str]]: """ Returns `project`, else the client's `default_project`, else the ID of the only active project. The last is the project the server puts a create into when it is sent no `project`. With no active project, or more than one, the server rejects such a create, so this raises `DataMasqueException` instead. + Returns `None` for a server older than projects, which has one set of names and no project filter. """ if project is not None: @@ -332,7 +333,11 @@ def _resolve_project(self, project: Optional[Union[ProjectId, str]]) -> Union[Pr if self.default_project is not None: return self.default_project - response = self.make_request("GET", "/api/projects/", params={"limit": 1}) + response = self.make_request("GET", "/api/projects/", params={"limit": 1}, require_status_check=False) + if response.status_code == 404: + return None + + self._raise_for_status(response) page = Page[Project].model_validate(response.json()) if page.count != 1: raise DataMasqueException( diff --git a/datamasque/client/discovery_config_libraries.py b/datamasque/client/discovery_config_libraries.py index c0e978b..a534dca 100644 --- a/datamasque/client/discovery_config_libraries.py +++ b/datamasque/client/discovery_config_libraries.py @@ -32,7 +32,7 @@ def get_discovery_config_library(self, library_id: DiscoveryConfigLibraryId) -> return DiscoveryConfigLibrary.model_validate(response.json()) def _get_discovery_config_library_id_by_name( - self, name: str, project: Union[ProjectId, str] + self, name: str, project: Optional[Union[ProjectId, str]] ) -> Optional[DiscoveryConfigLibraryId]: """Returns the ID of the library with the given name in the given project, or `None` if there is none.""" diff --git a/datamasque/client/discovery_configs.py b/datamasque/client/discovery_configs.py index 69f5cac..461bdf4 100644 --- a/datamasque/client/discovery_configs.py +++ b/datamasque/client/discovery_configs.py @@ -35,7 +35,7 @@ def get_discovery_config(self, config_id: DiscoveryConfigId) -> DiscoveryConfig: return DiscoveryConfig.model_validate(response.json()) def _get_discovery_config_id_by_name( - self, name: str, config_type: DiscoveryConfigType, project: Union[ProjectId, str] + self, name: str, config_type: DiscoveryConfigType, project: Optional[Union[ProjectId, str]] ) -> Optional[DiscoveryConfigId]: """Return the id of the config matching name, type and project via a single list request, or `None`.""" diff --git a/datamasque/client/models/runs.py b/datamasque/client/models/runs.py index 6c1bd3d..9cb7c5f 100644 --- a/datamasque/client/models/runs.py +++ b/datamasque/client/models/runs.py @@ -142,12 +142,13 @@ class RunInfo(BaseModel): start_time: Optional[datetime] = None end_time: Optional[datetime] = None options: Optional[dict[str, Any]] = None - project: str + project: Optional[str] = None """ Its project's `-` reference, e.g. `1-default`. The project of the run's ruleset, else its discovery config, else its source connection. The run moves when that resource moves to another project. + `None` from a server older than projects. """ @model_validator(mode="before") @@ -166,8 +167,8 @@ class UnfinishedRun(BaseModel): ruleset_name: str status: MaskingRunStatus destination_connection: Optional[RunConnectionRef] = None - project: str - """Its project's `-` reference, e.g. `1-default`.""" + project: Optional[str] = None + """Its project's `-` reference, e.g. `1-default`. `None` from a server older than projects.""" @model_validator(mode="before") @classmethod diff --git a/datamasque/client/rg_configs.py b/datamasque/client/rg_configs.py index cf5237d..67437ff 100644 --- a/datamasque/client/rg_configs.py +++ b/datamasque/client/rg_configs.py @@ -34,7 +34,7 @@ def get_rg_config(self, config_id: RGConfigId) -> RGConfig: response = self.make_request("GET", f"/api/ruleset-generation-configs/{config_id}/") return RGConfig.model_validate(response.json()) - def _get_rg_config_id_by_name(self, name: str, project: Union[ProjectId, str]) -> Optional[RGConfigId]: + def _get_rg_config_id_by_name(self, name: str, project: Optional[Union[ProjectId, str]]) -> Optional[RGConfigId]: """Return the id of the config matching the name and project via a single list request, or `None`.""" response = self.make_request( diff --git a/datamasque/client/ruleset_libraries.py b/datamasque/client/ruleset_libraries.py index 20003c3..2742b0e 100644 --- a/datamasque/client/ruleset_libraries.py +++ b/datamasque/client/ruleset_libraries.py @@ -35,7 +35,9 @@ def get_ruleset_library(self, library_id: RulesetLibraryId) -> RulesetLibrary: response = self.make_request("GET", f"/api/ruleset-libraries/{library_id}/") return RulesetLibrary.model_validate(response.json()) - def _get_ruleset_library_id_by_name(self, name: str, project: Union[ProjectId, str]) -> Optional[RulesetLibraryId]: + def _get_ruleset_library_id_by_name( + self, name: str, project: Optional[Union[ProjectId, str]] + ) -> Optional[RulesetLibraryId]: """Return the ID of the library with the given name in the given project, or `None` if there is none.""" response = self.make_request( diff --git a/datamasque/client/table_references.py b/datamasque/client/table_references.py index 04ec955..9766bba 100644 --- a/datamasque/client/table_references.py +++ b/datamasque/client/table_references.py @@ -50,7 +50,9 @@ def get_table_reference_by_name(self, name: str, project: ProjectIdentifier) -> return matches[0] - def _get_table_reference_id_by_name(self, name: str, project: Union[ProjectId, str]) -> Optional[TableReferenceId]: + def _get_table_reference_id_by_name( + self, name: str, project: Optional[Union[ProjectId, str]] + ) -> Optional[TableReferenceId]: """Return the ID of the table reference with the given name in the given project, or `None` if there is none.""" response = self.make_request("GET", "/api/table-references/", params={"project": project}) diff --git a/tests/test_default_project.py b/tests/test_default_project.py index 899ad5b..d3a0e0b 100644 --- a/tests/test_default_project.py +++ b/tests/test_default_project.py @@ -18,6 +18,7 @@ from tests.helpers import active_projects_page, database_connection_config BASE_URL = "http://test-server" +EMPTY_PAGE = {"count": 0, "next": None, "previous": None, "results": []} def make_client(default_project: Optional[str] = None) -> DataMasqueClient: @@ -41,36 +42,41 @@ def create_connection(client: DataMasqueClient, project: Optional[str], m: reque def create_discovery_config(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.get(f"{BASE_URL}/api/discovery/configs/", json=EMPTY_PAGE) m.post( f"{BASE_URL}/api/discovery/configs/", json={"id": "dc-1", "name": "dc", "config_type": "database", "project": "2-acme"}, ) config = DiscoveryConfig(name="dc", yaml="labels: []\n", config_type="database", project=project) - return client.create_discovery_config(config) + return client.create_or_update_discovery_config(config) def create_rg_config(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.get(f"{BASE_URL}/api/ruleset-generation-configs/", json=EMPTY_PAGE) m.post(f"{BASE_URL}/api/ruleset-generation-configs/", json={"id": "rg-1", "name": "rg", "project": "2-acme"}) - return client.create_rg_config(RGConfig(name="rg", yaml="x: 1\n", project=project)) + return client.create_or_update_rg_config(RGConfig(name="rg", yaml="x: 1\n", project=project)) def create_table_reference(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.get(f"{BASE_URL}/api/table-references/", json=[]) m.post( f"{BASE_URL}/api/table-references/", json={"id": "tr-1", "name": "tr", "connection": "c-1", "source": "people.csv", "project": "2-acme"}, ) table_reference = TableReference(name="tr", connection=ConnectionId("c-1"), source="people.csv", project=project) - return client.create_table_reference(table_reference) + return client.create_or_update_table_reference(table_reference) def create_ruleset_library(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.get(f"{BASE_URL}/api/ruleset-libraries/", json=EMPTY_PAGE) m.post(f"{BASE_URL}/api/ruleset-libraries/", json={"id": "rl-1", "name": "rl", "project": "2-acme"}) - return client.create_ruleset_library(RulesetLibrary(name="rl", yaml="version: '1.0'\n", project=project)) + return client.create_or_update_ruleset_library(RulesetLibrary(name="rl", yaml="version: '1.0'\n", project=project)) def create_discovery_config_library(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.get(f"{BASE_URL}/api/discovery/config-libraries/", json=[]) m.post(f"{BASE_URL}/api/discovery/config-libraries/", json={"id": "dl-1", "name": "dl", "project": "2-acme"}) - return client.create_discovery_config_library( + return client.create_or_update_discovery_config_library( DiscoveryConfigLibrary(name="dl", yaml="labels: []\n", project=project) ) @@ -114,6 +120,17 @@ def test_a_create_with_no_default_project_sends_no_project(create: Create) -> No assert "project" not in m.last_request.json() +@pytest.mark.parametrize("create", CREATES) +def test_a_create_on_a_server_without_projects_sends_no_project(create: Create) -> None: + """A server older than projects returns 404 for `/api/projects/`, and a create must still work there.""" + with requests_mock.Mocker() as m: + m.get(f"{BASE_URL}/api/projects/", status_code=404) + create(make_client(), None, m) + + assert "project" not in m.last_request.json() + assert all("project" not in request.qs for request in m.request_history) + + def test_a_connection_update_sends_no_project() -> None: """An update leaves the connection in its own project, whatever the default.""" client = make_client(default_project="1-default") diff --git a/tests/test_runs.py b/tests/test_runs.py index 85f1aff..681db58 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -365,6 +365,22 @@ def test_get_run_info(client): assert result.name == "r1" +def test_run_info_from_a_server_without_projects_has_no_project(client): + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/runs/1/", + json={ + "id": 1, + "status": "finished", + "mask_type": "database", + "source_connection_name": "conn1", + "ruleset_name": "rs1", + }, + ) + + assert client.get_run_info(1).project is None + + def test_masking_run_request_model_dump_minimal(): """A minimal request dumps with an empty `options` object (the server rejects missing `options`).""" req = MaskingRunRequest(connection="conn-1", ruleset="rs-1")