diff --git a/HISTORY.rst b/HISTORY.rst index 191e613..7dea79c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,8 +2,39 @@ History ======= -1.6.0 (unreleased) ------------------- +1.7.0 (2026-11-06) +------------------ + +* Added project models and management APIs. +* Added resource management APIs. +* Added ``default_project`` to ``DataMasqueInstanceConfig``. +* ``ConnectionConfig``, ``Ruleset``, ``RulesetLibrary``, ``DiscoveryConfig``, ``DiscoveryConfigLibrary``, ``RGConfig`` + and ``TableReference`` send their ``project`` to the server when it is set. + It takes a project's ``-`` reference, such as ``Project.reference``, or its ID as a string. + Left unset, a create goes into the client's ``default_project``, else the server's default project. + Each create sets the model's ``project`` to the project the server stored it in. +* **Breaking:** names are unique per project, so every by-name helper takes a required ``project``: + ``get_*_by_name``, ``delete_*_by_name_if_exists`` and ``get_file_of_type_by_name``. + These ``project`` parameters take a ``Project`` or its ``ProjectId``. + ``create_or_update_*`` updates an existing resource only when it has the same name in the same project. + If the model has no ``project``, it looks in the client's ``default_project``, else the server's only active project, + and raises ``DataMasqueException`` when there is no active project or more than one. +* **Breaking:** ``upload_file`` and ``upload_file_if_not_exists`` take a required ``project``. +* **Breaking:** ``namespace`` is removed from ``RulesetLibrary`` and ``DiscoveryConfigLibrary``, + and the client drops it from server responses so that it is never sent back. + The library by-name helpers take ``project`` in its place. +* **Breaking:** ``get_unfinished_runs`` returns its runs keyed by ``ConnectionId`` instead of connection name, + because two connections in different projects can share a name. +* ``list_connections``, ``list_table_references`` and ``list_files_of_type`` take an optional ``project`` filter. +* ``DataMasqueFile`` gains a read-only ``project`` field, which is ``None`` for built-in seeds. +* ``get_generated_rulesets`` sets each ruleset's ``project`` to the connection's project. +* Added Git APIs for rulesets, ruleset libraries, discovery configs and discovery config libraries. +* Added bundle export and import APIs. + +Requires server version 3.26.21 + +1.6.0 +----- * Added a read-only ``project`` field to ``ConnectionConfig``, ``Ruleset``, ``RulesetLibrary``, ``DiscoveryConfig``, ``DiscoveryConfigLibrary``, ``RGConfig`` and ``TableReference``: diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index 2122c6d..5399310 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -23,10 +23,21 @@ InvalidDiscoveryConfigError, InvalidLibraryError, InvalidRulesetError, + ProjectDeleteConflictError, RGConfigNotFoundError, RunNotCancellableError, ) +from datamasque.client.git import GitExtension from datamasque.client.ifm import DataMasqueIfmClient +from datamasque.client.models.bundle import ( + BundleImportCounts, + BundleImportItemResults, + BundleImportResult, + BundleImportStatus, + LibraryImportResult, + RulesetImportResult, + SeedFileImportResult, +) from datamasque.client.models.connection import ( AzureConnectionConfig, CassandraConnectionConfig, @@ -100,7 +111,7 @@ SnowflakeKeyFile, SslZipFile, ) -from datamasque.client.models.git import GitSnapshot +from datamasque.client.models.git import GitFile, GitFileCommit, GitSnapshot from datamasque.client.models.ifm import ( DataMasqueIfmInstanceConfig, IfmLog, @@ -115,6 +126,27 @@ RulesetPlanUpdateRequest, ) from datamasque.client.models.license import LicenseInfo, SwitchableLicenseMetadata +from datamasque.client.models.project import ( + Project, + ProjectCreateRequest, + ProjectDeleteConflict, + ProjectDeleteConflictReason, + ProjectId, + ProjectIdentifier, + ProjectUpdateRequest, +) +from datamasque.client.models.resource import ( + ConnectionResource, + DiscoveryConfigResource, + Resource, + ResourceMoveConflict, + ResourceMoveConflictReason, + ResourceMoveResult, + ResourceOrdering, + ResourcesByType, + RulesetResource, +) +from datamasque.client.models.resource_type import ResourceTypeName from datamasque.client.models.rg_config import RGConfig, RGConfigId from datamasque.client.models.ruleset import Ruleset, RulesetId, RulesetType from datamasque.client.models.ruleset_library import RulesetLibrary, RulesetLibraryId @@ -182,12 +214,17 @@ "AzureConnectionConfig", "BooleanPreview", "BooleanStatistics", + "BundleImportCounts", + "BundleImportItemResults", + "BundleImportResult", + "BundleImportStatus", "CassandraConnectionConfig", "ColumnKind", "ColumnPreview", "CommonStatistics", "ConnectionConfig", "ConnectionId", + "ConnectionResource", "ConstraintColumns", "DataMasqueApiError", "DataMasqueClient", @@ -208,6 +245,7 @@ "DiscoveryConfigLibrary", "DiscoveryConfigLibraryId", "DiscoveryConfigNotFoundError", + "DiscoveryConfigResource", "DiscoveryConfigType", "DiscoveryMatch", "CosmosDbConnectionConfig", @@ -231,6 +269,9 @@ "FirstCharsStatistics", "ForeignKeyRef", "GcsConnectionConfig", + "GitExtension", + "GitFile", + "GitFileCommit", "GitSnapshot", "HashColumnsTableConfig", "IfmAuthError", @@ -248,6 +289,7 @@ "LengthEntry", "LengthUnit", "LengthsStatistics", + "LibraryImportResult", "LicenseInfo", "LicenseLock", "Locator", @@ -269,14 +311,30 @@ "PatternComposition", "PatternEntry", "PatternsStatistics", + "Project", + "ProjectCreateRequest", + "ProjectDeleteConflict", + "ProjectDeleteConflictError", + "ProjectDeleteConflictReason", + "ProjectId", + "ProjectIdentifier", + "ProjectUpdateRequest", "RGConfig", "RGConfigId", "RGConfigNotFoundError", "ReferencingForeignKey", + "Resource", + "ResourceMoveConflict", + "ResourceMoveConflictReason", + "ResourceMoveResult", + "ResourceOrdering", + "ResourceTypeName", + "ResourcesByType", "Ruleset", "RulesetGenerationRequest", "RulesetGenerationWithRGConfigRequest", "RulesetId", + "RulesetImportResult", "RulesetLibrary", "RulesetLibraryId", "RulesetPlan", @@ -284,6 +342,7 @@ "RulesetPlanOptions", "RulesetPlanPartialUpdateRequest", "RulesetPlanUpdateRequest", + "RulesetResource", "RulesetType", "RunConnectionRef", "RunId", @@ -300,6 +359,7 @@ "SchemaDiscoveryRequest", "SchemaDiscoveryResult", "SeedFile", + "SeedFileImportResult", "SelectedColumns", "SelectedData", "SelectedFileData", diff --git a/datamasque/client/base.py b/datamasque/client/base.py index 673356d..d45ebb7 100644 --- a/datamasque/client/base.py +++ b/datamasque/client/base.py @@ -17,10 +17,13 @@ from datamasque.client.exceptions import ( DataMasqueApiError, + DataMasqueException, DataMasqueNotReadyError, DataMasqueTransportError, ) from datamasque.client.models.dm_instance import DataMasqueInstanceConfig +from datamasque.client.models.pagination import Page +from datamasque.client.models.project import Project, ProjectId from datamasque.client.spcs import install_spcs_gateway_auth logger = logging.getLogger(__name__) @@ -130,6 +133,7 @@ class BaseClient: password: Optional[str] verify_ssl: bool token_source: Optional[Callable[[], str]] + default_project: Optional[str] def __init__(self, connection_config: DataMasqueInstanceConfig) -> None: self.base_url = connection_config.base_url @@ -137,6 +141,7 @@ def __init__(self, connection_config: DataMasqueInstanceConfig) -> None: self.password = connection_config.password self.verify_ssl = connection_config.verify_ssl self.token_source = connection_config.token_source + self.default_project = connection_config.default_project self._session = _build_session(self.verify_ssl) self._timeout = (connection_config.connect_timeout_seconds, connection_config.read_timeout_seconds) if connection_config.spcs_pat: @@ -313,6 +318,40 @@ def _delete_if_exists(self, path: str, *, params: Optional[dict] = None) -> None self._raise_for_status(response) + def _resolve_project(self, project: Optional[Union[ProjectId, str]]) -> Union[ProjectId, str]: + """ + Returns `project`, else the client's `default_project`, else the ID of the only active project. + + The last is the project the server puts a create into when it is sent no `project`. + With no active project, or more than one, the server rejects such a create, + so this raises `DataMasqueException` instead. + """ + + if project is not None: + return project + if self.default_project is not None: + return self.default_project + + response = self.make_request("GET", "/api/projects/", params={"limit": 1}) + page = Page[Project].model_validate(response.json()) + if page.count != 1: + raise DataMasqueException( + f"No `project` is set and the server has {page.count} active projects. Set `project` to choose one." + ) + + return page.results[0].id + + def _default_project_field(self, project: Optional[Union[ProjectId, str]]) -> dict[str, str]: + """ + Returns the `project` field a create sends when the resource names no `project`: the client's `default_project`. + + Empty when the resource names one, which its own fields carry, or when there is no `default_project`. + """ + + if project is None and self.default_project is not None: + return {"project": self.default_project} + return {} + def _iter_paginated( self, path: str, @@ -321,8 +360,14 @@ def _iter_paginated( params: Optional[dict] = None, page_size: int = 100, ) -> Iterator[_T]: + """Iterate every `T` across all pages of an admin-server list endpoint.""" + + for results in self._iter_pages(path, params=params, page_size=page_size): + yield from (model.model_validate(item) for item in results) + + def _iter_pages(self, path: str, *, params: Optional[dict] = None, page_size: int = 100) -> Iterator[Any]: """ - Iterate every `T` across all pages of an admin-server list endpoint. + Iterate the `results` of each page of an admin-server list endpoint. Opts into pagination by sending `limit`/`offset` on the first request, then follows the absolute `next` URL returned by the server. @@ -338,7 +383,7 @@ def _iter_paginated( while url: response = self.make_request("GET", url, params=current_params) data = response.json() - yield from (model.model_validate(item) for item in data["results"]) + yield data["results"] url = data.get("next") # The `next` URL is absolute and already contains the pagination cursor; # do not re-send our initial params alongside it. diff --git a/datamasque/client/bundles.py b/datamasque/client/bundles.py new file mode 100644 index 0000000..889629a --- /dev/null +++ b/datamasque/client/bundles.py @@ -0,0 +1,134 @@ +import logging +from collections.abc import Sequence +from typing import Optional, Union + +from datamasque.client.base import BaseClient, FileOrContent, UploadFile, read_file_or_content +from datamasque.client.models.bundle import BundleImportResult +from datamasque.client.models.project import ProjectIdentifier, unwrap_project_id +from datamasque.client.models.ruleset import Ruleset, RulesetId, unwrap_ruleset_id + +logger = logging.getLogger(__name__) + + +class BundleClient(BaseClient): + """ + Export and import bundle API methods. Mixed into `DataMasqueClient`. + + The server accepts these only with the login token that `POST /api/auth/token/login/` returns. + They fail with a 403 for a client whose `token_source` returns a user's long-lived API token, + the `api_token` that `GET /api/users/me/` gives. + """ + + def export_bundle( + self, + rulesets: Optional[Sequence[Union[Ruleset, RulesetId]]] = None, + *, + include_seed_files: bool = False, + ) -> bytes: + """ + Returns a Version 2 export bundle, a Zip archive that files every item under its project's name. + + The bundle holds `rulesets`, given as `Ruleset`s or their IDs, and the libraries they use; + or, when `rulesets` is not given, every ruleset and library. + With `include_seed_files`, it also holds the custom seed files those rulesets and libraries refer to. + Pass the bundle to `import_bundle`, on this server or another, + to put each item in the project of the same name. + For example, `Path("export.zip").write_bytes(client.export_bundle())` saves it to a file. + + Raises: + TypeError: `rulesets` is a single `Ruleset` or ID, not a sequence of them. + ValueError: `rulesets` is empty, or a `Ruleset` in it has not been created on the server. + DataMasqueApiError: the server rejects the request, + for example with a 403 when the client authenticates with a long-lived API token. + """ + + if isinstance(rulesets, (str, Ruleset)): + raise TypeError("Pass `rulesets` as a sequence, such as `[ruleset]`") + if rulesets is not None and not rulesets: + raise ValueError("Pass at least one ruleset, or omit `rulesets` to export every ruleset") + + data: dict[str, object] = {"include_seed_files": include_seed_files} + if rulesets is not None: + data["ruleset_ids"] = [unwrap_ruleset_id(ruleset) for ruleset in rulesets] + + response = self.make_request("POST", "/api/export/v2/", data=data) + return response.content + + def import_bundle( + self, + bundle: FileOrContent, + *, + overwrite_rulesets: Optional[bool] = None, + overwrite_libraries: bool = False, + overwrite_seed_files: bool = False, + fallback_project: Optional[ProjectIdentifier] = None, + ) -> BundleImportResult: + """ + Imports an export bundle, in the Version 2 or deprecated Version 1 format. + + Returns what the import did with each item. + + `bundle` is a path to the Zip archive (as `str` or `Path`), its `bytes`, + or a binary file-like object, such as a file opened with `"rb"`. + Each item goes into the project the bundle files it under, + which is created when it does not exist, if the caller is an administrator. + Items that name no project, such as a Version 1 bundle's rulesets and seed files, + go into `fallback_project`, given as a `Project`, its `ProjectId` or its `-` form, + or else the server's earliest-created active project; + the result's `fallback_project` names the project they went into. + The client's `default_project` does not apply. + + With `overwrite_rulesets` unset, a ruleset whose ID exists is left unchanged, + and a ruleset whose name is in use under another ID is imported under a new name; + `True` overwrites the existing ruleset and `False` leaves it unchanged. + Whatever the options, a ruleset or library whose ID is not on the server is left unchanged + when its name is in use by one with identical content. + `BundleImportStatus` lists every outcome. + `overwrite_libraries` and `overwrite_seed_files` overwrite existing libraries and seed files when `True`, + and leave them unchanged when `False`. + + The import can succeed for some items and fail for others: + check each type's `metadata.error`, or each item's `status`. + + Raises: + ValueError: `fallback_project` is a string that is not a project reference in the form `-`. + DataMasqueApiError: the server rejects the import, and imports nothing. + For example, a 400 when the bundle is not a valid export, `fallback_project` is not an active project, + or the bundle names a project that cannot be created with that name; + or a 403 when the bundle names a project that does not exist and the caller is not an administrator, + or when the client authenticates with a long-lived API token. + """ + + file_name, content = read_file_or_content(bundle, "bundle.zip") + content.seek(0) + + data: dict[str, object] = { + "overwrite_libraries": overwrite_libraries, + "overwrite_seed_files": overwrite_seed_files, + } + if overwrite_rulesets is not None: + data["overwrite_rulesets"] = overwrite_rulesets + if fallback_project is not None: + data["fallback_project"] = str(unwrap_project_id(fallback_project)) + + response = self.make_request( + "POST", + "/api/import/v2/", + data=data, + files=[ + UploadFile( + field_name="zip_archive", + filename=file_name, + content=content, + content_type="application/zip", + ), + ], + ) + result = BundleImportResult.model_validate(response.json()) + result.is_deprecated_bundle = response.headers.get("Deprecation", "").lower() == "true" + if result.is_deprecated_bundle: + logger.warning("Imported a deprecated Version 1 bundle; export a Version 2 bundle to keep each project") + if result.fallback_project is not None: + logger.info('Items that named no project went into project "%s"', result.fallback_project) + + return result diff --git a/datamasque/client/connections.py b/datamasque/client/connections.py index e41e46b..43e7652 100644 --- a/datamasque/client/connections.py +++ b/datamasque/client/connections.py @@ -1,8 +1,10 @@ import logging +from typing import Optional, Union from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueException from datamasque.client.models.connection import ConnectionConfig, ConnectionId, validate_connection +from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id logger = logging.getLogger(__name__) @@ -10,25 +12,39 @@ class ConnectionClient(BaseClient): """Connection-related API methods. Mixed into `DataMasqueClient`.""" - def list_connections(self) -> list[ConnectionConfig]: + def list_connections(self, project: Optional[ProjectIdentifier] = None) -> list[ConnectionConfig]: """ - Lists all configured connections. + Lists all configured connections, or only those in `project` when it is given. + `project` is a `Project`, its `ProjectId` or its `-` form. Note that database passwords and connection strings are returned encrypted over the API and so are `None` on the returned `ConnectionConfig` objects. """ - response = self.make_request("GET", "/api/connections/") + return self._list_connections(unwrap_project_id(project) if project is not None else None) + + def _list_connections(self, project: Optional[Union[ProjectId, str]]) -> list[ConnectionConfig]: + """Lists the connections in `project`, given as an ID or `-` reference, or in every project.""" + + params = {"project": project} if project is not None else None + response = self.make_request("GET", "/api/connections/", params=params) return [validate_connection(payload) for payload in response.json()] def create_or_update_connection(self, connection_config: ConnectionConfig) -> ConnectionConfig: - """Creates or updates the connection in DM, and sets the `id` field on the given `connection_config`.""" + """ + Creates or updates the connection in DM, and sets the `id` field on the given `connection_config`. + + An existing connection with the same name in the same project is updated. + With no `project` set, the client's `default_project` is used, else the server's only active project, + and `DataMasqueException` is raised when it has no active project or more than one. + """ connection_id = connection_config.id - all_connections = self.list_connections() connections_matching_name = [ - connection for connection in all_connections if connection.name == connection_config.name + connection + for connection in self._list_connections(self._resolve_project(connection_config.project)) + if connection.name == connection_config.name ] if connections_matching_name: connection_id = connections_matching_name[0].id @@ -37,6 +53,7 @@ def create_or_update_connection(self, connection_config: ConnectionConfig) -> Co "version": "1.0", } | connection_config.model_dump(exclude_none=True, by_alias=True, mode="json") if connection_id is None: + data |= self._default_project_field(connection_config.project) response = self.make_request("POST", "/api/connections/", data=data) else: response = self.make_request("PUT", f"/api/connections/{connection_id}/", data=data) @@ -45,6 +62,7 @@ def create_or_update_connection(self, connection_config: ConnectionConfig) -> Co server_connection_id = ConnectionId(connection_data["id"]) logger.debug("%s creation successful", type(connection_config).__name__) connection_config.id = server_connection_id + connection_config.project = connection_data.get("project") return connection_config def delete_connection_by_id_if_exists(self, connection_id: ConnectionId) -> None: @@ -52,11 +70,17 @@ def delete_connection_by_id_if_exists(self, connection_id: ConnectionId) -> None self._delete_if_exists(f"/api/connections/{connection_id}/") - def delete_connection_by_name_if_exists(self, connection_name: str) -> None: - """Deletes the connection with the given name. No-op if the connection does not exist.""" + def delete_connection_by_name_if_exists(self, connection_name: str, project: ProjectIdentifier) -> None: + """ + Deletes the connection with the given name in the given project. + + `project` is a `Project`, its `ProjectId` or its `-` form. + No-op if the connection does not exist. + """ - all_connections = self.list_connections() - connections_matching_name = [connection for connection in all_connections if connection.name == connection_name] + connections_matching_name = [ + connection for connection in self.list_connections(project) if connection.name == connection_name + ] for connection in connections_matching_name: if connection.id is None: raise DataMasqueException(f'Server returned a connection named "{connection.name}" without an `id`.') diff --git a/datamasque/client/discovery.py b/datamasque/client/discovery.py index 2faaaad..9145298 100644 --- a/datamasque/client/discovery.py +++ b/datamasque/client/discovery.py @@ -15,7 +15,7 @@ InvalidDiscoveryConfigError, RGConfigNotFoundError, ) -from datamasque.client.models.connection import ConnectionId +from datamasque.client.models.connection import ConnectionId, validate_connection from datamasque.client.models.data_selection import ( SelectedColumns, SelectedData, @@ -247,6 +247,7 @@ def get_generated_rulesets(self, connection_id: ConnectionId) -> list[Ruleset]: and `DataMasqueException` if it failed or was cancelled. Note that the ruleset(s) have autogenerated names, which you may want to customize before uploading. + Their `project` is the connection's project. """ status = self.get_async_ruleset_generation_task_status(connection_id) @@ -273,6 +274,8 @@ def get_generated_rulesets(self, connection_id: ConnectionId) -> list[Ruleset]: # (which carries the generated ruleset inline as `generated_ruleset`). # `requests` follows the 303 transparently, so we distinguish by the presence of # a `Content-Disposition: attachment` header, which Django's `FileResponse` sets on the ZIP response. + project = validate_connection(self.make_request("GET", f"/api/connections/{connection_id}/").json()).project + response = self.make_request( method="GET", path=f"/api/async-generate-ruleset/{connection_id}/download-rulesets/", @@ -285,7 +288,9 @@ def get_generated_rulesets(self, connection_id: ConnectionId) -> list[Ruleset]: if file_info.filename.endswith((".yml", ".yaml")): with zip_file.open(file_info) as file: yaml_content = file.read().decode("utf-8") - rulesets.append(Ruleset(name=Path(file_info.filename).stem, yaml=yaml_content)) + rulesets.append( + Ruleset(name=Path(file_info.filename).stem, project=project, yaml=yaml_content) + ) if not rulesets: raise DataMasqueException( @@ -302,7 +307,7 @@ def get_generated_rulesets(self, connection_id: ConnectionId) -> list[Ruleset]: f"but no ruleset was returned on the task-status record." ) - return [Ruleset(name="generated_ruleset", yaml=generated)] + return [Ruleset(name="generated_ruleset", project=project, yaml=generated)] def start_schema_discovery_run(self, discovery_config: SchemaDiscoveryRequest) -> RunId: """ diff --git a/datamasque/client/discovery_config_libraries.py b/datamasque/client/discovery_config_libraries.py index bd1e1fb..c0e978b 100644 --- a/datamasque/client/discovery_config_libraries.py +++ b/datamasque/client/discovery_config_libraries.py @@ -1,9 +1,10 @@ import logging -from typing import Optional +from typing import Optional, Union from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary, DiscoveryConfigLibraryId +from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id logger = logging.getLogger(__name__) @@ -30,22 +31,18 @@ def get_discovery_config_library(self, library_id: DiscoveryConfigLibraryId) -> response = self.make_request("GET", f"/api/discovery/config-libraries/{library_id}/") return DiscoveryConfigLibrary.model_validate(response.json()) - def _get_discovery_config_library_id_by_name(self, name: str, namespace: str) -> Optional[DiscoveryConfigLibraryId]: - """ - Returns the ID of the library with the given name and namespace, or `None` if there is none. - - The listing is filtered by name to keep the response small; - the namespace is matched client-side, - since the API's namespace filter cannot select the default (empty) namespace. - """ + def _get_discovery_config_library_id_by_name( + self, name: str, project: Union[ProjectId, str] + ) -> Optional[DiscoveryConfigLibraryId]: + """Returns the ID of the library with the given name in the given project, or `None` if there is none.""" response = self.make_request( "GET", "/api/discovery/config-libraries/", - params={"name_exact": name}, + params={"name_exact": name, "project": project}, ) entries = [DiscoveryConfigLibrary.model_validate(item) for item in response.json()] - matches = [entry for entry in entries if entry.name == name and entry.namespace == namespace] + matches = [entry for entry in entries if entry.name == name] if not matches: return None @@ -57,15 +54,17 @@ def _get_discovery_config_library_id_by_name(self, name: str, namespace: str) -> return matches[0].id - def get_discovery_config_library_by_name(self, name: str, namespace: str = "") -> Optional[DiscoveryConfigLibrary]: + def get_discovery_config_library_by_name( + self, name: str, project: ProjectIdentifier + ) -> Optional[DiscoveryConfigLibrary]: """ - Looks for a discovery config library matching the given name and namespace (case-sensitive, exact match). + Looks for a discovery config library matching the given name (case-sensitive, exact match) in the given project. - Library names are unique within a namespace. + `project` is a `Project`, its `ProjectId` or its `-` form. Returns it (with full YAML content) if found, otherwise `None`. """ - library_id = self._get_discovery_config_library_id_by_name(name, namespace) + library_id = self._get_discovery_config_library_id_by_name(name, unwrap_project_id(project)) if library_id is None: return None @@ -82,10 +81,13 @@ def create_discovery_config_library(self, library: DiscoveryConfigLibrary) -> Di if not library.yaml: raise ValueError("Cannot create a discovery config library without YAML content (yaml is empty)") - data = library.model_dump(exclude_none=True, by_alias=True, mode="json") + data = library.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( + library.project + ) response = self.make_request("POST", "/api/discovery/config-libraries/", data=data) created = DiscoveryConfigLibrary.model_validate(response.json()) library.id = created.id + library.project = created.project library.is_valid = created.is_valid library.validation_error = created.validation_error library.usage_count = created.usage_count @@ -123,12 +125,14 @@ def update_discovery_config_library(self, library: DiscoveryConfigLibrary) -> Di def create_or_update_discovery_config_library(self, library: DiscoveryConfigLibrary) -> DiscoveryConfigLibrary: """ - Creates the library, or updates the existing one with the same name and namespace. + Creates the library, or updates the existing one with the same name in the same project. + With no `project` set, the client's `default_project` is used, else the server's only active project, + and `DataMasqueException` is raised when it has no active project or more than one. Sets the library's `id` property. """ - library_id = self._get_discovery_config_library_id_by_name(library.name, library.namespace) + library_id = self._get_discovery_config_library_id_by_name(library.name, self._resolve_project(library.project)) if library_id is not None: library.id = library_id return self.update_discovery_config_library(library) @@ -151,15 +155,15 @@ def delete_discovery_config_library_by_id_if_exists( self._delete_if_exists(f"/api/discovery/config-libraries/{library_id}/", params=params) def delete_discovery_config_library_by_name_if_exists( - self, name: str, namespace: str = "", *, force: bool = False + self, name: str, project: ProjectIdentifier, *, force: bool = False ) -> None: """ - Deletes the discovery config library with the given name and namespace. + Deletes the discovery config library with the given name in the given project. - Library names are unique within a namespace. + `project` is a `Project`, its `ProjectId` or its `-` form. No-op if no such library exists. """ - library_id = self._get_discovery_config_library_id_by_name(name, namespace) + library_id = self._get_discovery_config_library_id_by_name(name, unwrap_project_id(project)) if library_id is not None: self.delete_discovery_config_library_by_id_if_exists(library_id, force=force) diff --git a/datamasque/client/discovery_configs.py b/datamasque/client/discovery_configs.py index d19f900..69f5cac 100644 --- a/datamasque/client/discovery_configs.py +++ b/datamasque/client/discovery_configs.py @@ -1,10 +1,11 @@ import logging -from typing import Iterator, Optional +from typing import Iterator, Optional, Union from datamasque.client.base import BaseClient -from datamasque.client.exceptions import DataMasqueApiError, DataMasqueException +from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType from datamasque.client.models.pagination import Page +from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id logger = logging.getLogger(__name__) @@ -34,14 +35,14 @@ def get_discovery_config(self, config_id: DiscoveryConfigId) -> DiscoveryConfig: return DiscoveryConfig.model_validate(response.json()) def _get_discovery_config_id_by_name( - self, name: str, config_type: DiscoveryConfigType + self, name: str, config_type: DiscoveryConfigType, project: Union[ProjectId, str] ) -> Optional[DiscoveryConfigId]: - """Return the id of the config matching name and type via a single list request, or `None`.""" + """Return the id of the config matching name, type and project via a single list request, or `None`.""" response = self.make_request( "GET", "/api/discovery/configs/", - params={"name_exact": name, "config_type": config_type.value, "limit": 1}, + params={"name_exact": name, "config_type": config_type.value, "project": project, "limit": 1}, ) page = Page[DiscoveryConfig].model_validate(response.json()) if not page.results: @@ -56,15 +57,18 @@ def _get_discovery_config_id_by_name( return config_id - def get_discovery_config_by_name(self, name: str, config_type: DiscoveryConfigType) -> Optional[DiscoveryConfig]: + def get_discovery_config_by_name( + self, name: str, config_type: DiscoveryConfigType, project: ProjectIdentifier + ) -> Optional[DiscoveryConfig]: """ - Looks for a discovery config matching the given name and type (case-sensitive, exact match). + Looks for a discovery config matching the given name (case-sensitive, exact match), type and project. - Config names are unique per type, so a type is required to identify a single config. + Config names are unique per type and project, so both are required to identify a single config. + `project` is a `Project`, its `ProjectId` or its `-` form. Returns it if found, otherwise `None`. """ - config_id = self._get_discovery_config_id_by_name(name, config_type) + config_id = self._get_discovery_config_id_by_name(name, config_type, unwrap_project_id(project)) if config_id is None: return None @@ -82,10 +86,13 @@ def create_discovery_config(self, config: DiscoveryConfig) -> DiscoveryConfig: if not config.yaml: raise ValueError("Cannot create a discovery config without YAML content (yaml is empty)") - data = config.model_dump(exclude_none=True, by_alias=True, mode="json") + data = config.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( + config.project + ) response = self.make_request("POST", "/api/discovery/configs/", data=data) created = DiscoveryConfig.model_validate(response.json()) config.id = created.id + config.project = created.project config.is_valid = created.is_valid config.validation_error = created.validation_error config.validation_error_details = created.validation_error_details @@ -123,12 +130,16 @@ def update_discovery_config(self, config: DiscoveryConfig) -> DiscoveryConfig: def create_or_update_discovery_config(self, config: DiscoveryConfig) -> DiscoveryConfig: """ - Creates the config if it doesn't exist, or updates it if one with the same name already exists. + Creates the config, or updates the existing one with the same name and type in the same project. + With no `project` set, the client's `default_project` is used, else the server's only active project, + and `DataMasqueException` is raised when it has no active project or more than one. Sets the config's `id` property. """ - existing_id = self._get_discovery_config_id_by_name(config.name, config.config_type) + existing_id = self._get_discovery_config_id_by_name( + config.name, config.config_type, self._resolve_project(config.project) + ) if existing_id is not None: config.id = existing_id return self.update_discovery_config(config) @@ -144,24 +155,20 @@ def delete_discovery_config_by_id_if_exists(self, config_id: DiscoveryConfigId) self._delete_if_exists(f"/api/discovery/configs/{config_id}/") - def delete_discovery_config_by_name_if_exists(self, name: str, config_type: DiscoveryConfigType) -> None: + def delete_discovery_config_by_name_if_exists( + self, name: str, config_type: DiscoveryConfigType, project: ProjectIdentifier + ) -> None: """ - Deletes the discovery config with the given name and type. + Deletes the discovery config with the given name and type in the given project. - Config names are unique per type, so a type is required to identify a single config. + Config names are unique per type and project, so both are required to identify a single config. + `project` is a `Project`, its `ProjectId` or its `-` form. No-op if no such config exists. """ - matching = [ - config - for config in self.list_discovery_configs() - if config.name == name and config.config_type is config_type - ] - for config in matching: - if config.id is None: - raise DataMasqueException(f'Server returned a discovery config named "{config.name}" without an `id`.') - - self.delete_discovery_config_by_id_if_exists(config.id) + config_id = self._get_discovery_config_id_by_name(name, config_type, unwrap_project_id(project)) + if config_id is not None: + self.delete_discovery_config_by_id_if_exists(config_id) def get_default_discovery_config_yaml(self) -> str: """Returns the server's built-in default discovery configuration as a YAML string.""" diff --git a/datamasque/client/dmclient.py b/datamasque/client/dmclient.py index 7e8cc7f..6acaa69 100644 --- a/datamasque/client/dmclient.py +++ b/datamasque/client/dmclient.py @@ -1,10 +1,14 @@ from datamasque.client.base import FileOrContent, UploadFile +from datamasque.client.bundles import BundleClient from datamasque.client.connections import ConnectionClient from datamasque.client.discovery import DiscoveryClient from datamasque.client.discovery_config_libraries import DiscoveryConfigLibraryClient from datamasque.client.discovery_configs import DiscoveryConfigClient from datamasque.client.files import FileClient +from datamasque.client.git import GitClient from datamasque.client.license import LicenseClient +from datamasque.client.projects import ProjectClient +from datamasque.client.resources import ResourceClient from datamasque.client.rg_configs import RGConfigClient from datamasque.client.ruleset_libraries import RulesetLibraryClient from datamasque.client.rulesets import RulesetClient @@ -18,14 +22,18 @@ class DataMasqueClient( LicenseClient, + ProjectClient, + ResourceClient, ConnectionClient, RulesetClient, RulesetLibraryClient, FileClient, + BundleClient, RunClient, DiscoveryClient, DiscoveryConfigClient, DiscoveryConfigLibraryClient, + GitClient, TableReferenceClient, RGConfigClient, UserClient, diff --git a/datamasque/client/exceptions.py b/datamasque/client/exceptions.py index 9943b8b..747fd64 100644 --- a/datamasque/client/exceptions.py +++ b/datamasque/client/exceptions.py @@ -1,5 +1,7 @@ from requests import Response +from datamasque.client.models.project import ProjectDeleteConflict + class DataMasqueException(Exception): """Generic exception base class.""" @@ -66,6 +68,21 @@ class RGConfigNotFoundError(DataMasqueApiError): """ +class ProjectDeleteConflictError(DataMasqueApiError): + """ + Raised when `delete_project` cannot move the project's resources to the destination project. + + The server deletes and moves nothing in this case. + + Attributes: + conflicts: Every reason the resources cannot move. + """ + + def __init__(self, message: str, *, response: Response, conflicts: list[ProjectDeleteConflict]) -> None: + super().__init__(message, response=response) + self.conflicts = conflicts + + class DataMasqueTransportError(DataMasqueException): """ Raised when a request to the DataMasque server fails before any response is received. diff --git a/datamasque/client/files.py b/datamasque/client/files.py index 89d1a67..641b4bc 100644 --- a/datamasque/client/files.py +++ b/datamasque/client/files.py @@ -3,6 +3,7 @@ from datamasque.client.base import BaseClient, UploadFile, read_file_or_content from datamasque.client.models.files import DataMasqueFile +from datamasque.client.models.project import ProjectIdentifier, unwrap_project_id FileTypeT = TypeVar("FileTypeT", bound=DataMasqueFile) @@ -15,13 +16,15 @@ def upload_file( file_type: Type[FileTypeT], file_name: str, file_path_or_content: Union[str, bytes, Path], + project: ProjectIdentifier, ) -> FileTypeT: """ - Uploads a file of the given type to the DataMasque server. + Uploads a file of the given type to the DataMasque server, into the given project. `file_type` must be a concrete subclass of `DataMasqueFile` (`SeedFile`, `OracleWalletFile`, `SslZipFile`, `SnowflakeKeyFile`). `file_path_or_content` may be a path (as `str` or `Path`), raw `bytes`, or a file-like object. + `project` is a `Project`, its `ProjectId` or its `-` form. """ name, content = read_file_or_content(file_path_or_content, file_name) @@ -30,7 +33,7 @@ def upload_file( response = self.make_request( "POST", file_type.get_url(), - data={"name": file_name, **file_type.get_extra_form_data()}, + data={"name": file_name, "project": unwrap_project_id(project), **file_type.get_extra_form_data()}, files=[ UploadFile( field_name=file_type.get_content_param_name(), @@ -56,37 +59,50 @@ def delete_file_if_exists(self, file: DataMasqueFile) -> None: # file.get_url() ends with a slash so no need to insert one before the id self._delete_if_exists(f"{file.get_url()}{file.id}/") - def list_files_of_type(self, file_type: Type[FileTypeT]) -> list[FileTypeT]: - """Returns all files of the given type (a concrete subclass of `DataMasqueFile`).""" + def list_files_of_type( + self, file_type: Type[FileTypeT], project: Optional[ProjectIdentifier] = None + ) -> list[FileTypeT]: + """ + Returns all files of the given type (a concrete subclass of `DataMasqueFile`). + + Only files in `project` are returned when it is given. + `project` is a `Project`, its `ProjectId` or its `-` form. + """ - response = self.make_request("GET", file_type.get_url()) + params = {"project": unwrap_project_id(project)} if project is not None else None + response = self.make_request("GET", file_type.get_url(), params=params) return [file_type.model_validate(file) for file in response.json()] - def get_file_of_type_by_name(self, file_type: Type[FileTypeT], name: str) -> Optional[FileTypeT]: + def get_file_of_type_by_name( + self, file_type: Type[FileTypeT], name: str, project: ProjectIdentifier + ) -> Optional[FileTypeT]: """ - Looks for a file of the given type (a concrete subclass of `DataMasqueFile`) with the given `name`. + Looks for a file of the given type (a concrete subclass of `DataMasqueFile`) by name in the given project. Returns it if found, otherwise `None`. """ - matching_files = [f for f in self.list_files_of_type(file_type) if f.name == name] + matching_files = [f for f in self.list_files_of_type(file_type, project) if f.name == name] return matching_files[0] if matching_files else None - def upload_file_if_not_exists(self, file_type: Type[FileTypeT], file_path: Union[str, Path]) -> Optional[FileTypeT]: + def upload_file_if_not_exists( + self, file_type: Type[FileTypeT], file_path: Union[str, Path], project: ProjectIdentifier + ) -> Optional[FileTypeT]: """ - Upload a file only if one with the same name doesn't already exist. + Upload a file only if one with the same name doesn't already exist in the project. Args: file_type: A concrete subclass of `DataMasqueFile` (e.g., SeedFile, OracleWalletFile). file_path: Path to the file to upload. + project: A `Project`, its `ProjectId` or its `-` form. Returns: The uploaded file object if a new file was uploaded, or None if a file - with the same name already exists. + with the same name already exists in the project. """ file_path = Path(file_path) - if self.get_file_of_type_by_name(file_type, file_path.name) is not None: + if self.get_file_of_type_by_name(file_type, file_path.name, project) is not None: return None - return self.upload_file(file_type, file_path.name, file_path) + return self.upload_file(file_type, file_path.name, file_path, project) diff --git a/datamasque/client/git.py b/datamasque/client/git.py new file mode 100644 index 0000000..fc238d9 --- /dev/null +++ b/datamasque/client/git.py @@ -0,0 +1,463 @@ +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import Literal, Optional, Union + +from pydantic import BaseModel + +from datamasque.client.base import BaseClient +from datamasque.client.models.discovery_config import DiscoveryConfig +from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary +from datamasque.client.models.git import GitFile +from datamasque.client.models.ruleset import Ruleset +from datamasque.client.models.ruleset_library import RulesetLibrary + +GitExtension = Literal[".yml", ".yaml"] +"""The extensions the server accepts for a pushed file.""" + +DEFAULT_GIT_EXTENSION: GitExtension = ".yml" + + +@dataclass(frozen=True) +class _GitResourceType: + """The endpoint and request field names one resource type's git operations use.""" + + endpoint: str + name_field: str + content_field: str + folder_param: str + """The pull parameter naming the file's folder, as the file list shows it.""" + + +_RULESET = _GitResourceType("/api/ruleset-git/", "ruleset_name", "ruleset_content", "folder") +_RULESET_LIBRARY = _GitResourceType("/api/library-git/", "library_name", "library_content", "namespace") +_DISCOVERY_CONFIG = _GitResourceType("/api/discovery-config-git/", "config_name", "config_content", "folder") +_DISCOVERY_CONFIG_LIBRARY = _GitResourceType( + "/api/discovery-config-library-git/", "library_name", "library_content", "namespace" +) + + +class _GitPull(BaseModel): + """The response to a pull.""" + + config_yaml: str + + +class _GitPushPath(BaseModel): + """The response to a push-path request.""" + + path: str + + +def _branch_param(branch: Optional[str]) -> dict[str, str]: + return {} if branch is None else {"branch": branch} + + +def _require_yaml(yaml: Optional[str], description: str, getter: str) -> str: + if not yaml: + raise ValueError( + f"Cannot push a {description} to git without YAML content (yaml is empty or unset); " + f"fetch the {description} with its YAML using `{getter}` first" + ) + + return yaml + + +class GitClient(BaseClient): + """ + Git push, pull and file listing for rulesets, ruleset libraries, discovery configs and their libraries. + + Mixed into `DataMasqueClient`. + The server reads and writes the repository in its Git settings with the user's SSH key, + so both must be set up for the user the client authenticates as. + Each `branch` parameter defaults to the branch in the Git settings. + + A push writes `//` under the Git directory path, + where `` is the project's `name`, not its `reference`, + and the type folders are `rulesets/`, `libraries/`, `discovery-configs/` and `discovery-config-libraries/`. + A resource pushed before the server's upgrade to projects keeps the file it had then, + until it is renamed or moved to another project: + a ruleset at the top of the Git directory path, + a discovery config directly in `discovery-configs/`, + a library of the `Default` project directly in its type folder, + and a library of another namespace in the namespace's folder, + such as `libraries/bank/au/` for the namespace that the upgrade made the project `bank_au`. + + The methods give paths in two forms: + + - `get_*_git_push_path` returns the file's path in the repository without its extension, + starting with the Git directory path. + - `list_*_git_files` returns each file's path with its extension, + relative to the Git directory path for rulesets and to the type folder for the other types. + The matching `pull_*_from_git` method takes a path in this form. + + For example, to restore a ruleset to the commit before its latest one and push it back: + + .. code-block:: python + + banking = next(project for project in client.iter_projects() if project.name == "Banking") + file = next(f for f in client.list_ruleset_git_files() if f.path == "rulesets/Banking/customers.yml") + yaml = client.pull_ruleset_from_git(file.path, file.commits[1].commit_sha) + ruleset = client.create_or_update_ruleset(Ruleset(name="customers", project=banking.reference, yaml=yaml)) + client.push_ruleset_to_git(ruleset, "Restore the previous customers ruleset") + """ + + def list_ruleset_git_files(self, *, branch: Optional[str] = None) -> list[GitFile]: + """ + Lists the ruleset files in the git repository. + + Paths are relative to the Git directory path: + `rulesets//customers.yml` for a project's ruleset, + and `customers.yml` for one pushed before the server's upgrade to projects. + """ + + return self._list_git_files(_RULESET, branch) + + def pull_ruleset_from_git(self, path: str, commit_sha: str, *, branch: Optional[str] = None) -> str: + """ + Returns the YAML of a ruleset file at `commit_sha`. + + `path` is the file's path as `list_ruleset_git_files` lists it. + Saves nothing: set it as a ruleset's `yaml` and save that ruleset to keep it. + """ + + return self._pull_from_git(_RULESET, path, commit_sha, branch) + + def push_ruleset_to_git( + self, + ruleset: Ruleset, + commit_message: str, + *, + branch: Optional[str] = None, + extension: GitExtension = DEFAULT_GIT_EXTENSION, + ) -> None: + """ + Commits the ruleset's `yaml` to the git repository and pushes it. + + The file is the ruleset's name with `extension` (`.yml` or `.yaml`) in its project's folder, + unless the saved ruleset kept an older file (see `GitClient`). + A push with the other extension writes a second file and leaves the first. + With no `project` set, the project is the saved ruleset's when it has an `id`, + else the client's `default_project`, else the server's only active project. + The ruleset's `id`, else its name, type and project, identify the saved ruleset the push is for. + With an `id`, the ruleset's name, type and project must match the saved ruleset's. + The push sends the given ruleset's `yaml`, not the saved one, so save the ruleset first. + The server records the pushed commit as the saved ruleset's git provenance when their YAML matches, + but the given ruleset is not updated. + + Raises: + ValueError: The ruleset's `yaml` is empty. + DataMasqueApiError: With status 409 when another ruleset uses or would write the file, + and with status 400 when nothing changed, the repository has a file where the path needs a folder, + or no project is set and the server does not have exactly one active project. + """ + + self._push_to_git( + _RULESET, + self._ruleset_git_target(ruleset), + _require_yaml(ruleset.yaml, "ruleset", "list_rulesets"), + commit_message, + branch=branch, + extension=extension, + ) + + def get_ruleset_git_push_path(self, ruleset: Ruleset) -> str: + """ + Returns the path `push_ruleset_to_git` would write the ruleset to. + + The path is relative to the repository and has no extension. + """ + + return self._get_git_push_path(_RULESET, self._ruleset_git_target(ruleset)) + + def list_ruleset_library_git_files(self, *, branch: Optional[str] = None) -> list[GitFile]: + """ + Lists the ruleset library files in the git repository. + + Paths are relative to `libraries/` under the Git directory path: + `/emails.yml` for a project's library, + `emails.yml` for a library of the `Default` project pushed before the server's upgrade to projects, + and `/emails.yml` for a library in a namespace whose project the upgrade named differently, + such as `bank/au/emails.yml` in the project `bank_au`. + """ + + return self._list_git_files(_RULESET_LIBRARY, branch) + + def pull_ruleset_library_from_git(self, path: str, commit_sha: str, *, branch: Optional[str] = None) -> str: + """ + Returns the YAML of a library file at `commit_sha`. + + `path` is the file's path as `list_ruleset_library_git_files` lists it. + Saves nothing: set it as a library's `yaml` and save that library to keep it. + """ + + return self._pull_from_git(_RULESET_LIBRARY, path, commit_sha, branch) + + def push_ruleset_library_to_git( + self, + library: RulesetLibrary, + commit_message: str, + *, + branch: Optional[str] = None, + extension: GitExtension = DEFAULT_GIT_EXTENSION, + ) -> None: + """ + Commits the library's `yaml` to the git repository and pushes it. + + The file is the library's name with `extension` (`.yml` or `.yaml`) in its project's folder, + unless the saved library kept an older file (see `GitClient`). + A push with the other extension writes a second file and leaves the first. + With no `project` set, the project is the saved library's when it has an `id`, + else the client's `default_project`, else the server's only active project. + The library's `id`, else its name and project, identify the saved library the push is for. + With an `id`, the library's name and project must match the saved library's. + The push sends the given library's `yaml`, not the saved one, so save the library first. + The server records the pushed commit as the saved library's git provenance when their YAML matches, + but the given library is not updated. + + Raises: + ValueError: The library's `yaml` is empty or unset. + DataMasqueApiError: With status 409 when another library uses or would write the file, + and with status 400 when nothing changed, the repository has a file where the path needs a folder, + or no project is set and the server does not have exactly one active project. + """ + + self._push_to_git( + _RULESET_LIBRARY, + self._library_git_target(_RULESET_LIBRARY, library), + _require_yaml(library.yaml, "ruleset library", "get_ruleset_library"), + commit_message, + branch=branch, + extension=extension, + ) + + def get_ruleset_library_git_push_path(self, library: RulesetLibrary) -> str: + """ + Returns the path `push_ruleset_library_to_git` would write the library to. + + The path is relative to the repository and has no extension. + """ + + return self._get_git_push_path(_RULESET_LIBRARY, self._library_git_target(_RULESET_LIBRARY, library)) + + def list_discovery_config_git_files(self, *, branch: Optional[str] = None) -> list[GitFile]: + """ + Lists the discovery config files in the git repository. + + Paths are relative to `discovery-configs/` under the Git directory path: + `/customers.yml` for a project's config, + and `customers.yml` for one pushed before the server's upgrade to projects. + """ + + return self._list_git_files(_DISCOVERY_CONFIG, branch) + + def pull_discovery_config_from_git(self, path: str, commit_sha: str, *, branch: Optional[str] = None) -> str: + """ + Returns the YAML of a config file at `commit_sha`. + + `path` is the file's path as `list_discovery_config_git_files` lists it. + Saves nothing: set it as a config's `yaml` and save that config to keep it. + """ + + return self._pull_from_git(_DISCOVERY_CONFIG, path, commit_sha, branch) + + def push_discovery_config_to_git( + self, + config: DiscoveryConfig, + commit_message: str, + *, + branch: Optional[str] = None, + extension: GitExtension = DEFAULT_GIT_EXTENSION, + ) -> None: + """ + Commits the discovery config's `yaml` to the git repository and pushes it. + + The file is the config's name with `extension` (`.yml` or `.yaml`) in its project's folder, + unless the saved config kept an older file (see `GitClient`). + A push with the other extension writes a second file and leaves the first. + With no `project` set, the project is the saved config's when it has an `id`, + else the client's `default_project`, else the server's only active project. + The config's `id`, else its name, type and project, identify the saved config the push is for. + With an `id`, the config's name, type and project must match the saved config's. + The push sends the given config's `yaml`, not the saved one, so save the config first. + The server records the pushed commit on the saved config when their YAML matches, + but `DiscoveryConfig` does not expose it. + + Raises: + ValueError: The config's `yaml` is empty or unset. + DataMasqueApiError: With status 409 when another config uses or would write the file, + and with status 400 when nothing changed, the repository has a file where the path needs a folder, + or no project is set and the server does not have exactly one active project. + """ + + self._push_to_git( + _DISCOVERY_CONFIG, + self._discovery_config_git_target(config), + _require_yaml(config.yaml, "discovery config", "get_discovery_config"), + commit_message, + branch=branch, + extension=extension, + ) + + def get_discovery_config_git_push_path(self, config: DiscoveryConfig) -> str: + """ + Returns the path `push_discovery_config_to_git` would write the config to. + + The path is relative to the repository and has no extension. + """ + + return self._get_git_push_path(_DISCOVERY_CONFIG, self._discovery_config_git_target(config)) + + def list_discovery_config_library_git_files(self, *, branch: Optional[str] = None) -> list[GitFile]: + """ + Lists the discovery config library files in the git repository. + + Paths are relative to `discovery-config-libraries/` under the Git directory path: + `/emails.yml` for a project's library, + `emails.yml` for a library of the `Default` project pushed before the server's upgrade to projects, + and `/emails.yml` for a library in a namespace whose project the upgrade named differently, + such as `bank/au/emails.yml` in the project `bank_au`. + """ + + return self._list_git_files(_DISCOVERY_CONFIG_LIBRARY, branch) + + def pull_discovery_config_library_from_git( + self, path: str, commit_sha: str, *, branch: Optional[str] = None + ) -> str: + """ + Returns the YAML of a library file at `commit_sha`. + + `path` is the file's path as `list_discovery_config_library_git_files` lists it. + Saves nothing: set it as a library's `yaml` and save that library to keep it. + """ + + return self._pull_from_git(_DISCOVERY_CONFIG_LIBRARY, path, commit_sha, branch) + + def push_discovery_config_library_to_git( + self, + library: DiscoveryConfigLibrary, + commit_message: str, + *, + branch: Optional[str] = None, + extension: GitExtension = DEFAULT_GIT_EXTENSION, + ) -> None: + """ + Commits the discovery config library's `yaml` to the git repository and pushes it. + + The file is the library's name with `extension` (`.yml` or `.yaml`) in its project's folder, + unless the saved library kept an older file (see `GitClient`). + A push with the other extension writes a second file and leaves the first. + With no `project` set, the project is the saved library's when it has an `id`, + else the client's `default_project`, else the server's only active project. + The library's `id`, else its name and project, identify the saved library the push is for. + With an `id`, the library's name and project must match the saved library's. + The push sends the given library's `yaml`, not the saved one, so save the library first. + The server records the pushed commit on the saved library when their YAML matches, + but `DiscoveryConfigLibrary` does not expose it. + + Raises: + ValueError: The library's `yaml` is empty or unset. + DataMasqueApiError: With status 409 when another library uses or would write the file, + and with status 400 when nothing changed, the repository has a file where the path needs a folder, + or no project is set and the server does not have exactly one active project. + """ + + self._push_to_git( + _DISCOVERY_CONFIG_LIBRARY, + self._library_git_target(_DISCOVERY_CONFIG_LIBRARY, library), + _require_yaml(library.yaml, "discovery config library", "get_discovery_config_library"), + commit_message, + branch=branch, + extension=extension, + ) + + def get_discovery_config_library_git_push_path(self, library: DiscoveryConfigLibrary) -> str: + """ + Returns the path `push_discovery_config_library_to_git` would write the library to. + + The path is relative to the repository and has no extension. + """ + + return self._get_git_push_path( + _DISCOVERY_CONFIG_LIBRARY, + self._library_git_target(_DISCOVERY_CONFIG_LIBRARY, library), + ) + + def _ruleset_git_target(self, ruleset: Ruleset) -> dict[str, str]: + return self._git_target( + _RULESET, ruleset.name, ruleset.project, ruleset.id, mask_type=ruleset.ruleset_type.value + ) + + def _discovery_config_git_target(self, config: DiscoveryConfig) -> dict[str, str]: + return self._git_target( + _DISCOVERY_CONFIG, config.name, config.project, config.id, config_type=config.config_type.value + ) + + def _library_git_target( + self, resource_type: _GitResourceType, library: Union[RulesetLibrary, DiscoveryConfigLibrary] + ) -> dict[str, str]: + return self._git_target(resource_type, library.name, library.project, library.id) + + def _git_target( + self, + resource_type: _GitResourceType, + name: str, + project: Optional[str], + resource_id: Optional[str], + **type_fields: str, + ) -> dict[str, str]: + """ + Returns the fields that pick the file a push writes, as the push and push-path requests take them. + + An unset `project` is left for the server to take from the saved resource when there is an `id`, + and is otherwise the client's `default_project` when there is one. + """ + + fields: dict[str, str] = {resource_type.name_field: name, **type_fields} + if resource_id is not None: + fields["id"] = resource_id + if project is not None: + fields["project"] = project + elif resource_id is None: + fields |= self._default_project_field(None) + + return fields + + def _list_git_files(self, resource_type: _GitResourceType, branch: Optional[str]) -> list[GitFile]: + response = self.make_request("GET", f"{resource_type.endpoint}files/", params=_branch_param(branch)) + return [GitFile(path=path, commits=commits) for path, commits in response.json().items()] + + def _pull_from_git(self, resource_type: _GitResourceType, path: str, commit_sha: str, branch: Optional[str]) -> str: + file_path = PurePosixPath(path) + folder = file_path.parent + params = { + resource_type.name_field: file_path.stem, + "extension": file_path.suffix, + resource_type.folder_param: "" if folder == PurePosixPath(".") else folder.as_posix(), + "commit_id": commit_sha, + **_branch_param(branch), + } + response = self.make_request("GET", resource_type.endpoint, params=params) + return _GitPull.model_validate(response.json()).config_yaml + + def _push_to_git( + self, + resource_type: _GitResourceType, + target: dict[str, str], + content: str, + commit_message: str, + *, + branch: Optional[str], + extension: GitExtension, + ) -> None: + data = { + **target, + resource_type.content_field: content, + "commit_message": commit_message, + "extension": extension, + **_branch_param(branch), + } + self.make_request("POST", resource_type.endpoint, data=data) + + def _get_git_push_path(self, resource_type: _GitResourceType, target: dict[str, str]) -> str: + response = self.make_request("GET", f"{resource_type.endpoint}push-path/", params=target) + return _GitPushPath.model_validate(response.json()).path diff --git a/datamasque/client/models/bundle.py b/datamasque/client/models/bundle.py new file mode 100644 index 0000000..0504b5d --- /dev/null +++ b/datamasque/client/models/bundle.py @@ -0,0 +1,165 @@ +"""Export and import bundle models for the DataMasque API.""" + +from enum import Enum +from typing import Generic, Optional, TypeVar + +from pydantic import BaseModel, ConfigDict + +from datamasque.client.models.ruleset import RulesetId, RulesetType +from datamasque.client.models.ruleset_library import RulesetLibraryId + + +class BundleImportStatus(str, Enum): + """What an import did with one item of a bundle.""" + + created = "CREATED" + """No item with that ID or name existed, so it was created. Counted in `created`.""" + created_duplicate = "CREATED_DUPLICATE" + """ + A ruleset with that name existed and `overwrite_rulesets` was not set, + so it was created under a new name, given in `imported_name`. + Counted in `created`. + """ + not_created = "NOT_CREATED" + """ + The item was left unchanged, because an item with that name already has identical content, + or because a ruleset with that ID exists and `overwrite_rulesets` was not set. + Not counted in any count but `processed`. + """ + restored = "RESTORED" + """ + An item with that ID, or with that name in the same project, had been deleted, + so it was brought back with the bundle's name and content. + Counted in `restored`. + """ + updated = "UPDATED" + """ + An existing item was overwritten, because the import's `overwrite_*` option for its type was `True`. + Counted in `updated`. + """ + skipped = "SKIPPED" + """ + An existing item was left unchanged, because the import's `overwrite_*` option for its type was `False`. + Counted in `skipped`. + """ + error = "ERROR" + """The item could not be imported; `message` says why. Counted in `error`.""" + + +class BundleImportCounts(BaseModel): + """How many items of one type an import processed, and what it did with them.""" + + model_config = ConfigDict(extra="allow") + + processed: int + """The number of items of the type in the bundle.""" + created: int + """The number of `CREATED` and `CREATED_DUPLICATE` items.""" + restored: int + """The number of `RESTORED` items.""" + error: int + """The number of `ERROR` items.""" + updated: Optional[int] = None + """The number of `UPDATED` items, or `None` for rulesets imported without `overwrite_rulesets`.""" + skipped: Optional[int] = None + """The number of `SKIPPED` items, or `None` for rulesets imported without `overwrite_rulesets`.""" + + +class RulesetImportResult(BaseModel): + """What an import did with one ruleset of a bundle.""" + + model_config = ConfigDict(extra="allow") + + exported_name: str + exported_id: Optional[RulesetId] = None + """`None` for a hand-built Zip archive without the `manifest.json` that an export writes.""" + exported_project: Optional[str] = None + """The name of the project the bundle files the ruleset under, or `None` for a Version 1 bundle.""" + imported_name: Optional[str] = None + """`None` on error.""" + imported_id: Optional[RulesetId] = None + """`None` on error.""" + imported_project: Optional[str] = None + """The name of the project the ruleset is in, or `None` on error.""" + ruleset_type: RulesetType + status: BundleImportStatus + message: str + """A description of the outcome, which can change between DataMasque versions; check `status` instead.""" + + +class LibraryImportResult(BaseModel): + """What an import did with one library of a bundle.""" + + model_config = ConfigDict(extra="allow") + + exported_name: str + exported_namespace: str + """ + The folder a Version 1 bundle files the library under, which names its project; + `""` for a library in no folder, and for every library in a Version 2 bundle. + """ + exported_project: Optional[str] = None + """The name of the project the bundle files the library under, or `None` for a Version 1 bundle.""" + exported_id: Optional[RulesetLibraryId] = None + """`None` for a hand-built Zip archive without the `manifest.json` that an export writes.""" + imported_name: Optional[str] = None + """`None` on error.""" + imported_id: Optional[RulesetLibraryId] = None + """`None` on error.""" + imported_project: Optional[str] = None + """The name of the project the library is in, or `None` on error.""" + status: BundleImportStatus + message: str + """A description of the outcome, which can change between DataMasque versions; check `status` instead.""" + + +class SeedFileImportResult(BaseModel): + """What an import did with one seed file of a bundle.""" + + model_config = ConfigDict(extra="allow") + + exported_name: str + exported_project: Optional[str] = None + """The name of the project the bundle files the seed file under, or `None` for a Version 1 bundle.""" + imported_name: Optional[str] = None + """`None` on error.""" + imported_project: Optional[str] = None + """The name of the project the seed file is in, or `None` on error.""" + status: BundleImportStatus + message: str + """A description of the outcome, which can change between DataMasque versions; check `status` instead.""" + + +ItemResultT = TypeVar("ItemResultT", RulesetImportResult, LibraryImportResult, SeedFileImportResult) + + +class BundleImportItemResults(BaseModel, Generic[ItemResultT]): + """The counts and per-item results of one item type in an import.""" + + model_config = ConfigDict(extra="allow") + + metadata: BundleImportCounts + data: list[ItemResultT] + + +class BundleImportResult(BaseModel): + """The outcome of `import_bundle`, grouped by item type.""" + + model_config = ConfigDict(extra="allow") + + rulesets: BundleImportItemResults[RulesetImportResult] + libraries: Optional[BundleImportItemResults[LibraryImportResult]] = None + """`None` when the bundle has no libraries.""" + seed_files: Optional[BundleImportItemResults[SeedFileImportResult]] = None + """`None` when the bundle has no seed files.""" + fallback_project: Optional[str] = None + """ + The name of the project that took the resources that named no project, such as a Version 1 bundle's rulesets. + `None` when the import created, restored or updated no such resources. + """ + is_deprecated_bundle: bool = False + """ + Whether the bundle is in the deprecated Version 1 format, + or is a hand-built Zip archive without `manifest.json`, which is read as Version 1. + `import_bundle` sets it from the response's `Deprecation` header, which the response body does not carry. + """ diff --git a/datamasque/client/models/connection.py b/datamasque/client/models/connection.py index 5bec97b..cde6bd0 100644 --- a/datamasque/client/models/connection.py +++ b/datamasque/client/models/connection.py @@ -121,13 +121,15 @@ class ConnectionConfig(BaseModel): model_config = ConfigDict(extra="allow", populate_by_name=True) name: str + project: Optional[str] = None + """ + The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, + or the project's ID as a string. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. + """ id: Optional[ConnectionId] = None # Server-populated and read-only, so it is excluded when serializing a config back to the API. license_lock: Optional[LicenseLock] = Field(default=None, exclude=True) - # Its project's `-` reference, e.g. `1-default`; - # `None` until it is created on the server. - # Server-populated, so excluded when serializing. - project: Optional[str] = Field(default=None, exclude=True) # Whether the connection's protected fields are currently locked under the license edit lock. target_frozen: bool = Field(default=False, exclude=True) # The protected fields currently locked, as server config keys (e.g. `schema`, not `db_schema`); diff --git a/datamasque/client/models/discovery_config.py b/datamasque/client/models/discovery_config.py index 086257f..3d767bb 100644 --- a/datamasque/client/models/discovery_config.py +++ b/datamasque/client/models/discovery_config.py @@ -40,13 +40,17 @@ class DiscoveryConfig(BaseModel): model_config = ConfigDict(extra="allow", populate_by_name=True) name: str + project: Optional[str] = None + """ + The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, + or the project's ID as a string. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. + """ yaml: Optional[str] = Field(default=None, alias="config_yaml") config_type: DiscoveryConfigType # Server-populated read-only fields, excluded from request bodies. id: Optional[DiscoveryConfigId] = Field(default=None, exclude=True) - project: Optional[str] = Field(default=None, exclude=True) - """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True) """Validation status; may be `in_progress` briefly after creating a large config.""" validation_error: Optional[str] = Field(default=None, exclude=True) diff --git a/datamasque/client/models/discovery_config_library.py b/datamasque/client/models/discovery_config_library.py index 629160f..06058bb 100644 --- a/datamasque/client/models/discovery_config_library.py +++ b/datamasque/client/models/discovery_config_library.py @@ -1,8 +1,9 @@ from datetime import datetime from typing import NewType, Optional -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator +from datamasque.client.models.project import drop_namespace from datamasque.client.models.status import ValidationStatus DiscoveryConfigLibraryId = NewType("DiscoveryConfigLibraryId", str) @@ -10,21 +11,24 @@ class DiscoveryConfigLibrary(BaseModel): """ - Represents a named, namespaced, persisted YAML discovery config library. + Represents a named, persisted YAML discovery config library. A library is untyped: the same library may be imported by both database and - file discovery configs, and its name is unique within a namespace. + file discovery configs, and its name is unique within its project. """ model_config = ConfigDict(extra="allow", populate_by_name=True) name: str - namespace: str = "" + project: Optional[str] = None + """ + The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, + or the project's ID as a string. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. + """ yaml: Optional[str] = Field(default=None, alias="config_yaml") # Server-populated read-only fields, excluded from request bodies. id: Optional[DiscoveryConfigLibraryId] = Field(default=None, exclude=True) - project: Optional[str] = Field(default=None, exclude=True) - """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True) """Validation status; libraries are validated synchronously on create/update.""" validation_error: Optional[str] = Field(default=None, exclude=True) @@ -33,3 +37,5 @@ class DiscoveryConfigLibrary(BaseModel): """Number of active discovery configs that import this library.""" created: Optional[datetime] = Field(default=None, exclude=True) modified: Optional[datetime] = Field(default=None, exclude=True) + + _drop_namespace = model_validator(mode="before")(drop_namespace) diff --git a/datamasque/client/models/dm_instance.py b/datamasque/client/models/dm_instance.py index 7b30a98..0f82152 100644 --- a/datamasque/client/models/dm_instance.py +++ b/datamasque/client/models/dm_instance.py @@ -49,6 +49,12 @@ class DataMasqueInstanceConfig(BaseModel): """How long a request may take to connect, including the TLS handshake.""" read_timeout_seconds: float = DEFAULT_READ_TIMEOUT_SECONDS """How long a request may wait for the server between bytes of its response.""" + default_project: Optional[str] = None + """The project a create goes into when the resource names none, as a `-` reference (e.g. `1-default`) + or a bare project id. + + Applies to connections, rulesets, discovery configs, RG configs, table references and libraries. + Leave unset to use the server's only active project.""" @model_validator(mode="after") def _validate_auth_source(self) -> "DataMasqueInstanceConfig": diff --git a/datamasque/client/models/files.py b/datamasque/client/models/files.py index 13c9846..6b8dc65 100644 --- a/datamasque/client/models/files.py +++ b/datamasque/client/models/files.py @@ -16,6 +16,8 @@ class DataMasqueFile(BaseModel): created_date: datetime modified_date: Optional[datetime] = None id: Optional[FileId] = None + project: Optional[str] = None + """Its project's `-` reference, or `None` for a built-in seed, which belongs to no project.""" @model_validator(mode="before") @classmethod diff --git a/datamasque/client/models/git.py b/datamasque/client/models/git.py index df8e71a..c409f6c 100644 --- a/datamasque/client/models/git.py +++ b/datamasque/client/models/git.py @@ -58,3 +58,32 @@ def _collapse_git_fields(cls, data: Any) -> Any: data["git"] = snapshot return data + + +class GitFileCommit(BaseModel): + """A commit that changed a file in the git repository.""" + + model_config = ConfigDict(extra="allow", populate_by_name=True) + + commit_sha: str = Field(alias="commit") + """The commit's SHA, which the `pull_*_from_git` methods take.""" + date: datetime + """When the commit was authored.""" + message: str + """The commit message.""" + + +class GitFile(BaseModel): + """A YAML file in the git repository, as a `list_*_git_files` method lists it.""" + + path: str + """ + The file's path with its extension, + relative to the Git directory path for rulesets and to the type folder for the other types. + Pass it to the matching `pull_*_from_git` method. + """ + commits: list[GitFileCommit] + """ + The file's most recent commits, newest first (the server caps how many). + The `pull_*_from_git` methods also take the SHA of an older commit, such as one from `git log`. + """ diff --git a/datamasque/client/models/project.py b/datamasque/client/models/project.py new file mode 100644 index 0000000..79c8c72 --- /dev/null +++ b/datamasque/client/models/project.py @@ -0,0 +1,133 @@ +"""Project models for the DataMasque API.""" + +from datetime import datetime +from enum import Enum +from typing import NewType, Optional, Union + +from pydantic import BaseModel, ConfigDict + +from datamasque.client.models.resource_type import ResourceTypeName + +ProjectId = NewType("ProjectId", int) + + +def drop_namespace(cls: type, data: object) -> object: + """ + Drops the deprecated `namespace` field that the server still returns on libraries. + + The models allow extra fields, so a kept `namespace` would be sent back on update, + and the server lets `namespace` override `project`. + """ + + if isinstance(data, dict) and "namespace" in data: + data = {key: value for key, value in data.items() if key != "namespace"} + + return data + + +class Project(BaseModel): + """A project: the top-level grouping that connections, rulesets and other resources belong to.""" + + model_config = ConfigDict(extra="allow") + + id: ProjectId + reference: str + """The `-` string other resources use to refer to this project.""" + name: str + description: str = "" + created: datetime + updated: datetime + has_git_synced_resources: Optional[bool] = None + """ + Whether any library, discovery config library, ruleset or discovery config in the project + that is not deleted is synced with git. + + A resource edited since its last push is not counted. + Only `get_project` sets it; other calls leave it `None`. + After a rename, a git-synced resource's next push writes a new copy under the new name and the old copy stays. + DataMasque does not move files in the repository. + """ + + +# A project, given as a `Project`, its `ProjectId`, or its `-` form such as a listed resource's `project`. +ProjectIdentifier = Union[ProjectId, Project, str] + + +def unwrap_project_id(project: ProjectIdentifier) -> ProjectId: + """ + Return the ID of `project`. + + Raises: + ValueError: `project` is a string that is not a project reference in the form `-`. + """ + if isinstance(project, Project): + return project.id + + if isinstance(project, str): + try: + return ProjectId(int(project.split("-")[0])) + except ValueError: + raise ValueError( + f'"{project}" is not a project ID or reference. ' + "Specify the ID as an integer, or a reference in the form `3-myproject`." + ) from None + + return project + + +class ProjectCreateRequest(BaseModel): + """ + Request body for `POST /api/projects/`. + + A name must not be `.`, start with `DataMasque_`, or contain `/`, `..` or a control character, + and must differ, ignoring case, from the name of every other active project. + The server trims leading and trailing whitespace from `name` and `description`. + """ + + model_config = ConfigDict(extra="forbid") + + name: str + description: str = "" + + +class ProjectUpdateRequest(BaseModel): + """ + Request body for `PATCH /api/projects/{id}/`. + + A field left as `None` is not sent, so the server keeps its current value. + A new `name` follows the rules of `ProjectCreateRequest`. + """ + + model_config = ConfigDict(extra="forbid") + + name: Optional[str] = None + description: Optional[str] = None + + +class ProjectDeleteConflictReason(str, Enum): + """Why a project's resources cannot move to the destination project when the project is deleted.""" + + name_in_use = "name_in_use" + """The destination project already contains a resource of the same type and name.""" + file_exists = "file_exists" + """The destination project already has a file or directory where the resource's files would move to.""" + unexpected_file = "unexpected_file" + """The project's directory contains a file or directory that belongs to none of its resources.""" + + +class ProjectDeleteConflict(BaseModel): + """A conflict that prevents a project's resources moving to the destination project when the project is deleted.""" + + model_config = ConfigDict(extra="allow") + + reason: ProjectDeleteConflictReason + detail: str + """A message that describes the conflict.""" + type: Optional[ResourceTypeName] = None + """For `name_in_use` and `file_exists`: the type of the resource that would move.""" + id: Optional[str] = None + """For `name_in_use` and `file_exists`: the ID of the resource that would move.""" + name: Optional[str] = None + """For `name_in_use` and `file_exists`: the name of the resource that would move.""" + path: Optional[str] = None + """For `file_exists` and `unexpected_file`: the path of the file or directory.""" diff --git a/datamasque/client/models/resource.py b/datamasque/client/models/resource.py new file mode 100644 index 0000000..4f50baa --- /dev/null +++ b/datamasque/client/models/resource.py @@ -0,0 +1,126 @@ +"""Resource listing models for the DataMasque API.""" + +from collections.abc import Mapping, Sequence +from datetime import datetime +from enum import Enum +from typing import Optional, Union + +from pydantic import BaseModel, ConfigDict + +from datamasque.client.models.discovery_config import DiscoveryConfigType +from datamasque.client.models.resource_type import ResourceTypeName +from datamasque.client.models.runs import MaskType + + +class Resource(BaseModel): + """ + One resource in the resources listing: the summary the listing gives, not the resource's full model. + + Connections, rulesets and discovery configs are listed as the subclasses that add the field each carries. + """ + + model_config = ConfigDict(extra="allow") + + id: str + name: str + project: Optional[str] = None + """Its project's `-` reference, or `None` for a resource in no project, such as a built-in seed.""" + created: Optional[datetime] = None + """`None` for a type that does not record it, currently connections.""" + modified: Optional[datetime] = None + """`None` for a type that does not record it, currently connections.""" + position: Optional[int] = None + """ + Its index, counting from 0, among every resource the listing matched, in the listing's ordering. + It changes with the filters and ordering, so it does not identify the resource. + Requires server version 3.26.21, and is `None` on an older server. + """ + + +class ResourceOrdering(str, Enum): + """ + A column `list_resources` sorts by, with the columns that break ties after it. + + Ties are broken by type, then name, then project name, then ID, leaving out the column sorted by. + Types sort in the order `ResourceTypeName` declares them. + Names and project names compare ignoring case. + """ + + type = "type" + name = "name" + project = "project" + """ + Sorts by project name. + A resource in no project sorts after every project, + or before every project when `descending` is set. + """ + modified = "modified" + """A resource with no `modified` time sorts as the oldest.""" + + +class ConnectionResource(Resource): + """A connection in the resources listing.""" + + mask_type: MaskType + + +class RulesetResource(Resource): + """A ruleset in the resources listing.""" + + mask_type: MaskType + + +class DiscoveryConfigResource(Resource): + """A discovery config in the resources listing.""" + + config_type: DiscoveryConfigType + + +# The model each type's rows are listed as; a type not named here is listed as `Resource`. +RESOURCE_MODEL_BY_TYPE_NAME: dict[str, type[Resource]] = { + ResourceTypeName.connections.value: ConnectionResource, + ResourceTypeName.rulesets.value: RulesetResource, + ResourceTypeName.discovery_configs.value: DiscoveryConfigResource, +} + +# Resources by type, each given as a listed `Resource` or its ID, in the shape `list_resources` returns. +ResourcesByType = Mapping[ResourceTypeName, Sequence[Union[Resource, str]]] + + +class ResourceMoveConflictReason(str, Enum): + """Why a resource cannot move to the destination project.""" + + already_in_destination = "already_in_destination" + """The resource is already in the destination project.""" + name_in_use = "name_in_use" + """The destination project already contains a resource of the same type and name.""" + name_in_selection = "name_in_selection" + """Another resource of the same type and name is moving too, from another project.""" + file_exists = "file_exists" + """The destination project already has a file or directory where the resource's files would move to.""" + + +class ResourceMoveConflict(BaseModel): + """A conflict that keeps a resource from moving to the destination project.""" + + model_config = ConfigDict(extra="allow") + + reason: ResourceMoveConflictReason + detail: str + """A message that describes the conflict.""" + type: ResourceTypeName + id: str + name: str + path: Optional[str] = None + """For `file_exists`: the path of the file or directory, relative to the DataMasque files directory.""" + + +class ResourceMoveResult(BaseModel): + """The outcome of `move_resources`: every resource asked for either moved or is listed in `conflicts`.""" + + model_config = ConfigDict(extra="allow") + + moved: int + """The number of resources that moved.""" + conflicts: list[ResourceMoveConflict] + """The resources that did not move, and why.""" diff --git a/datamasque/client/models/resource_type.py b/datamasque/client/models/resource_type.py new file mode 100644 index 0000000..edefcd3 --- /dev/null +++ b/datamasque/client/models/resource_type.py @@ -0,0 +1,19 @@ +"""The types of resource that belong to a project.""" + +from enum import Enum + + +class ResourceTypeName(str, Enum): + """A type of resource that belongs to a project, named as the resources listing groups it.""" + + connections = "connections" + connection_filesets = "connection_filesets" + oracle_wallets = "oracle_wallets" + snowflake_keys = "snowflake_keys" + table_references = "table_references" + rulesets = "rulesets" + ruleset_libraries = "ruleset_libraries" + discovery_configs = "discovery_configs" + discovery_config_libraries = "discovery_config_libraries" + ruleset_generation_configs = "ruleset_generation_configs" + seeds = "seeds" diff --git a/datamasque/client/models/rg_config.py b/datamasque/client/models/rg_config.py index 1403937..441306b 100644 --- a/datamasque/client/models/rg_config.py +++ b/datamasque/client/models/rg_config.py @@ -39,12 +39,16 @@ class RGConfig(BaseModel): model_config = ConfigDict(extra="allow", populate_by_name=True) name: str + project: Optional[str] = None + """ + The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, + or the project's ID as a string. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. + """ yaml: Optional[str] = Field(default=None, alias="config_yaml") # Server-populated read-only fields, excluded from request bodies. id: Optional[RGConfigId] = Field(default=None, exclude=True) - project: Optional[str] = Field(default=None, exclude=True) - """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True) validation_errors: list[ValidationErrorDetails] = Field(default_factory=list, exclude=True) created: Optional[datetime] = Field(default=None, exclude=True) diff --git a/datamasque/client/models/ruleset.py b/datamasque/client/models/ruleset.py index d14c2d4..a42be76 100644 --- a/datamasque/client/models/ruleset.py +++ b/datamasque/client/models/ruleset.py @@ -38,13 +38,17 @@ class Ruleset(GitTrackedEntity): """Represents a ruleset.""" name: str + project: Optional[str] = None + """ + The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, + or the project's ID as a string. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. + """ yaml: str = Field(default="", alias="config_yaml") ruleset_type: RulesetType = Field(default=RulesetType.database, alias="mask_type") # Server-populated read-only fields, excluded from request bodies. id: Optional[RulesetId] = Field(default=None, exclude=True) - project: Optional[str] = Field(default=None, exclude=True) - """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True) validation_errors: list[ValidationErrorDetails] = Field(default_factory=list, exclude=True) """Validation errors surfaced by the server; empty when valid.""" diff --git a/datamasque/client/models/ruleset_library.py b/datamasque/client/models/ruleset_library.py index 68929c2..dd54b20 100644 --- a/datamasque/client/models/ruleset_library.py +++ b/datamasque/client/models/ruleset_library.py @@ -1,9 +1,10 @@ from datetime import datetime from typing import NewType, Optional -from pydantic import Field +from pydantic import Field, model_validator from datamasque.client.models.git import GitTrackedEntity +from datamasque.client.models.project import drop_namespace from datamasque.client.models.status import ValidationErrorDetails, ValidationStatus RulesetLibraryId = NewType("RulesetLibraryId", str) @@ -13,13 +14,16 @@ class RulesetLibrary(GitTrackedEntity): """Represents a ruleset library.""" name: str - namespace: str = "" + project: Optional[str] = None + """ + The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, + or the project's ID as a string. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. + """ yaml: Optional[str] = Field(default=None, alias="config_yaml") # Server-populated read-only fields, excluded from request bodies. id: Optional[RulesetLibraryId] = Field(default=None, exclude=True) - project: Optional[str] = Field(default=None, exclude=True) - """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True) validation_errors: list[ValidationErrorDetails] = Field(default_factory=list, exclude=True) """Validation errors surfaced by the server; empty when valid.""" @@ -32,3 +36,5 @@ class RulesetLibrary(GitTrackedEntity): """ created: Optional[datetime] = Field(default=None, exclude=True) modified: Optional[datetime] = Field(default=None, exclude=True) + + _drop_namespace = model_validator(mode="before")(drop_namespace) diff --git a/datamasque/client/models/runs.py b/datamasque/client/models/runs.py index 09bae55..6c1bd3d 100644 --- a/datamasque/client/models/runs.py +++ b/datamasque/client/models/runs.py @@ -146,7 +146,8 @@ class RunInfo(BaseModel): """ Its project's `-` reference, e.g. `1-default`. - Taken from the ruleset's project when the run starts, and never changes after that. + The project of the run's ruleset, else its discovery config, else its source connection. + The run moves when that resource moves to another project. """ @model_validator(mode="before") diff --git a/datamasque/client/models/table_reference.py b/datamasque/client/models/table_reference.py index e03bb38..5b6c9b5 100644 --- a/datamasque/client/models/table_reference.py +++ b/datamasque/client/models/table_reference.py @@ -73,6 +73,12 @@ class TableReference(BaseModel): model_config = ConfigDict(extra="allow", populate_by_name=True) name: str + project: Optional[str] = None + """ + The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, + or the project's ID as a string. + Left as `None`, a create goes into the client's `default_project`, else the server's default project. + """ connection: Union[ConnectionId, ConnectionConfig] """The connection the data lives in. Accepts a `ConnectionId` or a `ConnectionConfig` (its `id` is extracted).""" source: str @@ -81,8 +87,6 @@ class TableReference(BaseModel): # Server-populated read-only fields, excluded from request bodies. id: Optional[TableReferenceId] = Field(default=None, exclude=True) - project: Optional[str] = Field(default=None, exclude=True) - """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" created: Optional[datetime] = Field(default=None, exclude=True) modified: Optional[datetime] = Field(default=None, exclude=True) diff --git a/datamasque/client/projects.py b/datamasque/client/projects.py new file mode 100644 index 0000000..ef64906 --- /dev/null +++ b/datamasque/client/projects.py @@ -0,0 +1,148 @@ +import logging +from typing import Iterator + +from requests import Response + +from datamasque.client.base import BaseClient +from datamasque.client.exceptions import ProjectDeleteConflictError +from datamasque.client.models.project import ( + Project, + ProjectCreateRequest, + ProjectDeleteConflict, + ProjectIdentifier, + ProjectUpdateRequest, + unwrap_project_id, +) + +logger = logging.getLogger(__name__) + + +def _get_conflicts(response: Response) -> list[ProjectDeleteConflict]: + return [ProjectDeleteConflict.model_validate(conflict) for conflict in response.json()["conflicts"]] + + +class ProjectClient(BaseClient): + """Project API methods. Mixed into `DataMasqueClient`.""" + + def iter_projects(self, *, include_archived: bool = False) -> Iterator[Project]: + """ + Lazily iterate projects via the paginated endpoint. + + Archived projects are skipped unless `include_archived` is set. + """ + + params = {"include_archived": "true"} if include_archived else None + return self._iter_paginated("/api/projects/", model=Project, params=params) + + def list_projects(self, *, include_archived: bool = False) -> list[Project]: + """ + Lists projects. + + Archived projects are left out unless `include_archived` is set. + """ + + return list(self.iter_projects(include_archived=include_archived)) + + def get_project(self, project: ProjectIdentifier) -> Project: + """ + Retrieves a single project, given as a `Project`, its `ProjectId` or its `-` form. + + Archived projects can be retrieved too. + """ + + response = self.make_request("GET", f"/api/projects/{unwrap_project_id(project)}/") + return Project.model_validate(response.json()) + + def create_project(self, request: ProjectCreateRequest) -> Project: + """ + Creates a new project on the server, and returns it as the server stored it. + + Raises: + DataMasqueApiError: the server rejects the name (see `ProjectCreateRequest`), or any other non-2xx response. + """ + + data = request.model_dump(mode="json") + response = self.make_request("POST", "/api/projects/", data=data) + project = Project.model_validate(response.json()) + logger.info('Creation of project "%s" successful', project.name) + return project + + def update_project(self, project: ProjectIdentifier, request: ProjectUpdateRequest) -> Project: + """ + Renames a project or changes its description, and returns the updated project. + + Only admin users can, and API tokens cannot. + Renaming invalidates rulesets, discovery configs and generator configs that import a library of the project + by the old project name. Imports without a project name stay valid. + It also moves the project's seed and credential files, keeping connections working. + Seed and table references in rulesets that name the old project are not rewritten. + + Raises: + DataMasqueApiError: the server rejects the name (see `ProjectCreateRequest`), + a file already exists where one of the project's files would move, + the project does not exist or is deleted, or any other non-2xx response. + """ + + project_id = unwrap_project_id(project) + data = request.model_dump(mode="json", exclude_none=True) + response = self.make_request("PATCH", f"/api/projects/{project_id}/", data=data) + updated_project = Project.model_validate(response.json()) + logger.info("Update of project %s successful", project_id) + return updated_project + + def get_project_delete_conflicts( + self, + project: ProjectIdentifier, + destination: ProjectIdentifier, + ) -> list[ProjectDeleteConflict]: + """ + Returns the conflicts that would prevent `delete_project` moving the project's resources to the destination. + + An empty list means `delete_project` can delete the project with this destination, + unless the resources in either project change first. + + Raises: + DataMasqueApiError: the destination does not exist, is deleted, or is the project; + the project does not exist or is already deleted; or any other non-2xx response. + """ + + response = self.make_request( + "GET", + f"/api/projects/{unwrap_project_id(project)}/delete-check/", + params={"destination": unwrap_project_id(destination)}, + ) + return _get_conflicts(response) + + def delete_project( + self, + project: ProjectIdentifier, + destination: ProjectIdentifier, + ) -> None: + """ + Deletes a project, after moving every resource in it to the destination project. + + When the resources cannot move, the server changes nothing. + `get_project_delete_conflicts` lists the same conflicts without trying to delete the project. + + Raises: + ProjectDeleteConflictError: the resources cannot move to the destination; its `conflicts` says why. + DataMasqueApiError: the destination does not exist, is deleted, or is the project; + the project does not exist or is already deleted; or any other non-2xx response. + """ + + project_id = unwrap_project_id(project) + response = self.make_request( + "DELETE", + f"/api/projects/{project_id}/", + params={"destination": unwrap_project_id(destination)}, + require_status_check=False, + ) + if response.status_code == 409: + raise ProjectDeleteConflictError( + str(response.json()["detail"]), + response=response, + conflicts=_get_conflicts(response), + ) + + self._raise_for_status(response) + logger.info("Deletion of project %s successful", project_id) diff --git a/datamasque/client/resources.py b/datamasque/client/resources.py new file mode 100644 index 0000000..380b61c --- /dev/null +++ b/datamasque/client/resources.py @@ -0,0 +1,141 @@ +import logging +from collections import defaultdict +from collections.abc import Sequence +from typing import Optional + +from datamasque.client.base import BaseClient +from datamasque.client.models.project import ProjectIdentifier, unwrap_project_id +from datamasque.client.models.resource import ( + RESOURCE_MODEL_BY_TYPE_NAME, + Resource, + ResourceMoveConflict, + ResourceMoveResult, + ResourceOrdering, + ResourcesByType, +) +from datamasque.client.models.resource_type import ResourceTypeName + +logger = logging.getLogger(__name__) + + +def _move_request(resources: ResourcesByType, destination: ProjectIdentifier) -> dict[str, object]: + keys = [ + {"type": type_name.value, "id": resource.id if isinstance(resource, Resource) else resource} + for type_name, group in resources.items() + for resource in group + ] + if not keys: + raise ValueError("Specify at least one resource to move") + + # The API takes the destination as a string: an id or `-`. + return {"destination": str(unwrap_project_id(destination)), "resources": keys} + + +class ResourceClient(BaseClient): + """Resource listing and move API methods. Mixed into `DataMasqueClient`.""" + + def list_resources( + self, + *, + projects: Optional[Sequence[ProjectIdentifier]] = None, + types: Optional[Sequence[ResourceTypeName]] = None, + search: Optional[str] = None, + ordering: ResourceOrdering = ResourceOrdering.type, + descending: bool = False, + page_size: int = 500, + ) -> dict[ResourceTypeName, list[Resource]]: + """ + Returns the resources the caller can manage, grouped by type, fetched in pages of `page_size`. + + `projects` limits them to those projects, each given as a `Project`, its `ProjectId` or its `-` form; + `types` to those types; and `search` to resources whose name contains every word of it, ignoring case. + Each group's resources come in `ordering`, descending when `descending` is set. + Columns that break ties are always sorted ascending. + Sort the resources of every group together by `position` to get the whole listing in order. + `ordering` and `descending` require server version 3.26.21. + An older server ignores them, and each resource's `position` is `None`. + The groups are the requested types, or every type the server has when `types` is not given. + An empty `projects` or `types` matches no resources, without a request to the server. + Connections, rulesets and discovery configs are listed as `ConnectionResource`, `RulesetResource` + and `DiscoveryConfigResource`. + + Raises: + DataMasqueApiError: the server rejects the request, for example because a project does not exist. + """ + + # `requests` leaves out a parameter whose value is an empty list, which would remove the filter. + if projects is not None and not projects: + return {type_name: [] for type_name in types or list(ResourceTypeName)} + + if types is not None and not types: + return {} + + params: dict[str, object] = {} + if projects is not None: + params["project"] = [unwrap_project_id(project) for project in projects] + if types is not None: + params["type"] = [type_name.value for type_name in types] + if search is not None: + params["search"] = search + params["ordering"] = f"-{ordering.value}" if descending else ordering.value + + resources: defaultdict[ResourceTypeName, list[Resource]] = defaultdict(list) + for results in self._iter_pages("/api/resources/", params=params, page_size=page_size): + for type_name, rows in results.items(): + model = RESOURCE_MODEL_BY_TYPE_NAME.get(type_name, Resource) + resources[ResourceTypeName(type_name)].extend(model.model_validate(row) for row in rows) + + return dict(resources) + + def get_resource_move_conflicts( + self, + resources: ResourcesByType, + destination: ProjectIdentifier, + ) -> list[ResourceMoveConflict]: + """ + Checks whether `resources` can move to the destination project, without moving them. + + `resources` can contain up to 2,000 resources. + + Returns: + The conflicts that would keep any of `resources` from moving (if any). + + Raises: + ValueError: `resources` is empty. + DataMasqueApiError: the server rejects the request, for example because the destination does not exist + or `resources` contains more than 2,000 resources. + """ + + response = self.make_request("POST", "/api/resources/move-check/", data=_move_request(resources, destination)) + return [ResourceMoveConflict.model_validate(conflict) for conflict in response.json()["conflicts"]] + + def move_resources( + self, + resources: ResourcesByType, + destination: ProjectIdentifier, + ) -> ResourceMoveResult: + """ + Moves `resources` to the destination project, with their files and runs. + + `resources` can be the result of `list_resources`, or any part of it, and can contain up to 2,000 resources. + A resource that conflicts with the destination, or with another resource in `resources`, + stays where it is and is listed in the result's `conflicts`; every other resource moves. + Rulesets, discovery configs and RG configs whose library imports resolve differently after the move + are marked invalid. + + Raises: + ValueError: `resources` is empty. + DataMasqueApiError: the server rejects the move, for example because the destination does not exist, + `resources` contains more than 2,000 resources, + or the caller does not have permission to move one of the resource types. + """ + + response = self.make_request("POST", "/api/resources/move/", data=_move_request(resources, destination)) + result = ResourceMoveResult.model_validate(response.json()) + logger.info( + "Moved %d resources to project %s; %d could not move", + result.moved, + unwrap_project_id(destination), + len(result.conflicts), + ) + return result diff --git a/datamasque/client/rg_configs.py b/datamasque/client/rg_configs.py index adb6066..cf5237d 100644 --- a/datamasque/client/rg_configs.py +++ b/datamasque/client/rg_configs.py @@ -1,9 +1,10 @@ import logging -from typing import Iterator, Optional +from typing import Iterator, Optional, Union from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.pagination import Page +from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id from datamasque.client.models.rg_config import RGConfig, RGConfigId logger = logging.getLogger(__name__) @@ -33,13 +34,13 @@ def get_rg_config(self, config_id: RGConfigId) -> RGConfig: response = self.make_request("GET", f"/api/ruleset-generation-configs/{config_id}/") return RGConfig.model_validate(response.json()) - def _get_rg_config_id_by_name(self, name: str) -> Optional[RGConfigId]: - """Return the id of the config matching the name via a single list request, or `None`.""" + def _get_rg_config_id_by_name(self, name: str, project: Union[ProjectId, str]) -> Optional[RGConfigId]: + """Return the id of the config matching the name and project via a single list request, or `None`.""" response = self.make_request( "GET", "/api/ruleset-generation-configs/", - params={"name_exact": name, "limit": 1}, + params={"name_exact": name, "project": project, "limit": 1}, ) page = Page[RGConfig].model_validate(response.json()) if not page.results: @@ -54,15 +55,16 @@ def _get_rg_config_id_by_name(self, name: str) -> Optional[RGConfigId]: return config_id - def get_rg_config_by_name(self, name: str) -> Optional[RGConfig]: + def get_rg_config_by_name(self, name: str, project: ProjectIdentifier) -> Optional[RGConfig]: """ - Looks for an RG config matching the given name (case-sensitive, exact match). + Looks for an RG config matching the given name (case-sensitive, exact match) in the given project. - RG configs are untyped and their names are unique, so the name alone identifies a single config. + RG configs are untyped and their names are unique per project. + `project` is a `Project`, its `ProjectId` or its `-` form. Returns it if found, otherwise `None`. """ - config_id = self._get_rg_config_id_by_name(name) + config_id = self._get_rg_config_id_by_name(name, unwrap_project_id(project)) if config_id is None: return None @@ -76,10 +78,13 @@ def create_rg_config(self, config: RGConfig) -> RGConfig: (`id`, `is_valid`, `validation_errors`, `created`, `modified`) and returns the config. """ - data = config.model_dump(exclude_none=True, by_alias=True, mode="json") + data = config.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( + config.project + ) response = self.make_request("POST", "/api/ruleset-generation-configs/", data=data) created = RGConfig.model_validate(response.json()) config.id = created.id + config.project = created.project config.is_valid = created.is_valid config.validation_errors = created.validation_errors config.created = created.created @@ -116,12 +121,14 @@ def update_rg_config(self, config: RGConfig) -> RGConfig: def create_or_update_rg_config(self, config: RGConfig) -> RGConfig: """ - Creates the config if it doesn't exist, or updates it if one with the same name already exists. + Creates the config, or updates the existing one with the same name in the same project. + With no `project` set, the client's `default_project` is used, else the server's only active project, + and `DataMasqueException` is raised when it has no active project or more than one. Sets the config's `id` property. """ - existing_id = self._get_rg_config_id_by_name(config.name) + existing_id = self._get_rg_config_id_by_name(config.name, self._resolve_project(config.project)) if existing_id is not None: config.id = existing_id return self.update_rg_config(config) @@ -137,14 +144,15 @@ def delete_rg_config_by_id_if_exists(self, config_id: RGConfigId) -> None: self._delete_if_exists(f"/api/ruleset-generation-configs/{config_id}/") - def delete_rg_config_by_name_if_exists(self, name: str) -> None: + def delete_rg_config_by_name_if_exists(self, name: str, project: ProjectIdentifier) -> None: """ - Deletes the RG config with the given name. + Deletes the RG config with the given name in the given project. + `project` is a `Project`, its `ProjectId` or its `-` form. No-op if no such config exists. """ - config_id = self._get_rg_config_id_by_name(name) + config_id = self._get_rg_config_id_by_name(name, unwrap_project_id(project)) if config_id is not None: self.delete_rg_config_by_id_if_exists(config_id) diff --git a/datamasque/client/ruleset_libraries.py b/datamasque/client/ruleset_libraries.py index ce5fddb..20003c3 100644 --- a/datamasque/client/ruleset_libraries.py +++ b/datamasque/client/ruleset_libraries.py @@ -1,9 +1,10 @@ import logging -from typing import Iterator, Optional +from typing import Iterator, Optional, Union from datamasque.client.base import BaseClient -from datamasque.client.exceptions import DataMasqueApiError, DataMasqueException +from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.pagination import Page +from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id from datamasque.client.models.ruleset import Ruleset from datamasque.client.models.ruleset_library import RulesetLibrary, RulesetLibraryId @@ -34,17 +35,13 @@ def get_ruleset_library(self, library_id: RulesetLibraryId) -> RulesetLibrary: response = self.make_request("GET", f"/api/ruleset-libraries/{library_id}/") return RulesetLibrary.model_validate(response.json()) - def get_ruleset_library_by_name(self, name: str, namespace: str = "") -> Optional[RulesetLibrary]: - """ - Looks for a ruleset library matching the given name and namespace (case-sensitive, exact match). - - Returns it (with full YAML content) if found, otherwise None. - """ + def _get_ruleset_library_id_by_name(self, name: str, project: Union[ProjectId, str]) -> Optional[RulesetLibraryId]: + """Return the ID of the library with the given name in the given project, or `None` if there is none.""" response = self.make_request( "GET", "/api/ruleset-libraries/", - params={"name_exact": name, "namespace_exact": namespace, "limit": 1}, + params={"name_exact": name, "project": project, "limit": 1}, ) page = Page[RulesetLibrary].model_validate(response.json()) if not page.results: @@ -57,6 +54,20 @@ def get_ruleset_library_by_name(self, name: str, namespace: str = "") -> Optiona response=response, ) + return library_id + + def get_ruleset_library_by_name(self, name: str, project: ProjectIdentifier) -> Optional[RulesetLibrary]: + """ + Looks for a ruleset library matching the given name (case-sensitive, exact match) in the given project. + + `project` is a `Project`, its `ProjectId` or its `-` form. + Returns it (with full YAML content) if found, otherwise None. + """ + + library_id = self._get_ruleset_library_id_by_name(name, unwrap_project_id(project)) + if library_id is None: + return None + return self.get_ruleset_library(library_id) def create_ruleset_library(self, library: RulesetLibrary) -> RulesetLibrary: @@ -68,10 +79,13 @@ def create_ruleset_library(self, library: RulesetLibrary) -> RulesetLibrary: and returns the library. """ - data = library.model_dump(exclude_none=True, by_alias=True, mode="json") + data = library.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( + library.project + ) response = self.make_request("POST", "/api/ruleset-libraries/", data=data) created_library = RulesetLibrary.model_validate(response.json()) library.id = created_library.id + library.project = created_library.project library.is_valid = created_library.is_valid library.validation_errors = created_library.validation_errors library.unlicensed_feature_warnings = created_library.unlicensed_feature_warnings @@ -104,14 +118,16 @@ def update_ruleset_library(self, library: RulesetLibrary) -> RulesetLibrary: def create_or_update_ruleset_library(self, library: RulesetLibrary) -> RulesetLibrary: """ - Creates the library if it doesn't exist, or updates it if a library with the same name already exists. + Creates the library, or updates the existing one with the same name in the same project. + With no `project` set, the client's `default_project` is used, else the server's only active project, + and `DataMasqueException` is raised when it has no active project or more than one. Sets the library's `id` property. """ - existing = self.get_ruleset_library_by_name(library.name, library.namespace) - if existing is not None: - library.id = existing.id + library_id = self._get_ruleset_library_id_by_name(library.name, self._resolve_project(library.project)) + if library_id is not None: + library.id = library_id return self.update_ruleset_library(library) return self.create_ruleset_library(library) @@ -131,21 +147,18 @@ def delete_ruleset_library_by_id_if_exists(self, library_id: RulesetLibraryId, * self._delete_if_exists(f"/api/ruleset-libraries/{library_id}/", params=params) def delete_ruleset_library_by_name_if_exists( - self, library_name: str, namespace: str = "", *, force: bool = False + self, library_name: str, project: ProjectIdentifier, *, force: bool = False ) -> None: """ - Deletes the ruleset library with the given name and namespace. + Deletes the ruleset library with the given name in the given project. + `project` is a `Project`, its `ProjectId` or its `-` form. No-op if the library does not exist. """ - all_libraries = self.list_ruleset_libraries() - matching = [lib for lib in all_libraries if lib.name == library_name and lib.namespace == namespace] - for lib in matching: - if lib.id is None: - raise DataMasqueException(f'Server returned a ruleset library named "{lib.name}" without an `id`.') - - self.delete_ruleset_library_by_id_if_exists(lib.id, force=force) + library_id = self._get_ruleset_library_id_by_name(library_name, unwrap_project_id(project)) + if library_id is not None: + self.delete_ruleset_library_by_id_if_exists(library_id, force=force) def iter_rulesets_using_library(self, library_id: RulesetLibraryId) -> Iterator[Ruleset]: """Lazily iterate rulesets that import the given library.""" diff --git a/datamasque/client/rulesets.py b/datamasque/client/rulesets.py index 96fc98c..5337c93 100644 --- a/datamasque/client/rulesets.py +++ b/datamasque/client/rulesets.py @@ -2,6 +2,7 @@ from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueException +from datamasque.client.models.project import ProjectIdentifier, unwrap_project_id from datamasque.client.models.ruleset import Ruleset, RulesetId, RulesetType logger = logging.getLogger(__name__) @@ -18,16 +19,23 @@ def list_rulesets(self) -> list[Ruleset]: def create_or_update_ruleset(self, ruleset: Ruleset) -> Ruleset: """ - Creates or updates a ruleset. + Creates the ruleset, or updates the existing one with the same name and type in the same project. + With no `project` set, the client's `default_project` is used, else the server's only active project, + and `DataMasqueException` is raised when it has no active project or more than one. Populates the given ruleset's `id`, `is_valid`, `validation_errors`, `unlicensed_feature_warnings`, and `git` fields from the server response, and returns the same ruleset instance for convenience. """ - data = ruleset.model_dump(exclude_none=True, by_alias=True, mode="json") + # The server finds the existing ruleset itself, so this only checks that an unset project resolves. + self._resolve_project(ruleset.project) + data = ruleset.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( + ruleset.project + ) response = self.make_request("POST", "/api/rulesets/", data=data, params={"upsert": "true"}) created = Ruleset.model_validate(response.json()) ruleset.id = created.id + ruleset.project = created.project ruleset.is_valid = created.is_valid ruleset.validation_errors = created.validation_errors ruleset.unlicensed_feature_warnings = created.unlicensed_feature_warnings @@ -45,20 +53,23 @@ def delete_ruleset_by_id_if_exists(self, ruleset_id: RulesetId) -> None: self._delete_if_exists(f"/api/rulesets/{ruleset_id}/") - def delete_ruleset_by_name_if_exists(self, ruleset_name: str, ruleset_type: RulesetType) -> None: + def delete_ruleset_by_name_if_exists( + self, ruleset_name: str, ruleset_type: RulesetType, project: ProjectIdentifier + ) -> None: """ - Deletes the ruleset with the given name and type. + Deletes the ruleset with the given name and type in the given project. - Ruleset names are unique per type, so a type is required to identify a single ruleset. + Ruleset names are unique per type and project, so both are required to identify a single ruleset. + `project` is a `Project`, its `ProjectId` or its `-` form. No-op if no such ruleset exists. """ - matching = [ - ruleset - for ruleset in self.list_rulesets() - if ruleset.name == ruleset_name and ruleset.ruleset_type is ruleset_type - ] - for ruleset in matching: + response = self.make_request( + "GET", + "/api/v2/rulesets/", + params={"name_exact": ruleset_name, "mask_type": ruleset_type.value, "project": unwrap_project_id(project)}, + ) + for ruleset in (Ruleset.model_validate(payload) for payload in response.json()): if ruleset.id is None: raise DataMasqueException(f'Server returned a ruleset named "{ruleset.name}" without an `id`.') diff --git a/datamasque/client/runs.py b/datamasque/client/runs.py index 9f4c87c..3668731 100644 --- a/datamasque/client/runs.py +++ b/datamasque/client/runs.py @@ -4,11 +4,13 @@ from datamasque.client.base import BaseClient from datamasque.client.exceptions import ( + DataMasqueException, FailedToStartError, InvalidLibraryError, InvalidRulesetError, RunNotCancellableError, ) +from datamasque.client.models.connection import ConnectionId from datamasque.client.models.runs import MaskingRunRequest, RunId, RunInfo, UnfinishedRun from datamasque.client.models.status import MaskingRunStatus @@ -63,10 +65,10 @@ def get_db_discovery_result_report(self, run_id: RunId, include_selection_column return response.content return response.text - def get_unfinished_runs(self) -> dict[str, UnfinishedRun]: - """Queries the DM instance for unfinished runs, and returns them organised by connection name.""" + def get_unfinished_runs(self) -> dict[ConnectionId, UnfinishedRun]: + """Queries the DM instance for unfinished runs, and returns them organised by connection ID.""" - unfinished_runs = {} + unfinished_runs: dict[ConnectionId, UnfinishedRun] = {} for status in ( MaskingRunStatus.queued, MaskingRunStatus.running, @@ -89,9 +91,22 @@ def get_unfinished_runs(self) -> dict[str, UnfinishedRun]: for run in data.get("results", []): unfinished_run = UnfinishedRun.model_validate(run) - unfinished_runs[unfinished_run.source_connection.name] = unfinished_run - if unfinished_run.destination_connection is not None: - unfinished_runs[unfinished_run.destination_connection.name] = unfinished_run + source = unfinished_run.source_connection + if source.id is None: + raise DataMasqueException( + f"Server returned unfinished run {unfinished_run.id} without a source connection ID." + ) + + unfinished_runs[source.id] = unfinished_run + + destination = unfinished_run.destination_connection + if destination is not None: + if destination.id is None: + raise DataMasqueException( + f"Server returned unfinished run {unfinished_run.id} without a destination connection ID." + ) + + unfinished_runs[destination.id] = unfinished_run return unfinished_runs diff --git a/datamasque/client/table_references.py b/datamasque/client/table_references.py index f191abf..04ec955 100644 --- a/datamasque/client/table_references.py +++ b/datamasque/client/table_references.py @@ -1,8 +1,9 @@ import logging -from typing import Optional +from typing import Optional, Union from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueApiError +from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id from datamasque.client.models.table_reference import TableReference, TableReferenceId logger = logging.getLogger(__name__) @@ -11,16 +12,19 @@ class TableReferenceClient(BaseClient): """Table reference CRUD API methods. Mixed into `DataMasqueClient`.""" - def list_table_references(self) -> list[TableReference]: + def list_table_references(self, project: Optional[ProjectIdentifier] = None) -> list[TableReference]: """ - Lists all table references. + Lists all table references, or only those in `project` when it is given. + + `project` is a `Project`, its `ProjectId` or its `-` form. Unlike most list endpoints, this one is not paginated; the server returns every table reference in a single response. Deleted table references are archived rather than removed, and are not listed. """ - response = self.make_request("GET", "/api/table-references/") + params = {"project": unwrap_project_id(project)} if project is not None else None + response = self.make_request("GET", "/api/table-references/", params=params) return [TableReference.model_validate(item) for item in response.json()] def get_table_reference(self, table_reference_id: TableReferenceId) -> TableReference: @@ -29,26 +33,27 @@ def get_table_reference(self, table_reference_id: TableReferenceId) -> TableRefe response = self.make_request("GET", f"/api/table-references/{table_reference_id}/") return TableReference.model_validate(response.json()) - def get_table_reference_by_name(self, name: str) -> Optional[TableReference]: + def get_table_reference_by_name(self, name: str, project: ProjectIdentifier) -> Optional[TableReference]: """ - Looks for a table reference with the given name (case-sensitive, exact match). + Looks for a table reference with the given name (case-sensitive, exact match) in the given project. - Names are unique across live table references, so at most one can match. + Names are unique across a project's live table references, so at most one can match. + `project` is a `Project`, its `ProjectId` or its `-` form. Returns it if found, otherwise `None`. - The endpoint takes no name filter, so the match is made client-side over the full listing. + The endpoint takes no name filter, so the match is made client-side over the project's listing. """ - matches = [reference for reference in self.list_table_references() if reference.name == name] + matches = [reference for reference in self.list_table_references(project) if reference.name == name] if not matches: return None return matches[0] - def _get_table_reference_id_by_name(self, name: str) -> Optional[TableReferenceId]: - """Return the ID of the table reference with the given name, or `None` if there is none.""" + def _get_table_reference_id_by_name(self, name: str, project: Union[ProjectId, str]) -> Optional[TableReferenceId]: + """Return the ID of the table reference with the given name in the given project, or `None` if there is none.""" - response = self.make_request("GET", "/api/table-references/") + response = self.make_request("GET", "/api/table-references/", params={"project": project}) matches = [item for item in response.json() if item.get("name") == name] if not matches: return None @@ -74,9 +79,13 @@ def create_table_reference(self, table_reference: TableReference) -> TableRefere data = table_reference.model_dump(by_alias=True, mode="json") if data.get("options") is None: data.pop("options", None) + if data.get("project") is None: + data.pop("project", None) + data |= self._default_project_field(table_reference.project) response = self.make_request("POST", "/api/table-references/", data=data) created = TableReference.model_validate(response.json()) table_reference.id = created.id + table_reference.project = created.project table_reference.options = created.options table_reference.created = created.created table_reference.modified = created.modified @@ -101,6 +110,8 @@ def update_table_reference(self, table_reference: TableReference) -> TableRefere data = table_reference.model_dump(by_alias=True, mode="json") if data.get("options") is None: data.pop("options", None) + if data.get("project") is None: + data.pop("project", None) response = self.make_request("PUT", f"/api/table-references/{table_reference.id}/", data=data) updated = TableReference.model_validate(response.json()) table_reference.options = updated.options @@ -110,12 +121,16 @@ def update_table_reference(self, table_reference: TableReference) -> TableRefere def create_or_update_table_reference(self, table_reference: TableReference) -> TableReference: """ - Creates the table reference, or updates the existing one with the same name. + Creates the table reference, or updates the existing one with the same name in the same project. + With no `project` set, the client's `default_project` is used, else the server's only active project, + and `DataMasqueException` is raised when it has no active project or more than one. Sets the table reference's `id` property. """ - existing_id = self._get_table_reference_id_by_name(table_reference.name) + existing_id = self._get_table_reference_id_by_name( + table_reference.name, self._resolve_project(table_reference.project) + ) if existing_id is not None: table_reference.id = existing_id return self.update_table_reference(table_reference) @@ -132,13 +147,14 @@ def delete_table_reference_by_id_if_exists(self, table_reference_id: TableRefere self._delete_if_exists(f"/api/table-references/{table_reference_id}/") - def delete_table_reference_by_name_if_exists(self, name: str) -> None: + def delete_table_reference_by_name_if_exists(self, name: str, project: ProjectIdentifier) -> None: """ - Deletes the table reference with the given name. + Deletes the table reference with the given name in the given project. + `project` is a `Project`, its `ProjectId` or its `-` form. No-op if no such table reference exists. """ - table_reference_id = self._get_table_reference_id_by_name(name) + table_reference_id = self._get_table_reference_id_by_name(name, unwrap_project_id(project)) if table_reference_id is not None: self.delete_table_reference_by_id_if_exists(table_reference_id) diff --git a/docs/client.models.rst b/docs/client.models.rst index e98789b..54c271f 100644 --- a/docs/client.models.rst +++ b/docs/client.models.rst @@ -4,6 +4,14 @@ datamasque.client.models package Submodules ---------- +datamasque.client.models.bundle module +-------------------------------------- + +.. automodule:: datamasque.client.models.bundle + :members: + :undoc-members: + :show-inheritance: + datamasque.client.models.connection module ------------------------------------------ @@ -44,6 +52,13 @@ datamasque.client.models.files module :undoc-members: :show-inheritance: +datamasque.client.models.git module +----------------------------------- + +.. automodule:: datamasque.client.models.git + :members: GitSnapshot, GitFile, GitFileCommit + :show-inheritance: + datamasque.client.models.ifm module ----------------------------------- @@ -60,6 +75,30 @@ datamasque.client.models.license module :undoc-members: :show-inheritance: +datamasque.client.models.project module +--------------------------------------- + +.. automodule:: datamasque.client.models.project + :members: + :undoc-members: + :show-inheritance: + +datamasque.client.models.resource module +---------------------------------------- + +.. automodule:: datamasque.client.models.resource + :members: + :undoc-members: + :show-inheritance: + +datamasque.client.models.resource_type module +--------------------------------------------- + +.. automodule:: datamasque.client.models.resource_type + :members: + :undoc-members: + :show-inheritance: + datamasque.client.models.rg_config module ----------------------------------------- diff --git a/docs/client.rst b/docs/client.rst index f013c64..fabcb5f 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -20,6 +20,14 @@ datamasque.client.base module :undoc-members: :show-inheritance: +datamasque.client.bundles module +-------------------------------- + +.. automodule:: datamasque.client.bundles + :members: + :undoc-members: + :show-inheritance: + datamasque.client.connections module ------------------------------------ @@ -60,6 +68,14 @@ datamasque.client.files module :undoc-members: :show-inheritance: +datamasque.client.git module +---------------------------- + +.. automodule:: datamasque.client.git + :members: + :undoc-members: + :show-inheritance: + datamasque.client.spcs module ----------------------------- @@ -84,6 +100,22 @@ datamasque.client.license module :undoc-members: :show-inheritance: +datamasque.client.projects module +--------------------------------- + +.. automodule:: datamasque.client.projects + :members: + :undoc-members: + :show-inheritance: + +datamasque.client.resources module +---------------------------------- + +.. automodule:: datamasque.client.resources + :members: + :undoc-members: + :show-inheritance: + datamasque.client.rg_configs module ----------------------------------- diff --git a/docs/conf.py b/docs/conf.py index 3ef2bd2..db0aa2b 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -47,10 +47,11 @@ # Python identifiers that DO resolve still get cross-linked. suppress_warnings = ["ref.any"] -# Exclude pydantic's `model_config` attribute from autodoc output — -# it's plumbing, not public API, and clutters every model's docs page. +# Exclude pydantic's `model_config` attribute and the `drop_namespace` validator from autodoc output. +# Both are plumbing, not public API, and `model_config` would clutter every model's docs page. +# Documenting `drop_namespace` would also fail the -W build: its `type` annotation matches every `.type` attribute. autodoc_default_options = { - "exclude-members": "model_config", + "exclude-members": "model_config,drop_namespace", } # Add any paths that contain templates here, relative to this directory. diff --git a/pyproject.toml b/pyproject.toml index b3183ec..7fced60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "datamasque-python" -version = "1.5.0" +version = "1.7.0" description = "Official Python client for the DataMasque data-masking API." authors = [ { name = "DataMasque Ltd" }, diff --git a/tests/conftest.py b/tests/conftest.py index 317778a..1052be1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,7 @@ from datamasque.client import DataMasqueClient from datamasque.client.models.dm_instance import DataMasqueInstanceConfig from datamasque.client.models.ruleset import Ruleset, RulesetType -from tests.helpers import database_connection_config, file_connection_config +from tests.helpers import PROJECT, database_connection_config, file_connection_config @pytest.fixture @@ -36,6 +36,7 @@ def existing_connection_json(): return { "id": "1", "name": "an_existing_connection", + "project": PROJECT, "mask_type": "database", "db_type": "mysql", "host": "my-host", @@ -51,6 +52,7 @@ def existing_rulesets_json(): { "id": "1", "name": "db_masking_ruleset", + "project": PROJECT, "mask_type": "database", "config_yaml": "version: '1.0'", "is_valid": "valid", @@ -58,6 +60,7 @@ def existing_rulesets_json(): { "id": "2", "name": "file_masking_ruleset", + "project": PROJECT, "mask_type": "file", "config_yaml": "version: '1.0'", "is_valid": "invalid", @@ -69,6 +72,7 @@ def existing_rulesets_json(): def ruleset(): return Ruleset( name="test_ruleset", + project=PROJECT, yaml="version: '1.0'\ntasks: []", ruleset_type=RulesetType.database, ) diff --git a/tests/helpers.py b/tests/helpers.py index 2303cfc..5c21f4e 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -9,9 +9,41 @@ S3ConnectionConfig, SnowflakeConnectionConfig, ) +from datamasque.client.models.project import Project, ProjectId fake = Faker() +PROJECT = "1-default" +"""The project reference the tests put resources in.""" + +PROJECT_ID = ProjectId(1) +"""The ID of `PROJECT`, as the client methods take it.""" + +OTHER_PROJECT = Project.model_validate( + { + "id": 2, + "reference": "2-banking", + "name": "Banking", + "description": "", + "created": "2026-10-01T10:00:00Z", + "updated": "2026-10-01T10:00:00Z", + } +) +"""A second project, for tests that pass a `Project` to a client method.""" + + +def active_projects_page(count: int) -> dict[str, object]: + """Builds the `GET /api/projects/?limit=1` response of a server with `count` active projects, `PROJECT` first.""" + project = { + "id": PROJECT_ID, + "reference": PROJECT, + "name": "Default", + "description": "", + "created": "2026-10-01T10:00:00Z", + "updated": "2026-10-01T10:00:00Z", + } + return {"count": count, "next": None, "previous": None, "results": [project] if count else []} + def parse_multipart_form(request) -> dict: # noqa: C901 """ @@ -69,6 +101,7 @@ def parse_multipart_form(request) -> dict: # noqa: C901 def database_connection_config(): return DatabaseConnectionConfig( + project=PROJECT, name=fake.word(), user=fake.user_name(), password=fake.password(), @@ -82,6 +115,7 @@ def database_connection_config(): def file_connection_config(): return S3ConnectionConfig( + project=PROJECT, name=fake.word(), base_directory=fake.uri_path(), bucket=fake.uri_page(), @@ -94,6 +128,7 @@ def sample_mounted_share_connection_json(*, id, name): return { "name": name, "id": id, + "project": PROJECT, "mask_type": "file", "type": "mounted_share_connection", "base_directory": "", @@ -118,6 +153,7 @@ def make_ok_response() -> Response: def snowflake_connection_config(): return SnowflakeConnectionConfig( + project=PROJECT, name="snowflake", database="test_db", user="snowflake_user", diff --git a/tests/test_bundles.py b/tests/test_bundles.py new file mode 100644 index 0000000..3939511 --- /dev/null +++ b/tests/test_bundles.py @@ -0,0 +1,238 @@ +"""Tests for exporting and importing bundles with the DataMasque client.""" + +import json +from io import BytesIO +from pathlib import Path + +import pytest +import requests_mock + +from datamasque.client import ( + BundleImportStatus, + DataMasqueApiError, + DataMasqueClient, + Ruleset, + RulesetId, + RulesetType, +) +from tests.helpers import OTHER_PROJECT, parse_multipart_form + +BUNDLE = b"PK\x03\x04 bundle" + +V2_IMPORT_RESPONSE = { + "rulesets": { + "metadata": {"processed": 1, "created": 1, "restored": 0, "error": 0}, + "data": [ + { + "exported_name": "Customers", + "exported_project": "payments", + "exported_id": "9d641e97-adf7-4f22-9089-afc3711bf222", + "imported_name": "Customers", + "imported_id": "9d641e97-adf7-4f22-9089-afc3711bf222", + "imported_project": "payments", + "ruleset_type": "database", + "status": "CREATED", + "message": 'Ruleset named "Customers" did not exist, and was created.', + } + ], + }, + "libraries": { + "metadata": {"processed": 1, "created": 0, "restored": 0, "error": 0, "updated": 0, "skipped": 1}, + "data": [ + { + "exported_name": "pii_masks", + "exported_namespace": "", + "exported_project": "payments", + "exported_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "imported_name": "pii_masks", + "imported_namespace": "payments", + "imported_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "imported_project": "payments", + "status": "SKIPPED", + "message": 'A library with ID "a1b2c3d4-e5f6-7890-abcd-ef1234567890" already exists.', + } + ], + }, +} + +V1_IMPORT_RESPONSE = { + "rulesets": { + "metadata": {"processed": 1, "created": 1, "restored": 0, "error": 0}, + "data": [ + { + "exported_name": "Customers", + "exported_project": None, + "exported_id": None, + "imported_name": "Customers", + "imported_id": "04ea20f0-ad4c-498e-881f-b0bc79d83ba7", + "imported_project": "Banking", + "ruleset_type": "file", + "status": "CREATED", + "message": 'Ruleset named "Customers" did not exist, and was created.', + } + ], + }, + "seed_files": { + "metadata": {"processed": 1, "created": 0, "restored": 0, "error": 1, "updated": 0, "skipped": 0}, + "data": [ + { + "exported_name": "names.csv", + "exported_project": None, + "imported_name": None, + "imported_project": None, + "status": "ERROR", + "message": "The seed file could not be written.", + } + ], + }, + "fallback_project": "Banking", +} + + +def test_export_bundle_exports_everything_by_default(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.post("http://test-server/api/export/v2/", content=BUNDLE, headers={"Content-Type": "application/zip"}) + bundle = client.export_bundle() + + assert bundle == BUNDLE + assert m.last_request.json() == {"include_seed_files": False} + + +def test_export_bundle_sends_the_rulesets_and_seed_file_option(client: DataMasqueClient) -> None: + ruleset = Ruleset(name="Customers") + ruleset.id = RulesetId("9d641e97-adf7-4f22-9089-afc3711bf222") + with requests_mock.Mocker() as m: + m.post("http://test-server/api/export/v2/", content=BUNDLE) + client.export_bundle([ruleset, RulesetId("04ea20f0-ad4c-498e-881f-b0bc79d83ba7")], include_seed_files=True) + + assert m.last_request.json() == { + "include_seed_files": True, + "ruleset_ids": ["9d641e97-adf7-4f22-9089-afc3711bf222", "04ea20f0-ad4c-498e-881f-b0bc79d83ba7"], + } + + +def test_export_bundle_rejects_an_empty_sequence_of_rulesets(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + with pytest.raises(ValueError, match="at least one ruleset"): + client.export_bundle([]) + + assert not m.called + + +def test_export_bundle_rejects_a_ruleset_not_yet_created(client: DataMasqueClient) -> None: + with pytest.raises(ValueError, match="not been created"): + client.export_bundle([Ruleset(name="Customers")]) + + +@pytest.mark.parametrize("rulesets", [RulesetId("9d641e97-adf7-4f22-9089-afc3711bf222"), Ruleset(name="Customers")]) +def test_export_bundle_rejects_a_single_ruleset(client: DataMasqueClient, rulesets: object) -> None: + with pytest.raises(TypeError, match="as a sequence"): + client.export_bundle(rulesets) # type: ignore[arg-type] + + +def test_import_bundle_uploads_the_archive_with_default_options(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.post("http://test-server/api/import/v2/", json=V2_IMPORT_RESPONSE, status_code=201) + result = client.import_bundle(BUNDLE) + + form = parse_multipart_form(m.last_request) + assert form == { + "overwrite_libraries": "False", + "overwrite_seed_files": "False", + "zip_archive": {"filename": "bundle.zip", "content_type": "application/zip", "content": BUNDLE}, + } + (ruleset,) = result.rulesets.data + assert ruleset.imported_project == "payments" + assert ruleset.ruleset_type is RulesetType.database + assert ruleset.status is BundleImportStatus.created + assert result.rulesets.metadata.updated is None + assert result.libraries is not None + assert result.libraries.metadata.skipped == 1 + assert result.libraries.data[0].status is BundleImportStatus.skipped + assert result.seed_files is None + assert result.fallback_project is None + assert result.is_deprecated_bundle is False + + +@pytest.mark.parametrize(("overwrite", "form_value"), [(True, "True"), (False, "False")]) +def test_import_bundle_sends_its_options( + client: DataMasqueClient, tmp_path: Path, overwrite: bool, form_value: str +) -> None: + bundle_path = tmp_path / "export.zip" + bundle_path.write_bytes(BUNDLE) + with requests_mock.Mocker() as m: + m.post("http://test-server/api/import/v2/", json=V2_IMPORT_RESPONSE) + client.import_bundle( + bundle_path, + overwrite_rulesets=overwrite, + overwrite_libraries=overwrite, + overwrite_seed_files=overwrite, + fallback_project=OTHER_PROJECT, + ) + + form = parse_multipart_form(m.last_request) + assert form["zip_archive"]["filename"] == "export.zip" + assert form["overwrite_rulesets"] == form_value + assert form["overwrite_libraries"] == form_value + assert form["overwrite_seed_files"] == form_value + assert form["fallback_project"] == "2" + + +def test_import_bundle_uploads_all_of_a_partly_read_file(client: DataMasqueClient) -> None: + bundle = BytesIO(BUNDLE) + bundle.read(4) + with requests_mock.Mocker() as m: + m.post("http://test-server/api/import/v2/", json=V2_IMPORT_RESPONSE) + client.import_bundle(bundle) + + archive = parse_multipart_form(m.last_request)["zip_archive"] + assert archive["filename"] == "bundle.zip" + assert archive["content"] == BUNDLE + + +def test_import_bundle_rejects_a_fallback_project_that_is_not_a_reference(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + with pytest.raises(ValueError, match="not a project ID or reference"): + client.import_bundle(BUNDLE, fallback_project="banking") + + assert not m.called + + +def test_import_bundle_does_not_send_the_client_default_project(client: DataMasqueClient) -> None: + client.default_project = "1-default" + with requests_mock.Mocker() as m: + m.post("http://test-server/api/import/v2/", json=V1_IMPORT_RESPONSE) + client.import_bundle(BUNDLE) + + assert "fallback_project" not in parse_multipart_form(m.last_request) + + +def test_import_bundle_reports_a_v1_bundle_and_its_fallback_project(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.post( + "http://test-server/api/import/v2/", + json=V1_IMPORT_RESPONSE, + status_code=201, + headers={"Deprecation": "true"}, + ) + result = client.import_bundle(BUNDLE, fallback_project="2-banking") + + assert parse_multipart_form(m.last_request)["fallback_project"] == "2" + assert result.is_deprecated_bundle is True + assert result.fallback_project == "Banking" + assert result.rulesets.data[0].exported_project is None + assert result.seed_files is not None + (seed_file,) = result.seed_files.data + assert seed_file.status is BundleImportStatus.error + assert seed_file.imported_project is None + + +def test_import_bundle_raises_when_the_server_rejects_it(client: DataMasqueClient) -> None: + detail = 'The bundle names the project "payments", which does not exist. Creating it requires an administrator.' + with requests_mock.Mocker() as m: + m.post("http://test-server/api/import/v2/", status_code=403, text=json.dumps({"detail": detail})) + with pytest.raises(DataMasqueApiError) as error: + client.import_bundle(BUNDLE) + + assert error.value.response.status_code == 403 + assert error.value.response.json() == {"detail": detail} diff --git a/tests/test_connections.py b/tests/test_connections.py index cbc3f7d..9f22271 100644 --- a/tests/test_connections.py +++ b/tests/test_connections.py @@ -1,10 +1,12 @@ """Tests for `ConnectionClient` (CRUD + Snowflake-specific behaviour).""" from datetime import datetime, timezone +from typing import Union import pytest import requests_mock +from datamasque.client import DataMasqueClient, Project, ProjectId from datamasque.client.exceptions import DataMasqueApiError, DataMasqueException from datamasque.client.models.connection import ( AzureConnectionConfig, @@ -31,6 +33,9 @@ validate_connection, ) from tests.helpers import ( + OTHER_PROJECT, + PROJECT, + PROJECT_ID, sample_mounted_share_connection_json, snowflake_connection_config, ) @@ -96,6 +101,69 @@ def test_create_or_update_connection_create_fail(client, connection_config, exis client.create_or_update_connection(connection_config) +def test_create_or_update_connection_sends_the_project( + client: DataMasqueClient, connection_config: ConnectionConfig +) -> None: + """The lookup for an existing connection is filtered to the connection's project, and the body carries it.""" + with requests_mock.Mocker() as m: + m.get("http://test-server/api/connections/", json=[], status_code=200) + m.post("http://test-server/api/connections/", json={"id": "2"}, status_code=201) + + client.create_or_update_connection(connection_config) + + assert m.request_history[0].qs == {"project": [PROJECT]} + assert m.last_request.json()["project"] == PROJECT + + +def test_create_or_update_connection_ignores_same_name_in_another_project( + client: DataMasqueClient, connection_config: ConnectionConfig +) -> None: + """A same-named connection in another project is not updated, because the listing is filtered by project.""" + other_project_connection = sample_mounted_share_connection_json(name=connection_config.name, id="1") | { + "project": "2-banking" + } + with requests_mock.Mocker() as m: + m.get("http://test-server/api/connections/", json=[other_project_connection], status_code=200) + m.get( + f"http://test-server/api/connections/?project={PROJECT}", + json=[], + status_code=200, + complete_qs=True, + ) + m.post("http://test-server/api/connections/", json={"id": "2"}, status_code=201) + + result = client.create_or_update_connection(connection_config) + + assert result.id == ConnectionId("2") + assert [request.method for request in m.request_history] == ["GET", "POST"] + + +@pytest.mark.parametrize("project", [ProjectId(2), OTHER_PROJECT]) +def test_list_connections_filters_by_project(client: DataMasqueClient, project: Union[ProjectId, Project]) -> None: + with requests_mock.Mocker() as m: + m.get("http://test-server/api/connections/", json=[], status_code=200) + client.list_connections(project) + + assert m.last_request.qs == {"project": ["2"]} + + +def test_list_connections_keeps_same_name_in_different_projects(client: DataMasqueClient) -> None: + """Connection names are unique per project, so an unfiltered listing can hold two connections with one name.""" + connections = [ + sample_mounted_share_connection_json(name="shared", id="1"), + sample_mounted_share_connection_json(name="shared", id="2") | {"project": OTHER_PROJECT.reference}, + ] + with requests_mock.Mocker() as m: + m.get("http://test-server/api/connections/", json=connections, status_code=200) + result = client.list_connections() + + assert m.last_request.qs == {} + assert [(connection.id, connection.project) for connection in result] == [ + ("1", PROJECT), + ("2", OTHER_PROJECT.reference), + ] + + def test_list_connnections(client): with requests_mock.Mocker() as m: m.get( @@ -104,6 +172,7 @@ def test_list_connnections(client): # Realistic API data, hence why it's so verbose { "name": "s3", + "project": PROJECT, "bucket": "my-s3-bucket", "base_directory": "", "type": "s3_connection", @@ -117,6 +186,7 @@ def test_list_connnections(client): }, { "name": "azure", + "project": PROJECT, "type": "azure_blob_connection", "base_directory": "", "mask_type": "file", @@ -131,6 +201,7 @@ def test_list_connnections(client): }, { "name": "mounted_share_dest", + "project": PROJECT, "type": "mounted_share_connection", "base_directory": "dest", "mask_type": "file", @@ -149,6 +220,7 @@ def test_list_connnections(client): "db_type": "mysql", "database": "mydatabase", "name": "mysql", + "project": PROJECT, "schema": "", "is_read_only": False, "password_encrypted": "some_base64_here", @@ -163,6 +235,7 @@ def test_list_connnections(client): "version": "1.0", "mask_type": "database", "name": "db_dynamo", + "project": PROJECT, "s3_bucket_name": "my-dynamo-staging-bucket", "dynamo_append_datetime": False, "dynamo_append_suffix": "-masked", @@ -193,6 +266,7 @@ def test_list_connnections(client): "db_type": "mssql_linked", "database": "database_name", "name": "mssql-linked", + "project": PROJECT, "schema": "", "is_read_only": False, "password_encrypted": "some_base64_here", @@ -208,6 +282,7 @@ def test_list_connnections(client): "version": "1.0", "mask_type": "database", "name": "db_dynamo_2", + "project": PROJECT, "s3_bucket_name": "my-dynamo-staging-bucket-2", "dynamo_append_datetime": False, "dynamo_append_suffix": "-masked", @@ -232,6 +307,7 @@ def test_list_connnections(client): "db_type": "snowflake", "database": "icicle", "name": "snowflake", + "project": PROJECT, "schema": "snowball", "snowflake_role": "snowballs do indeed roll", "snowflake_account_id": "ABCDEF-123456", @@ -253,6 +329,7 @@ def test_list_connnections(client): "db_type": "snowflake", "database": "igloo", "name": "snowflake_minimal_with_key", + "project": PROJECT, "snowflake_account_id": "ACCOUNT-1234", "snowflake_warehouse": "clothing_store", "snowflake_private_key": "2831289a-4398-abcd-4112-fe09a1239f89", @@ -381,10 +458,11 @@ def test_delete_connection_by_name(client): status_code=200, ) - client.delete_connection_by_name_if_exists(connection_name) + client.delete_connection_by_name_if_exists(connection_name, PROJECT_ID) assert m.call_count == 3 assert m.request_history[0].method == "GET" + assert m.request_history[0].qs == {"project": [str(PROJECT_ID)]} assert m.request_history[1].method == "DELETE" assert m.request_history[2].method == "DELETE" @@ -405,7 +483,7 @@ def test_delete_connection_that_does_not_exist(client): ], status_code=200, ) - client.delete_connection_by_name_if_exists("my_connection") + client.delete_connection_by_name_if_exists("my_connection", PROJECT_ID) assert m.call_count == 1 assert m.request_history[0].method == "GET" @@ -467,6 +545,7 @@ def test_list_snowflake_connection_from_older_server_drops_external_stage(client "db_type": "snowflake", "database": "s3_db", "name": "snowflake_s3", + "project": PROJECT, "snowflake_account_id": "S3-ACCOUNT", "snowflake_warehouse": "s3_warehouse", "snowflake_storage_integration_name": "s3_integration", @@ -494,6 +573,7 @@ def test_s3_connection_model_validate(): payload = { "id": "88dabb63-aca5-4cc4-8f76-f78736a42f39", "name": "s3", + "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "data/", @@ -519,6 +599,7 @@ def test_connection_model_validate_parses_license_lock() -> None: payload = { "id": "88dabb63-aca5-4cc4-8f76-f78736a42f39", "name": "s3", + "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "data/", @@ -542,6 +623,7 @@ def test_connection_model_validate_null_license_lock() -> None: payload = { "id": "id-1", "name": "s3", + "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "", @@ -559,6 +641,7 @@ def test_connection_model_validate_parses_freeze_fields() -> None: payload = { "id": "db-1", "name": "pg", + "project": PROJECT, "mask_type": "database", "db_type": "postgres", "host": "h", @@ -582,6 +665,7 @@ def test_connection_freeze_fields_default_when_absent() -> None: payload = { "id": "db-1", "name": "pg", + "project": PROJECT, "mask_type": "database", "db_type": "postgres", "host": "h", @@ -610,6 +694,7 @@ def test_connection_freeze_fields_default_when_absent() -> None: { "id": "id-1", "name": "s3", + "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "", @@ -624,6 +709,7 @@ def test_connection_freeze_fields_default_when_absent() -> None: { "id": "db-1", "name": "pg", + "project": PROJECT, "mask_type": "database", "db_type": "postgres", "host": "h", @@ -639,6 +725,7 @@ def test_connection_freeze_fields_default_when_absent() -> None: { "id": "dyn-1", "name": "dyn", + "project": PROJECT, "mask_type": "database", "db_type": "dynamo_db", "s3_bucket_name": "bucket", @@ -653,6 +740,7 @@ def test_connection_freeze_fields_default_when_absent() -> None: { "id": "mongo-1", "name": "mongo", + "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -671,6 +759,7 @@ def test_connection_freeze_fields_default_when_absent() -> None: { "id": "sf-1", "name": "snowflake", + "project": PROJECT, "mask_type": "database", "db_type": "snowflake", "user": "snowman", @@ -713,6 +802,7 @@ def test_s3_connection_model_validate_no_iam_role(): payload = { "id": "id-1", "name": "s3", + "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "", @@ -729,6 +819,7 @@ def test_databricks_connection_model_validate(): payload = { "id": "db-id-1", "name": "databricks", + "project": PROJECT, "mask_type": "database", "db_type": "databricks", "server_hostname": "adb-1234.azuredatabricks.net", @@ -755,6 +846,7 @@ def test_databricks_connection_model_validate_blanks_encrypted_token(): payload = { "id": "db-id-2", "name": "databricks", + "project": PROJECT, "mask_type": "database", "db_type": "databricks", "server_hostname": "adb-1234.azuredatabricks.net", @@ -774,6 +866,7 @@ def test_validate_connection_dispatches_databricks(): payload = { "id": "db-id-3", "name": "databricks", + "project": PROJECT, "mask_type": "database", "db_type": "databricks", "server_hostname": "adb-1234.azuredatabricks.net", @@ -792,6 +885,7 @@ def test_azure_connection_model_validate_blanks_encrypted_connection_string(): payload = { "id": "490502e5-5bf6-4abb-b67b-c6091d40ecf0", "name": "azure", + "project": PROJECT, "mask_type": "file", "type": "azure_blob_connection", "base_directory": "", @@ -814,6 +908,7 @@ def test_gcs_connection_model_validate_blanks_encrypted_service_account_key(): payload = { "id": "0f9f5d2e-8d1a-4a1f-9f57-3f2c1b0c9a11", "name": "gcs", + "project": PROJECT, "mask_type": "file", "type": "gcs_connection", "base_directory": "inbound", @@ -835,6 +930,7 @@ def test_gcs_connection_model_validate_blanks_encrypted_service_account_key(): def test_gcs_connection_serializes_the_key_for_create(): conn = GcsConnectionConfig( + project=PROJECT, name="gcs", bucket="my-gcs-bucket", service_account_key='{"type": "service_account"}', @@ -864,6 +960,7 @@ def test_database_connection_model_validate_drops_schema_for_mysql(): payload = { "id": "f0557fb3-1c9a-4cb1-bcf4-9699cf496bf7", "name": "mysql", + "project": PROJECT, "mask_type": "database", "db_type": "mysql", "host": "my-mysql-host", @@ -886,6 +983,7 @@ def test_database_connection_model_validate_keeps_schema_for_postgres(): payload = { "id": "abc", "name": "pg", + "project": PROJECT, "mask_type": "database", "db_type": "postgres", "host": "pg-host", @@ -904,6 +1002,7 @@ def test_database_connection_model_validate_databricks_lakebase(): payload = { "id": "abc-lakebase", "name": "lakebase", + "project": PROJECT, "mask_type": "database", "db_type": "databricks_lakebase", "host": "lakebase-host", @@ -925,6 +1024,7 @@ def test_mssql_linked_connection_model_validate_includes_linked_server(): payload = { "id": "48a7af45-f63f-4e05-bf9f-7b1cc3a0e89d", "name": "mssql-linked", + "project": PROJECT, "mask_type": "database", "db_type": "mssql_linked", "host": "mssql-linked-host", @@ -947,6 +1047,7 @@ def test_dynamo_connection_model_validate_with_sse(): payload = { "id": "d7257552-0485-4806-b0fb-d72b4d268073", "name": "db_dynamo", + "project": PROJECT, "mask_type": "database", "db_type": "dynamo_db", "s3_bucket_name": "my-dynamo-staging-bucket", @@ -975,6 +1076,7 @@ def test_dynamo_connection_model_validate_without_sse_uses_default(): payload = { "id": "id-2", "name": "db_dynamo", + "project": PROJECT, "mask_type": "database", "db_type": "dynamo_db", "s3_bucket_name": "bucket", @@ -996,6 +1098,7 @@ def test_snowflake_connection_model_validate_drops_external_stage_location(stage payload = { "id": "f0557fb3-1c9a-4cb1-bcf4-9699cf496bf7", "name": "snowflake", + "project": PROJECT, "mask_type": "database", "db_type": "snowflake", "user": "snowman", @@ -1029,6 +1132,7 @@ def test_snowflake_connection_model_validate_minimal_payload(): payload = { "id": "id-3", "name": "snowflake", + "project": PROJECT, "mask_type": "database", "db_type": "snowflake", "user": "frosty", @@ -1049,6 +1153,7 @@ def test_connection_config_dispatch_picks_subclass(): s3_payload = { "id": "id-s3", "name": "s3", + "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "", @@ -1059,6 +1164,7 @@ def test_connection_config_dispatch_picks_subclass(): db_payload = { "id": "id-pg", "name": "pg", + "project": PROJECT, "mask_type": "database", "db_type": "postgres", "host": "h", @@ -1088,6 +1194,7 @@ def test_dynamo_connection_round_trip_with_iam_role_and_prefix(): payload = { "id": "id-dynamo-1", "name": "db_dynamo", + "project": PROJECT, "mask_type": "database", "db_type": "dynamo_db", "s3_bucket_name": "staging-bucket", @@ -1111,6 +1218,7 @@ def test_dynamo_connection_round_trip_with_iam_role_and_prefix(): def test_dynamo_connection_model_dump_omits_unset_iam_role_and_prefix(): """When the new optional fields are unset, `model_dump` must omit them entirely (not send `null`).""" conn = DynamoConnectionConfig( + project=PROJECT, name="db_dynamo", s3_bucket_name="bucket", dynamo_append_datetime=False, @@ -1129,6 +1237,7 @@ def test_dynamo_model_validate_defaults_to_none_when_fields_absent(): payload = { "id": "id-dynamo-2", "name": "db_dynamo", + "project": PROJECT, "mask_type": "database", "db_type": "dynamo_db", "s3_bucket_name": "bucket", @@ -1146,6 +1255,7 @@ def test_dynamo_model_validate_defaults_to_none_when_fields_absent(): def test_mongo_connection_model_dump_minimal(): """An unauthenticated, plain TCP connection sends only the required keys plus the defaulted booleans.""" conn = MongoConnectionConfig( + project=PROJECT, name="mongo", host="mongo.example", database="people", @@ -1164,6 +1274,7 @@ def test_mongo_connection_model_dump_minimal(): def test_mongo_connection_model_dump_full(): conn = MongoConnectionConfig( + project=PROJECT, name="mongo", host="mongo.example", port=27018, @@ -1195,6 +1306,7 @@ def test_mongo_connection_model_dump_full(): def test_mongo_connection_model_dump_omits_falsy_optional_flags(): """`tls`, `direct_connection`, `user`, `password`, and `replica_set` are only sent when truthy.""" conn = MongoConnectionConfig( + project=PROJECT, name="mongo", host="mongo.example", database="people", @@ -1212,7 +1324,7 @@ def test_mongo_connection_model_dump_omits_falsy_optional_flags(): def test_mongo_connection_retry_writes_defaults_true_and_is_omitted(): """retry_writes defaults to True (matching the server default), so it is left out of the payload.""" - conn = MongoConnectionConfig(name="mongo", host="mongo.example", database="people") + conn = MongoConnectionConfig(project=PROJECT, name="mongo", host="mongo.example", database="people") assert conn.retry_writes is True api_dict = conn.model_dump(exclude_none=True, by_alias=True, mode="json") assert "retry_writes" not in api_dict @@ -1220,14 +1332,18 @@ def test_mongo_connection_retry_writes_defaults_true_and_is_omitted(): def test_mongo_connection_sends_retry_writes_when_disabled(): """retry_writes is sent only when explicitly disabled (e.g. AWS DocumentDB rejects retryable writes).""" - conn = MongoConnectionConfig(name="mongo", host="mongo.example", database="people", retry_writes=False) + conn = MongoConnectionConfig( + project=PROJECT, name="mongo", host="mongo.example", database="people", retry_writes=False + ) api_dict = conn.model_dump(exclude_none=True, by_alias=True, mode="json") assert api_dict["retry_writes"] is False def test_documentdb_connection_sends_retry_writes_when_disabled(): """A DocumentDB connection inherits retry_writes and sends it when disabled.""" - conn = DocumentDbConnectionConfig(name="docdb", host="docdb.example", database="people", retry_writes=False) + conn = DocumentDbConnectionConfig( + project=PROJECT, name="docdb", host="docdb.example", database="people", retry_writes=False + ) api_dict = conn.model_dump(exclude_none=True, by_alias=True, mode="json") assert api_dict["db_type"] == "documentdb" assert api_dict["retry_writes"] is False @@ -1237,6 +1353,7 @@ def test_mongo_connection_model_validate_blanks_encrypted_password(): payload = { "id": "mongo-id-1", "name": "mongo", + "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -1268,6 +1385,7 @@ def test_mongo_connection_model_validate_defaults_when_optional_fields_missing() payload = { "id": "mongo-id-2", "name": "mongo-min", + "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -1288,6 +1406,7 @@ def test_connection_config_dispatch_picks_mongo_subclass(): payload = { "id": "mongo-id-3", "name": "mongo", + "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -1298,6 +1417,7 @@ def test_connection_config_dispatch_picks_mongo_subclass(): def test_mongo_tls_ca_fields_emitted_when_tls_enabled(): conn = MongoConnectionConfig( + project=PROJECT, name="mongo", host="mongo.example", database="people", @@ -1312,7 +1432,7 @@ def test_mongo_tls_ca_fields_emitted_when_tls_enabled(): def test_mongo_tls_ca_fields_omitted_when_unset_but_tls_enabled(): - conn = MongoConnectionConfig(name="mongo", host="mongo.example", database="people", tls=True) + conn = MongoConnectionConfig(project=PROJECT, name="mongo", host="mongo.example", database="people", tls=True) d = conn.model_dump(exclude_none=True, by_alias=True, mode="json") assert d["tls"] is True assert "tls_ca_file" not in d @@ -1322,6 +1442,7 @@ def test_mongo_tls_ca_fields_omitted_when_unset_but_tls_enabled(): def test_mongo_tls_ca_fields_omitted_when_tls_disabled(): """Even when set, the TLS-dependent fields are not sent while TLS is disabled.""" conn = MongoConnectionConfig( + project=PROJECT, name="mongo", host="mongo.example", database="people", @@ -1338,6 +1459,7 @@ def test_mongo_tls_ca_fields_roundtrip(): payload = { "id": "mongo-tls-1", "name": "mongo", + "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -1356,6 +1478,7 @@ def test_mongo_tls_ca_fields_default_when_missing(): payload = { "id": "mongo-tls-2", "name": "mongo", + "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -1368,6 +1491,7 @@ def test_mongo_tls_ca_fields_default_when_missing(): def test_documentdb_connection_model_dump(): conn = DocumentDbConnectionConfig( + project=PROJECT, name="docdb", host="dtq-documentdb.cluster.example", database="people", @@ -1391,6 +1515,7 @@ def test_connection_config_dispatch_picks_documentdb_subclass(): payload = { "id": "docdb-id-1", "name": "docdb", + "project": PROJECT, "mask_type": "database", "db_type": "documentdb", "host": "dtq-documentdb.cluster.example", @@ -1404,6 +1529,7 @@ def test_connection_config_dispatch_picks_documentdb_subclass(): def _cosmosdb_connection(**overrides) -> CosmosDbConnectionConfig: """Build a Cosmos DB config, leaving the port to the connection type's own default.""" return CosmosDbConnectionConfig( + project=PROJECT, name="cosmos", host="dtq-cosmos.mongo.cosmos.azure.com", database="people", @@ -1449,6 +1575,7 @@ def test_connection_config_dispatch_picks_cosmosdb_subclass(): payload = { "id": "cosmos-id-1", "name": "cosmos", + "project": PROJECT, "mask_type": "database", "db_type": "cosmosdb", "host": "dtq-cosmos.mongo.cosmos.azure.com", @@ -1473,6 +1600,7 @@ def test_database_connection_config_rejects_document_store_database_types( """`DatabaseConnectionConfig` is for SQL engines; each document store has its own class.""" with pytest.raises(ValueError, match=message): DatabaseConnectionConfig( + project=PROJECT, name="doc-store", host="doc-store.example", port=27017, @@ -1486,6 +1614,7 @@ def test_database_connection_config_rejects_document_store_database_types( def test_create_or_update_mongo_connection(client): """End-to-end: a Mongo connection round-trips through `create_or_update_connection`.""" conn = MongoConnectionConfig( + project=PROJECT, name="mongo", host="mongo.example", database="people", @@ -1513,6 +1642,7 @@ def test_create_or_update_mongo_connection(client): def _database_connection(database_type: DatabaseType, **extra) -> DatabaseConnectionConfig: return DatabaseConnectionConfig( + project=PROJECT, name="conn", host="mydb.abc123.ap-southeast-2.rds.amazonaws.com", port=5432, @@ -1607,6 +1737,7 @@ def test_database_connection_config_rejects_engines_with_their_own_class( ) -> None: with pytest.raises(ValueError, match=message): DatabaseConnectionConfig( + project=PROJECT, name="own-class", host="db.example", port=1, @@ -1619,6 +1750,7 @@ def test_database_connection_config_rejects_engines_with_their_own_class( def test_cassandra_connection_serializes_password_as_dbpassword(): conn = CassandraConnectionConfig( + project=PROJECT, name="cassandra", host="cassandra.example", database="people", @@ -1643,9 +1775,9 @@ def test_cassandra_connection_serializes_password_as_dbpassword(): def test_cassandra_connection_without_credentials_omits_user_and_password(): """Cassandra allows unauthenticated access; the server rejects a username sent without a password.""" - d = CassandraConnectionConfig(name="cassandra", host="cassandra.example", database="people").model_dump( - exclude_none=True, by_alias=True, mode="json" - ) + d = CassandraConnectionConfig( + project=PROJECT, name="cassandra", host="cassandra.example", database="people" + ).model_dump(exclude_none=True, by_alias=True, mode="json") assert "user" not in d assert "dbpassword" not in d @@ -1655,6 +1787,7 @@ def test_connection_config_dispatch_picks_cassandra_subclass(): payload = { "id": "cassandra-id-1", "name": "cassandra", + "project": PROJECT, "mask_type": "database", "db_type": "cassandra", "host": "cassandra.example", @@ -1678,6 +1811,7 @@ def test_connection_config_dispatch_picks_cassandra_subclass(): def test_sap_ase_connection_serializes_without_a_schema(): """ASE has no schema setting: the schema is the table owner.""" conn = SapAseConnectionConfig( + project=PROJECT, name="sap_ase", host="ase.example", database="people", @@ -1703,9 +1837,9 @@ def test_sap_ase_connection_serializes_without_a_schema(): def test_sap_ase_connection_omits_unset_tls_options(): - d = SapAseConnectionConfig(name="sap_ase", host="ase.example", database="people", user="sa").model_dump( - exclude_none=True, by_alias=True, mode="json" - ) + d = SapAseConnectionConfig( + project=PROJECT, name="sap_ase", host="ase.example", database="people", user="sa" + ).model_dump(exclude_none=True, by_alias=True, mode="json") assert "tls_server_name" not in d assert "connect_timeout" not in d @@ -1716,6 +1850,7 @@ def test_connection_config_dispatch_picks_sap_ase_subclass(): payload = { "id": "sap-ase-id-1", "name": "sap_ase", + "project": PROJECT, "mask_type": "database", "db_type": "sap_ase", "host": "ase.example", @@ -1734,6 +1869,7 @@ def test_connection_config_dispatch_picks_sap_ase_subclass(): def test_salesforce_connection_serializes_for_create(): conn = SalesforceConnectionConfig( + project=PROJECT, name="salesforce", instance_url="https://example--uat.sandbox.my.salesforce.com", login_url="https://example--uat.sandbox.my.salesforce.com", @@ -1760,6 +1896,7 @@ def test_salesforce_connection_serializes_for_create(): def test_salesforce_connection_without_passphrase_omits_it(): """An unencrypted private key needs no passphrase.""" d = SalesforceConnectionConfig( + project=PROJECT, name="salesforce", instance_url="https://example.my.salesforce.com", login_url="https://example.my.salesforce.com", @@ -1775,6 +1912,7 @@ def test_connection_config_dispatch_picks_salesforce_subclass(): payload = { "id": "salesforce-id-1", "name": "salesforce", + "project": PROJECT, "mask_type": "database", "db_type": "salesforce", "instance_url": "https://example.my.salesforce.com", diff --git a/tests/test_default_project.py b/tests/test_default_project.py new file mode 100644 index 0000000..899ad5b --- /dev/null +++ b/tests/test_default_project.py @@ -0,0 +1,137 @@ +"""Tests for the client's `default_project`, which a create uses when the resource names no project.""" + +from collections.abc import Callable +from typing import Any, Optional + +import pytest +import requests_mock + +from datamasque.client import DataMasqueClient +from datamasque.client.models.connection import ConnectionId +from datamasque.client.models.discovery_config import DiscoveryConfig +from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary +from datamasque.client.models.dm_instance import DataMasqueInstanceConfig +from datamasque.client.models.rg_config import RGConfig +from datamasque.client.models.ruleset import Ruleset, RulesetType +from datamasque.client.models.ruleset_library import RulesetLibrary +from datamasque.client.models.table_reference import TableReference +from tests.helpers import active_projects_page, database_connection_config + +BASE_URL = "http://test-server" + + +def make_client(default_project: Optional[str] = None) -> DataMasqueClient: + return DataMasqueClient( + DataMasqueInstanceConfig(base_url=BASE_URL, username="u", password="p", default_project=default_project) + ) + + +def create_ruleset(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post(f"{BASE_URL}/api/rulesets/?upsert=true", json={"id": "r-1", "name": "r", "project": "2-acme"}) + ruleset = Ruleset(name="r", yaml="version: '1.0'", ruleset_type=RulesetType.database, project=project) + return client.create_or_update_ruleset(ruleset) + + +def create_connection(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.get(f"{BASE_URL}/api/connections/", json=[]) + m.post(f"{BASE_URL}/api/connections/", json={"id": "c-1", "project": "2-acme"}) + connection = database_connection_config() + connection.project = project + return client.create_or_update_connection(connection) + + +def create_discovery_config(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post( + f"{BASE_URL}/api/discovery/configs/", + json={"id": "dc-1", "name": "dc", "config_type": "database", "project": "2-acme"}, + ) + config = DiscoveryConfig(name="dc", yaml="labels: []\n", config_type="database", project=project) + return client.create_discovery_config(config) + + +def create_rg_config(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post(f"{BASE_URL}/api/ruleset-generation-configs/", json={"id": "rg-1", "name": "rg", "project": "2-acme"}) + return client.create_rg_config(RGConfig(name="rg", yaml="x: 1\n", project=project)) + + +def create_table_reference(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post( + f"{BASE_URL}/api/table-references/", + json={"id": "tr-1", "name": "tr", "connection": "c-1", "source": "people.csv", "project": "2-acme"}, + ) + table_reference = TableReference(name="tr", connection=ConnectionId("c-1"), source="people.csv", project=project) + return client.create_table_reference(table_reference) + + +def create_ruleset_library(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post(f"{BASE_URL}/api/ruleset-libraries/", json={"id": "rl-1", "name": "rl", "project": "2-acme"}) + return client.create_ruleset_library(RulesetLibrary(name="rl", yaml="version: '1.0'\n", project=project)) + + +def create_discovery_config_library(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: + m.post(f"{BASE_URL}/api/discovery/config-libraries/", json={"id": "dl-1", "name": "dl", "project": "2-acme"}) + return client.create_discovery_config_library( + DiscoveryConfigLibrary(name="dl", yaml="labels: []\n", project=project) + ) + + +Create = Callable[[DataMasqueClient, Optional[str], requests_mock.Mocker], Any] + +CREATES = [ + pytest.param(create_ruleset, id="ruleset"), + pytest.param(create_connection, id="connection"), + pytest.param(create_discovery_config, id="discovery_config"), + pytest.param(create_rg_config, id="rg_config"), + pytest.param(create_table_reference, id="table_reference"), + pytest.param(create_ruleset_library, id="ruleset_library"), + pytest.param(create_discovery_config_library, id="discovery_config_library"), +] + + +@pytest.mark.parametrize("create", CREATES) +def test_a_create_sends_the_resources_project_over_the_default(create: Create) -> None: + with requests_mock.Mocker() as m: + create(make_client(default_project="1-default"), "2-acme", m) + + assert m.last_request.json()["project"] == "2-acme" + + +@pytest.mark.parametrize("create", CREATES) +def test_a_create_sends_the_default_project_when_the_resource_names_none(create: Create) -> None: + with requests_mock.Mocker() as m: + create(make_client(default_project="1-default"), None, m) + + assert m.last_request.json()["project"] == "1-default" + assert all(request.path != "/api/projects/" for request in m.request_history) + + +@pytest.mark.parametrize("create", CREATES) +def test_a_create_with_no_default_project_sends_no_project(create: Create) -> None: + with requests_mock.Mocker() as m: + m.get(f"{BASE_URL}/api/projects/", json=active_projects_page(1)) + create(make_client(), None, m) + + assert "project" not in m.last_request.json() + + +def test_a_connection_update_sends_no_project() -> None: + """An update leaves the connection in its own project, whatever the default.""" + client = make_client(default_project="1-default") + connection = database_connection_config() + connection.project = None + with requests_mock.Mocker() as m: + m.get(f"{BASE_URL}/api/connections/", json=[{**connection.model_dump(mode="json"), "id": "c-1"}]) + m.put(f"{BASE_URL}/api/connections/c-1/", json={"id": "c-1", "project": "2-acme"}) + + client.create_or_update_connection(connection) + + assert m.last_request.method == "PUT" + assert "project" not in m.last_request.json() + + +def test_an_upsert_looks_for_the_name_in_the_default_project() -> None: + with requests_mock.Mocker() as m: + create_connection(make_client(default_project="1-default"), None, m) + + lookup = m.request_history[0] + assert (lookup.method, lookup.path, lookup.qs) == ("GET", "/api/connections/", {"project": ["1-default"]}) diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 83f9348..a42a396 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -46,10 +46,12 @@ from datamasque.client.models.connection import ConnectionId, DatabaseConnectionConfig, DatabaseType from datamasque.client.models.data_selection import SelectedColumns, SelectedFileData, UserSelection from datamasque.client.models.status import AsyncRulesetGenerationTaskStatus -from tests.helpers import parse_multipart_form +from tests.helpers import PROJECT, parse_multipart_form, sample_mounted_share_connection_json DISCOVERY_CONFIG_ID = "aaaaaaaa-1111-2222-3333-444444444444" RG_CONFIG_ID = "bbbbbbbb-1111-2222-3333-444444444444" +CONNECTION_PROJECT = "2-banking" +"""The project of the connection the generated rulesets come from.""" def test_generate_ruleset(client): @@ -159,6 +161,14 @@ def test_poll_async_ruleset_generation(client): assert status is AsyncRulesetGenerationTaskStatus.failed +def mock_source_connection(m: requests_mock.Mocker, connection_id: ConnectionId) -> None: + """Mocks the connection lookup that `get_generated_rulesets` makes to find the rulesets' project.""" + m.get( + f"http://test-server/api/connections/{connection_id}/", + json=sample_mounted_share_connection_json(id=connection_id, name="source") | {"project": CONNECTION_PROJECT}, + ) + + def test_get_generated_rulesets_success(client): connection_id = ConnectionId("1") yaml_content_1 = b""" @@ -185,6 +195,7 @@ def test_get_generated_rulesets_success(client): """ with requests_mock.Mocker() as m: + mock_source_connection(m, connection_id) m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", json={"status": "finished"}, @@ -211,6 +222,7 @@ def test_get_generated_rulesets_success(client): assert rulesets[0].yaml == yaml_content_1.decode("utf-8") assert rulesets[1].name == "ruleset2" assert rulesets[1].yaml == yaml_content_2.decode("utf-8") + assert [ruleset.project for ruleset in rulesets] == [CONNECTION_PROJECT, CONNECTION_PROJECT] def test_get_generated_rulesets_finished_with_warnings_success(client): @@ -229,6 +241,7 @@ def test_get_generated_rulesets_finished_with_warnings_success(client): """ with requests_mock.Mocker() as m: + mock_source_connection(m, connection_id) m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", json={"status": "finished_with_warnings"}, @@ -259,6 +272,7 @@ def test_get_generated_rulesets_empty_archive_raises(client): connection_id = ConnectionId("1") with requests_mock.Mocker() as m: + mock_source_connection(m, connection_id) m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", json={"status": "finished"}, @@ -287,6 +301,7 @@ def test_get_generated_rulesets_from_selection_success(client): generated_yaml = 'version: "1.0"\ntasks:\n- type: mask_table\n table: users\n key: id\n rules: []\n' with requests_mock.Mocker() as m: + mock_source_connection(m, connection_id) # Status check and the redirect target are the same URL; both resolve to the same JSON. m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", @@ -305,12 +320,14 @@ def test_get_generated_rulesets_from_selection_success(client): assert rulesets[0].yaml == generated_yaml # The server doesn't return a name — callers set one before create_or_update_ruleset. assert rulesets[0].name == "generated_ruleset" + assert rulesets[0].project == CONNECTION_PROJECT def test_get_generated_rulesets_from_selection_empty_ruleset_raises(client): """A finished task with no `generated_ruleset` in the JSON body raises a clear error.""" connection_id = ConnectionId("1") with requests_mock.Mocker() as m: + mock_source_connection(m, connection_id) m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", json={"status": "finished", "generated_ruleset": None}, @@ -376,6 +393,7 @@ def test_get_generated_rulesets_download_fail(client): connection_id = ConnectionId("1") with requests_mock.Mocker() as m: + mock_source_connection(m, connection_id) m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", json={"status": "finished"}, @@ -660,6 +678,7 @@ def test_schema_discovery_request_model_dump_includes_set_fields(): def test_discovery_requests_accept_connection_config_objects(): """All three discovery request models accept a full `ConnectionConfig` and extract its `id`.""" connection = DatabaseConnectionConfig( + project=PROJECT, id=ConnectionId("conn-uuid"), name="prod_db", db_type=DatabaseType.postgres, @@ -1242,14 +1261,16 @@ def test_schema_discovery_from_config_request_accepts_discovery_config_id(): def test_schema_discovery_from_config_request_unwraps_discovery_config_model(): """Passing a full `DiscoveryConfig` object substitutes its `id` for the wire payload.""" - config = DiscoveryConfig(name="my_cfg", config_type="database", id=DiscoveryConfigId(DISCOVERY_CONFIG_ID)) + config = DiscoveryConfig( + project=PROJECT, name="my_cfg", config_type="database", id=DiscoveryConfigId(DISCOVERY_CONFIG_ID) + ) req = SchemaDiscoveryFromConfigRequest(connection="conn-1", discovery_config=config) assert req.model_dump(exclude_none=True, mode="json")["discovery_config"] == DISCOVERY_CONFIG_ID def test_schema_discovery_from_config_request_rejects_unsaved_discovery_config(): """A `DiscoveryConfig` without an `id` cannot be used yet — raises immediately.""" - config = DiscoveryConfig(name="my_cfg", config_type="database") + config = DiscoveryConfig(project=PROJECT, name="my_cfg", config_type="database") with pytest.raises(ValueError, match="id is None"): SchemaDiscoveryFromConfigRequest(connection="conn-1", discovery_config=config) @@ -1352,7 +1373,9 @@ def test_start_schema_discovery_run_from_config_sends_schemas(client): def test_start_file_data_discovery_run_from_config_sends_discovery_config(client): """`start_file_data_discovery_run_from_config` posts only the connection and discovery_config id.""" - config = DiscoveryConfig(name="my_cfg", config_type="file", id=DiscoveryConfigId(DISCOVERY_CONFIG_ID)) + config = DiscoveryConfig( + project=PROJECT, name="my_cfg", config_type="file", id=DiscoveryConfigId(DISCOVERY_CONFIG_ID) + ) req = FileDataDiscoveryFromConfigRequest(connection="conn-1", discovery_config=config) with requests_mock.Mocker() as m: m.post("http://test-server/api/run-file-data-discovery/v2/", json={"id": 99}, status_code=201) @@ -1618,7 +1641,7 @@ def test_generate_ruleset_with_rg_config_none_sends_null(client): def test_ruleset_generation_with_rg_config_request_unwraps_rg_config_model(): """Passing a full `RGConfig` object substitutes its `id` for the wire payload.""" - config = RGConfig(name="my_rgc", id=RGConfigId(RG_CONFIG_ID)) + config = RGConfig(project=PROJECT, name="my_rgc", id=RGConfigId(RG_CONFIG_ID)) req = RulesetGenerationWithRGConfigRequest( connection="conn-1", selected_columns={"public": {"users": ["email"]}}, @@ -1629,7 +1652,7 @@ def test_ruleset_generation_with_rg_config_request_unwraps_rg_config_model(): def test_ruleset_generation_with_rg_config_request_rejects_unsaved_rg_config(): """An `RGConfig` without an `id` cannot be used yet — raises immediately.""" - config = RGConfig(name="my_rgc") + config = RGConfig(project=PROJECT, name="my_rgc") with pytest.raises(ValueError, match="id is None"): RulesetGenerationWithRGConfigRequest( connection="conn-1", @@ -1759,7 +1782,7 @@ def test_start_async_ruleset_generation_with_rg_config_unwraps_rg_config_model(c """A full `RGConfig` object is unwrapped to its id in the request body.""" connection_id = ConnectionId("1") selected_columns = SelectedColumns(columns={"public": {"users": ["col1"]}}) - config = RGConfig(name="my_rgc", id=RGConfigId(RG_CONFIG_ID)) + config = RGConfig(project=PROJECT, name="my_rgc", id=RGConfigId(RG_CONFIG_ID)) with requests_mock.Mocker() as m: m.post(f"http://test-server/api/async-generate-ruleset/v2/{connection_id}/", status_code=201) diff --git a/tests/test_discovery_config_libraries.py b/tests/test_discovery_config_libraries.py index 20a0d4d..c73f76b 100644 --- a/tests/test_discovery_config_libraries.py +++ b/tests/test_discovery_config_libraries.py @@ -13,6 +13,7 @@ ) from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.status import ValidationStatus +from tests.helpers import PROJECT, PROJECT_ID LIBRARY_ID_1 = "aaaaaaaa-1111-2222-3333-444444444444" LIBRARY_ID_2 = "bbbbbbbb-1111-2222-3333-444444444444" @@ -25,7 +26,7 @@ def sample_library_list_response() -> list[dict[str, Any]]: { "id": LIBRARY_ID_1, "name": "my_library", - "namespace": "org", + "project": PROJECT, "is_valid": "valid", "validation_error": None, "usage_count": 0, @@ -35,7 +36,7 @@ def sample_library_list_response() -> list[dict[str, Any]]: { "id": LIBRARY_ID_2, "name": "another_library", - "namespace": "", + "project": PROJECT, "is_valid": "invalid", "validation_error": "bad yaml", "usage_count": 2, @@ -51,7 +52,7 @@ def sample_library_detail_response() -> dict[str, Any]: return { "id": LIBRARY_ID_1, "name": "my_library", - "namespace": "org", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "valid", "validation_error": None, @@ -65,14 +66,14 @@ def sample_library_detail_response() -> dict[str, Any]: def config_library() -> DiscoveryConfigLibrary: return DiscoveryConfigLibrary( name="test_library", - namespace="test_ns", + project=PROJECT, yaml="labels: []\nmetadata_rules: []\nidd_rules: []\n", ) def test_library_is_untyped() -> None: """A library carries no `config_type`; the same library serves both config types.""" - library = DiscoveryConfigLibrary(name="test_library", yaml="labels: []\n") + library = DiscoveryConfigLibrary(name="test_library", project=PROJECT, yaml="labels: []\n") assert "config_type" not in library.model_dump(by_alias=True) @@ -90,7 +91,7 @@ def test_list_discovery_config_libraries( assert len(libraries) == 2 assert libraries[0].id == DiscoveryConfigLibraryId(LIBRARY_ID_1) assert libraries[0].name == "my_library" - assert libraries[0].namespace == "org" + assert libraries[0].project == PROJECT assert libraries[0].yaml is None assert libraries[0].is_valid is ValidationStatus.valid assert libraries[0].usage_count == 0 @@ -124,7 +125,7 @@ def test_get_discovery_config_library(client: DataMasqueClient, sample_library_d assert library.id == DiscoveryConfigLibraryId(LIBRARY_ID_1) assert library.name == "my_library" - assert library.namespace == "org" + assert library.project == PROJECT assert library.yaml == "labels: []\nmetadata_rules: []\nidd_rules: []\n" assert library.is_valid is ValidationStatus.valid assert library.usage_count == 0 @@ -146,67 +147,12 @@ def test_get_discovery_config_library_by_name_found( json=sample_library_detail_response, status_code=200, ) - library = client.get_discovery_config_library_by_name("my_library", "org") + library = client.get_discovery_config_library_by_name("my_library", PROJECT_ID) assert library is not None assert library.name == "my_library" assert library.yaml == "labels: []\nmetadata_rules: []\nidd_rules: []\n" - assert "name_exact=my_library" in m.request_history[0].url - - -def test_get_discovery_config_library_by_name_matches_namespace_client_side( - client: DataMasqueClient, sample_library_detail_response: dict[str, Any] -) -> None: - """The server's `namespace_exact` filter ignores empty strings, so namespace matching happens client-side.""" - same_name_two_namespaces = [ - { - "id": LIBRARY_ID_1, - "name": "my_library", - "namespace": "org", - "is_valid": "valid", - "created": "2025-01-01T12:00:00Z", - "modified": "2025-01-02T12:00:00Z", - }, - { - "id": LIBRARY_ID_2, - "name": "my_library", - "namespace": "", - "is_valid": "valid", - "created": "2025-01-01T12:00:00Z", - "modified": "2025-01-02T12:00:00Z", - }, - ] - - with requests_mock.Mocker() as m: - m.get( - "http://test-server/api/discovery/config-libraries/", - json=same_name_two_namespaces, - status_code=200, - ) - m.get( - f"http://test-server/api/discovery/config-libraries/{LIBRARY_ID_1}/", - json=sample_library_detail_response, - status_code=200, - ) - library = client.get_discovery_config_library_by_name("my_library", "org") - - assert library is not None - assert library.id == DiscoveryConfigLibraryId(LIBRARY_ID_1) - - -def test_get_discovery_config_library_by_name_default_namespace_does_not_match_other_namespaces( - client: DataMasqueClient, sample_library_list_response: list[dict[str, Any]] -) -> None: - with requests_mock.Mocker() as m: - m.get( - "http://test-server/api/discovery/config-libraries/", - json=[sample_library_list_response[0]], - status_code=200, - ) - library = client.get_discovery_config_library_by_name("my_library") - - assert library is None - assert m.call_count == 1 + assert m.request_history[0].qs == {"name_exact": ["my_library"], "project": [str(PROJECT_ID)]} def test_get_discovery_config_library_by_name_not_found(client: DataMasqueClient) -> None: @@ -216,7 +162,7 @@ def test_get_discovery_config_library_by_name_not_found(client: DataMasqueClient json=[], status_code=200, ) - library = client.get_discovery_config_library_by_name("nonexistent") + library = client.get_discovery_config_library_by_name("nonexistent", PROJECT_ID) assert library is None @@ -226,7 +172,7 @@ def test_get_discovery_config_library_by_name_raises_when_server_omits_id(client list_response_without_id = [ { "name": "my_library", - "namespace": "org", + "project": PROJECT, "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -240,14 +186,14 @@ def test_get_discovery_config_library_by_name_raises_when_server_omits_id(client status_code=200, ) with pytest.raises(DataMasqueApiError, match="without an `id`"): - client.get_discovery_config_library_by_name("my_library", "org") + client.get_discovery_config_library_by_name("my_library", PROJECT_ID) def test_create_discovery_config_library(client: DataMasqueClient, config_library: DiscoveryConfigLibrary) -> None: create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "valid", "validation_error": None, @@ -274,7 +220,7 @@ def test_create_discovery_config_library(client: DataMasqueClient, config_librar request_body = m.last_request.json() assert request_body == { "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", } @@ -285,7 +231,7 @@ def test_create_discovery_config_library_reports_validation_error( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "invalid", "validation_error": "metadata_rules is not a list", @@ -312,7 +258,7 @@ def test_update_discovery_config_library(client: DataMasqueClient, config_librar update_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "valid", "validation_error": None, @@ -338,7 +284,7 @@ def test_update_discovery_config_library(client: DataMasqueClient, config_librar request_body = m.last_request.json() assert request_body == { "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", } @@ -366,7 +312,7 @@ def test_create_or_update_discovery_config_library_create( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "valid", "validation_error": None, @@ -389,8 +335,9 @@ def test_create_or_update_discovery_config_library_create( assert result.id == DiscoveryConfigLibraryId(LIBRARY_ID_1) assert m.request_history[0].method == "GET" - assert "name_exact=test_library" in m.request_history[0].url + assert m.request_history[0].qs == {"name_exact": ["test_library"], "project": [PROJECT]} assert m.request_history[1].method == "POST" + assert m.request_history[1].json()["project"] == PROJECT def test_create_or_update_discovery_config_library_update( @@ -400,7 +347,7 @@ def test_create_or_update_discovery_config_library_update( { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "is_valid": "valid", "created": "2025-06-01T10:00:00Z", "modified": "2025-06-01T10:00:00Z", @@ -409,7 +356,7 @@ def test_create_or_update_discovery_config_library_update( update_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "valid", "validation_error": None, @@ -480,10 +427,10 @@ def test_delete_discovery_config_library_by_name( f"http://test-server/api/discovery/config-libraries/{LIBRARY_ID_1}/", status_code=204, ) - client.delete_discovery_config_library_by_name_if_exists("my_library", "org") + client.delete_discovery_config_library_by_name_if_exists("my_library", PROJECT_ID) assert m.request_history[0].method == "GET" - assert "name_exact=my_library" in m.request_history[0].url + assert m.request_history[0].qs == {"name_exact": ["my_library"], "project": [str(PROJECT_ID)]} assert m.request_history[1].method == "DELETE" @@ -496,17 +443,45 @@ def test_delete_discovery_config_library_by_name_not_found( json=sample_library_list_response, status_code=200, ) - client.delete_discovery_config_library_by_name_if_exists("nonexistent") + client.delete_discovery_config_library_by_name_if_exists("nonexistent", PROJECT_ID) assert m.call_count == 1 assert m.request_history[0].method == "GET" def test_create_discovery_config_library_with_empty_yaml_raises(client: DataMasqueClient) -> None: - library = DiscoveryConfigLibrary(name="test_library", yaml="") + library = DiscoveryConfigLibrary(name="test_library", project=PROJECT, yaml="") with requests_mock.Mocker() as m: with pytest.raises(ValueError, match="without YAML content"): client.create_discovery_config_library(library) assert not any(request.method == "POST" for request in m.request_history) + + +def test_namespace_in_a_response_is_dropped_and_not_sent_back( + client: DataMasqueClient, sample_library_detail_response: dict[str, Any] +) -> None: + """The server lets a sent `namespace` override `project`, so the client never echoes one back.""" + response = {**sample_library_detail_response, "namespace": "org"} + + with requests_mock.Mocker() as m: + m.get( + f"http://test-server/api/discovery/config-libraries/{LIBRARY_ID_1}/", + json=response, + status_code=200, + ) + m.put( + f"http://test-server/api/discovery/config-libraries/{LIBRARY_ID_1}/", + json=response, + status_code=200, + ) + library = client.get_discovery_config_library(DiscoveryConfigLibraryId(LIBRARY_ID_1)) + client.update_discovery_config_library(library) + + assert "namespace" not in library.model_dump() + assert m.last_request.json() == { + "name": "my_library", + "project": PROJECT, + "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", + } diff --git a/tests/test_discovery_configs.py b/tests/test_discovery_configs.py index d34c47d..a700251 100644 --- a/tests/test_discovery_configs.py +++ b/tests/test_discovery_configs.py @@ -8,7 +8,7 @@ from pydantic import JsonValue, ValidationError from datamasque.client import DataMasqueClient -from datamasque.client.exceptions import DataMasqueApiError, DataMasqueException +from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.discovery_config import ( DiscoveryConfig, DiscoveryConfigId, @@ -16,6 +16,7 @@ unwrap_discovery_config_id, ) from datamasque.client.models.status import ValidationStatus +from tests.helpers import PROJECT, PROJECT_ID CONFIG_ID_1 = "aaaaaaaa-1111-2222-3333-444444444444" CONFIG_ID_2 = "bbbbbbbb-1111-2222-3333-444444444444" @@ -31,6 +32,7 @@ def sample_config_list_response() -> dict[str, Any]: { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -39,6 +41,7 @@ def sample_config_list_response() -> dict[str, Any]: { "id": CONFIG_ID_2, "name": "another_config", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-02-01T12:00:00Z", @@ -53,6 +56,7 @@ def sample_config_detail_response() -> dict[str, Any]: return { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "config_type": "database", "archived": False, @@ -65,6 +69,7 @@ def sample_config_detail_response() -> dict[str, Any]: def discovery_config() -> DiscoveryConfig: return DiscoveryConfig( name="test_config", + project=PROJECT, yaml="labels: []\nmetadata_rules: []\nidd_rules: []\n", config_type="database", ) @@ -95,6 +100,7 @@ def test_list_discovery_configs_pagination(client: DataMasqueClient) -> None: { "id": CONFIG_ID_1, "name": "c1", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -103,6 +109,7 @@ def test_list_discovery_configs_pagination(client: DataMasqueClient) -> None: { "id": CONFIG_ID_2, "name": "c2", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -118,6 +125,7 @@ def test_list_discovery_configs_pagination(client: DataMasqueClient) -> None: { "id": "cccccccc-1111-2222-3333-444444444444", "name": "c3", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -171,6 +179,7 @@ def test_get_discovery_config_by_name_found( { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -190,12 +199,16 @@ def test_get_discovery_config_by_name_found( json=sample_config_detail_response, status_code=200, ) - config = client.get_discovery_config_by_name("my_config", DiscoveryConfigType.database) + config = client.get_discovery_config_by_name("my_config", DiscoveryConfigType.database, PROJECT_ID) assert config is not None assert config.name == "my_config" - assert "name_exact=my_config" in m.request_history[0].url - assert "config_type=database" in m.request_history[0].url + assert m.request_history[0].qs == { + "name_exact": ["my_config"], + "config_type": ["database"], + "project": [str(PROJECT_ID)], + "limit": ["1"], + } def test_get_discovery_config_by_name_not_found(client: DataMasqueClient) -> None: @@ -207,7 +220,7 @@ def test_get_discovery_config_by_name_not_found(client: DataMasqueClient) -> Non json=empty_response, status_code=200, ) - config = client.get_discovery_config_by_name("nonexistent", DiscoveryConfigType.database) + config = client.get_discovery_config_by_name("nonexistent", DiscoveryConfigType.database, PROJECT_ID) assert config is None @@ -220,6 +233,7 @@ def test_get_discovery_config_by_name_raises_when_server_omits_id(client: DataMa "results": [ { "name": "my_config", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -235,13 +249,14 @@ def test_get_discovery_config_by_name_raises_when_server_omits_id(client: DataMa status_code=200, ) with pytest.raises(DataMasqueApiError, match="without an `id`"): - client.get_discovery_config_by_name("my_config", DiscoveryConfigType.database) + client.get_discovery_config_by_name("my_config", DiscoveryConfigType.database, PROJECT_ID) def test_create_discovery_config(client: DataMasqueClient, discovery_config: DiscoveryConfig) -> None: create_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "config_type": "database", "is_valid": "in_progress", @@ -268,6 +283,7 @@ def test_create_discovery_config(client: DataMasqueClient, discovery_config: Dis assert request_body["name"] == "test_config" assert request_body["config_yaml"] == "labels: []\nmetadata_rules: []\nidd_rules: []\n" assert request_body["config_type"] == "database" + assert request_body["project"] == PROJECT # Server-managed read-only fields must not be echoed back in the request body. assert "id" not in request_body assert "is_valid" not in request_body @@ -281,6 +297,7 @@ def test_update_discovery_config(client: DataMasqueClient, discovery_config: Dis update_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "config_type": "database", "archived": False, @@ -326,6 +343,7 @@ def test_create_or_update_discovery_config_create(client: DataMasqueClient, disc create_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "config_type": "database", "archived": False, @@ -340,7 +358,9 @@ def test_create_or_update_discovery_config_create(client: DataMasqueClient, disc assert result.id == DiscoveryConfigId(CONFIG_ID_1) assert m.request_history[0].method == "GET" + assert m.request_history[0].qs["project"] == [PROJECT] assert m.request_history[1].method == "POST" + assert m.request_history[1].json()["project"] == PROJECT def test_create_or_update_discovery_config_update(client: DataMasqueClient, discovery_config: DiscoveryConfig) -> None: @@ -352,6 +372,7 @@ def test_create_or_update_discovery_config_update(client: DataMasqueClient, disc { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-06-01T10:00:00Z", @@ -362,6 +383,7 @@ def test_create_or_update_discovery_config_update(client: DataMasqueClient, disc update_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "config_type": "database", "archived": False, @@ -399,20 +421,28 @@ def test_delete_discovery_config_by_id_not_found(client: DataMasqueClient) -> No def test_delete_discovery_config_by_name(client: DataMasqueClient, sample_config_list_response: dict[str, Any]) -> None: with requests_mock.Mocker() as m: - m.get("http://test-server/api/discovery/configs/", json=sample_config_list_response, status_code=200) + # The server filters by name, so only the matching config is listed. + only_match = {**sample_config_list_response, "count": 1, "results": sample_config_list_response["results"][:1]} + m.get("http://test-server/api/discovery/configs/", json=only_match, status_code=200) m.delete(f"http://test-server/api/discovery/configs/{CONFIG_ID_1}/", status_code=204) - client.delete_discovery_config_by_name_if_exists("my_config", DiscoveryConfigType.database) + client.delete_discovery_config_by_name_if_exists("my_config", DiscoveryConfigType.database, PROJECT_ID) assert m.request_history[0].method == "GET" + assert m.request_history[0].qs == { + "name_exact": ["my_config"], + "config_type": ["database"], + "project": [str(PROJECT_ID)], + "limit": ["1"], + } assert m.request_history[1].method == "DELETE" -def test_delete_discovery_config_by_name_not_found( - client: DataMasqueClient, sample_config_list_response: dict[str, Any] -) -> None: +def test_delete_discovery_config_by_name_not_found(client: DataMasqueClient) -> None: + empty_response = {"count": 0, "next": None, "previous": None, "results": []} + with requests_mock.Mocker() as m: - m.get("http://test-server/api/discovery/configs/", json=sample_config_list_response, status_code=200) - client.delete_discovery_config_by_name_if_exists("nonexistent", DiscoveryConfigType.database) + m.get("http://test-server/api/discovery/configs/", json=empty_response, status_code=200) + client.delete_discovery_config_by_name_if_exists("nonexistent", DiscoveryConfigType.database, PROJECT_ID) assert m.call_count == 1 assert m.request_history[0].method == "GET" @@ -426,6 +456,7 @@ def test_delete_discovery_config_by_name_raises_when_server_omits_id(client: Dat "results": [ { "name": "my_config", + "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -440,29 +471,8 @@ def test_delete_discovery_config_by_name_raises_when_server_omits_id(client: Dat json=list_response_without_id, status_code=200, ) - with pytest.raises(DataMasqueException, match="without an `id`"): - client.delete_discovery_config_by_name_if_exists("my_config", DiscoveryConfigType.database) - - -def test_delete_discovery_config_by_name_only_deletes_matching_type(client: DataMasqueClient) -> None: - shared_name_response = { - "count": 2, - "next": None, - "previous": None, - "results": [ - {"id": CONFIG_ID_1, "name": "shared", "config_type": "database", "archived": False}, - {"id": CONFIG_ID_2, "name": "shared", "config_type": "file", "archived": False}, - ], - } - - with requests_mock.Mocker() as m: - m.get("http://test-server/api/discovery/configs/", json=shared_name_response, status_code=200) - m.delete(f"http://test-server/api/discovery/configs/{CONFIG_ID_2}/", status_code=204) - client.delete_discovery_config_by_name_if_exists("shared", DiscoveryConfigType.file) - - assert m.request_history[-1].method == "DELETE" - assert CONFIG_ID_2 in m.request_history[-1].url - assert CONFIG_ID_1 not in m.request_history[-1].url + with pytest.raises(DataMasqueApiError, match="without an `id`"): + client.delete_discovery_config_by_name_if_exists("my_config", DiscoveryConfigType.database, PROJECT_ID) def test_get_default_discovery_config_yaml(client: DataMasqueClient) -> None: @@ -486,6 +496,7 @@ def test_discovery_config_parses_validation_fields() -> None: { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "config_type": "database", "config_yaml": "labels: []", "is_valid": "invalid", @@ -505,6 +516,7 @@ def test_discovery_config_validation_fields_optional() -> None: { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "config_type": "database", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -521,12 +533,12 @@ def test_unwrap_discovery_config_id_passes_through_strings() -> None: def test_unwrap_discovery_config_id_extracts_id_from_model() -> None: - config = DiscoveryConfig(name="x", config_type="database", id=DiscoveryConfigId(CONFIG_ID_1)) + config = DiscoveryConfig(name="x", project=PROJECT, config_type="database", id=DiscoveryConfigId(CONFIG_ID_1)) assert unwrap_discovery_config_id(config) == CONFIG_ID_1 def test_unwrap_discovery_config_id_raises_without_id() -> None: - config = DiscoveryConfig(name="x", config_type="database") + config = DiscoveryConfig(name="x", project=PROJECT, config_type="database") with pytest.raises(ValueError, match="id is None"): unwrap_discovery_config_id(config) @@ -535,6 +547,7 @@ def _build_config_response(is_valid: str, **extra: JsonValue) -> dict[str, JsonV return { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_type": "database", "is_valid": is_valid, "validation_error": None, @@ -595,7 +608,7 @@ def test_discovery_config_rejects_malformed_error_entry() -> None: def test_create_discovery_config_with_empty_yaml_raises(client: DataMasqueClient) -> None: - config = DiscoveryConfig(name="test_config", yaml="", config_type="database") + config = DiscoveryConfig(name="test_config", project=PROJECT, yaml="", config_type="database") with requests_mock.Mocker() as m: with pytest.raises(ValueError, match="without YAML content"): @@ -605,7 +618,7 @@ def test_create_discovery_config_with_empty_yaml_raises(client: DataMasqueClient def test_update_discovery_config_with_empty_yaml_raises(client: DataMasqueClient) -> None: - config = DiscoveryConfig(id=CONFIG_ID_1, name="test_config", yaml="", config_type="database") + config = DiscoveryConfig(id=CONFIG_ID_1, name="test_config", project=PROJECT, yaml="", config_type="database") with requests_mock.Mocker() as m: with pytest.raises(ValueError, match="without YAML content"): diff --git a/tests/test_files.py b/tests/test_files.py index 3b8f99b..37a2913 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -10,8 +10,8 @@ import requests_mock from datamasque.client import DataMasqueClient -from datamasque.client.models.files import OracleWalletFile, SeedFile, SnowflakeKeyFile, SslZipFile -from tests.helpers import fake, parse_multipart_form +from datamasque.client.models.files import DataMasqueFile, OracleWalletFile, SeedFile, SnowflakeKeyFile, SslZipFile +from tests.helpers import OTHER_PROJECT, PROJECT, PROJECT_ID, fake, parse_multipart_form @pytest.mark.parametrize( @@ -49,7 +49,7 @@ def test_upload_file(client, file_type, source_factory): "modified_date": mock_return_modified_date.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), }, ) - file = client.upload_file(file_type, name_of_file, source) + file = client.upload_file(file_type, name_of_file, source, PROJECT_ID) if isinstance(source, (str, Path)): m_open.assert_called_once_with(source, "rb") @@ -141,7 +141,7 @@ def test_get_files_by_type_and_name(client, file_type): }, ], ) - result = client.get_file_of_type_by_name(file_type, "file2") + result = client.get_file_of_type_by_name(file_type, "file2", PROJECT_ID) assert isinstance(result, file_type) assert result.id == id_2 @@ -202,7 +202,7 @@ def test_upload_file_if_not_exists_skips_when_same_name_exists(client, file_type }, ], ) - result = client.upload_file_if_not_exists(file_type, "already_here.csv") + result = client.upload_file_if_not_exists(file_type, "already_here.csv", PROJECT_ID) assert result is None # Only the list-by-type GET should have fired — no POST. @@ -231,7 +231,7 @@ def test_upload_file_if_not_exists_uploads_when_missing(client, file_type): "modified_date": "2024-01-01T00:00:00.000000Z", }, ) - result = client.upload_file_if_not_exists(file_type, Path("new_file.csv")) + result = client.upload_file_if_not_exists(file_type, Path("new_file.csv"), PROJECT_ID) assert isinstance(result, file_type) assert result.id == new_id @@ -264,7 +264,7 @@ def test_upload_file_retries_on_401(config): }, ], ) - client.upload_file(SeedFile, "seed.csv", file_content) + client.upload_file(SeedFile, "seed.csv", file_content, PROJECT_ID) assert m_request.call_count == 2 first_form = parse_multipart_form(m_request.request_history[0]) @@ -286,7 +286,7 @@ def test_upload_ssl_zip_file_sends_mysql_database_type(client): "modified_date": "2024-01-01T00:00:00.000000Z", }, ) - client.upload_file(SslZipFile, "certs.zip", b"zip bytes") + client.upload_file(SslZipFile, "certs.zip", b"zip bytes", PROJECT_ID) form = parse_multipart_form(m_request.request_history[0]) assert form["database_type"] == "mysql" @@ -306,7 +306,77 @@ def test_upload_non_ssl_zip_file_omits_database_type(client, file_type): "modified_date": "2024-01-01T00:00:00.000000Z", }, ) - client.upload_file(file_type, "file.bin", b"content") + client.upload_file(file_type, "file.bin", b"content", PROJECT_ID) form = parse_multipart_form(m_request.request_history[0]) assert "database_type" not in form + + +@pytest.mark.parametrize("file_type", [SeedFile, OracleWalletFile, SslZipFile, SnowflakeKeyFile]) +def test_upload_file_sends_project(client: DataMasqueClient, file_type: type[DataMasqueFile]) -> None: + with requests_mock.Mocker() as m_request: + m_request.post( + f"http://test-server/{file_type.get_url()}", + status_code=201, + json={ + "id": str(uuid.uuid4()), + "name": "file.bin", + "project": PROJECT, + "created_date": "2024-01-01T00:00:00.000000Z", + "modified_date": "2024-01-01T00:00:00.000000Z", + }, + ) + file = client.upload_file(file_type, "file.bin", b"content", PROJECT_ID) + + form = parse_multipart_form(m_request.request_history[0]) + assert form["project"] == str(PROJECT_ID) + assert file.project == PROJECT + + +@pytest.mark.parametrize("file_type", [SeedFile, OracleWalletFile, SslZipFile, SnowflakeKeyFile]) +def test_list_files_of_type_filters_by_project(client: DataMasqueClient, file_type: type[DataMasqueFile]) -> None: + with requests_mock.Mocker() as m_request: + m_request.get(f"http://test-server/{file_type.get_url()}?project={PROJECT_ID}", json=[], complete_qs=True) + assert client.list_files_of_type(file_type, PROJECT_ID) == [] + + +@pytest.mark.parametrize("file_type", [SeedFile, OracleWalletFile, SslZipFile, SnowflakeKeyFile]) +def test_get_file_of_type_by_name_is_scoped_to_the_project( + client: DataMasqueClient, file_type: type[DataMasqueFile] +) -> None: + """A file with the same name in another project is not matched, because the listing is filtered by project.""" + other_project_file = { + "id": str(uuid.uuid4()), + "name": "seed.csv", + "project": "2-other", + "created_date": "2024-01-01T00:00:00.000000Z", + } + with requests_mock.Mocker() as m_request: + m_request.get(f"http://test-server/{file_type.get_url()}", json=[other_project_file]) + m_request.get(f"http://test-server/{file_type.get_url()}?project={PROJECT_ID}", json=[], complete_qs=True) + result = client.get_file_of_type_by_name(file_type, "seed.csv", PROJECT_ID) + + assert result is None + assert m_request.last_request.qs == {"project": [str(PROJECT_ID)]} + + +def test_upload_file_sends_the_id_of_a_project(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m_request: + m_request.post( + f"http://test-server/{SeedFile.get_url()}", + status_code=201, + json={ + "id": str(uuid.uuid4()), + "name": "seed.csv", + "project": "2-banking", + "created_date": "2024-01-01T00:00:00.000000Z", + }, + ) + client.upload_file( + SeedFile, + "seed.csv", + b"content", + OTHER_PROJECT, + ) + + assert parse_multipart_form(m_request.request_history[0])["project"] == "2" diff --git a/tests/test_git.py b/tests/test_git.py new file mode 100644 index 0000000..98a99d8 --- /dev/null +++ b/tests/test_git.py @@ -0,0 +1,332 @@ +"""Tests for Git push, pull and file listing with the DataMasque client.""" + +from collections.abc import Callable +from datetime import datetime, timezone + +import pytest +import requests_mock +from pydantic import ValidationError + +from datamasque.client import ( + DataMasqueApiError, + DataMasqueClient, + DiscoveryConfig, + DiscoveryConfigLibrary, + DiscoveryConfigType, + GitFile, + Ruleset, + RulesetLibrary, + RulesetType, +) +from datamasque.client.models.dm_instance import DataMasqueInstanceConfig + +BASE_URL = "http://test-server" +COMMIT_SHA = "0123456789abcdef0123456789abcdef01234567" +COMMIT = {"commit": COMMIT_SHA, "date": "2026-10-01T10:00:00+00:00", "message": "Add customers"} + +LIST_METHODS: list[tuple[Callable[[DataMasqueClient], list[GitFile]], str, str]] = [ + (lambda client: client.list_ruleset_git_files(), "/api/ruleset-git/files/", "rulesets/Banking/customers.yml"), + (lambda client: client.list_ruleset_library_git_files(), "/api/library-git/files/", "Banking/emails.yml"), + ( + lambda client: client.list_discovery_config_git_files(), + "/api/discovery-config-git/files/", + "Banking/customers.yml", + ), + ( + lambda client: client.list_discovery_config_library_git_files(), + "/api/discovery-config-library-git/files/", + "Banking/emails.yml", + ), +] + + +@pytest.mark.parametrize(("list_files", "endpoint", "project_file"), LIST_METHODS) +def test_list_git_files_returns_each_file_with_its_commits( + client: DataMasqueClient, + list_files: Callable[[DataMasqueClient], list[GitFile]], + endpoint: str, + project_file: str, +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(f"{BASE_URL}{endpoint}", json={project_file: [COMMIT], "legacy.yml": []}) + + files = list_files(client) + + assert m.last_request.qs == {} + assert [file.path for file in files] == [project_file, "legacy.yml"] + assert files[0].commits[0].commit_sha == COMMIT_SHA + assert files[0].commits[0].date == datetime(2026, 10, 1, 10, tzinfo=timezone.utc) + assert files[0].commits[0].message == "Add customers" + assert files[1].commits == [] + + +def test_list_git_files_sends_the_branch(client: DataMasqueClient) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(f"{BASE_URL}/api/ruleset-git/files/", json={}) + + assert client.list_ruleset_git_files(branch="Feature/X") == [] + + assert m.last_request.qs == {"branch": ["Feature/X"]} + + +@pytest.mark.parametrize( + ("path", "folder", "name", "extension"), + [ + ("rulesets/Banking/Customers.yml", "rulesets/Banking", "Customers", ".yml"), + ("Customers.yaml", "", "Customers", ".yaml"), + ("rulesets/Banking/v1.customers.yml", "rulesets/Banking", "v1.customers", ".yml"), + ], +) +def test_pull_ruleset_from_git_splits_the_listed_path( + client: DataMasqueClient, path: str, folder: str, name: str, extension: str +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(f"{BASE_URL}/api/ruleset-git/", json={"config_yaml": "version: '1.0'"}) + + content = client.pull_ruleset_from_git(path, COMMIT_SHA, branch="main") + + assert content == "version: '1.0'" + assert m.last_request.qs == { + "ruleset_name": [name], + "extension": [extension], + "folder": [folder], + "commit_id": [COMMIT_SHA], + "branch": ["main"], + } + + +@pytest.mark.parametrize( + ("pull", "endpoint", "name_field", "folder_param"), + [ + (DataMasqueClient.pull_ruleset_library_from_git, "/api/library-git/", "library_name", "namespace"), + (DataMasqueClient.pull_discovery_config_from_git, "/api/discovery-config-git/", "config_name", "folder"), + ( + DataMasqueClient.pull_discovery_config_library_from_git, + "/api/discovery-config-library-git/", + "library_name", + "namespace", + ), + ], +) +def test_pull_from_git_sends_each_type_s_fields( + client: DataMasqueClient, + pull: Callable[[DataMasqueClient, str, str], str], + endpoint: str, + name_field: str, + folder_param: str, +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(f"{BASE_URL}{endpoint}", json={"config_yaml": "libraries: []"}) + + assert pull(client, "Bank/AU/Emails.yml", COMMIT_SHA) == "libraries: []" + + assert m.last_request.qs == { + name_field: ["Emails"], + "extension": [".yml"], + folder_param: ["Bank/AU"], + "commit_id": [COMMIT_SHA], + } + + +def test_push_ruleset_to_git_sends_the_ruleset(client: DataMasqueClient) -> None: + ruleset = Ruleset(name="customers", project="7-banking", yaml="version: '1.0'", ruleset_type=RulesetType.file) + ruleset.id = "a-ruleset-id" + with requests_mock.Mocker(case_sensitive=True) as m: + m.post(f"{BASE_URL}/api/ruleset-git/", status_code=204) + + client.push_ruleset_to_git(ruleset, "Update customers", branch="main", extension=".yaml") + + assert m.last_request.json() == { + "ruleset_name": "customers", + "mask_type": "file", + "id": "a-ruleset-id", + "project": "7-banking", + "ruleset_content": "version: '1.0'", + "commit_message": "Update customers", + "extension": ".yaml", + "branch": "main", + } + + +def test_push_and_push_path_leave_a_saved_resource_s_unset_project_to_the_server() -> None: + client = DataMasqueClient( + DataMasqueInstanceConfig(base_url=BASE_URL, username="u", password="p", default_project="1-default") + ) + library = RulesetLibrary(name="emails", yaml="libraries: []") + library.id = "a-library-id" + with requests_mock.Mocker(case_sensitive=True) as m: + m.post(f"{BASE_URL}/api/library-git/", status_code=204) + m.get(f"{BASE_URL}/api/library-git/push-path/", json={"path": "masking/libraries/Banking/emails"}) + + client.push_ruleset_library_to_git(library, "Add emails") + assert client.get_ruleset_library_git_push_path(library) == "masking/libraries/Banking/emails" + + push, push_path = m.request_history + assert push_path.qs == {"library_name": ["emails"], "id": ["a-library-id"]} + assert push.json() == { + "library_name": "emails", + "id": "a-library-id", + "library_content": "libraries: []", + "commit_message": "Add emails", + "extension": ".yml", + } + + +def test_push_and_push_path_send_the_default_project_for_an_unsaved_resource_with_no_project() -> None: + client = DataMasqueClient( + DataMasqueInstanceConfig(base_url=BASE_URL, username="u", password="p", default_project="1-default") + ) + library = DiscoveryConfigLibrary(name="emails", yaml="x: 1") + with requests_mock.Mocker(case_sensitive=True) as m: + m.post(f"{BASE_URL}/api/discovery-config-library-git/", status_code=204) + m.get( + f"{BASE_URL}/api/discovery-config-library-git/push-path/", + json={"path": "masking/discovery-config-libraries/Default/emails"}, + ) + + client.push_discovery_config_library_to_git(library, "Add emails") + client.get_discovery_config_library_git_push_path(library) + + push, push_path = m.request_history + assert push_path.qs == {"library_name": ["emails"], "project": ["1-default"]} + assert push.json() == { + "library_name": "emails", + "project": "1-default", + "library_content": "x: 1", + "commit_message": "Add emails", + "extension": ".yml", + } + + +def test_push_and_push_path_send_no_project_when_neither_the_resource_nor_the_client_sets_one( + client: DataMasqueClient, +) -> None: + ruleset = Ruleset(name="customers", yaml="version: '1.0'") + with requests_mock.Mocker(case_sensitive=True) as m: + m.post(f"{BASE_URL}/api/ruleset-git/", status_code=204) + m.get(f"{BASE_URL}/api/ruleset-git/push-path/", json={"path": "masking/rulesets/Banking/customers"}) + + client.push_ruleset_to_git(ruleset, "Add customers") + client.get_ruleset_git_push_path(ruleset) + + push, push_path = m.request_history + assert "project" not in push.json() + assert "project" not in push_path.qs + + +def test_push_discovery_config_to_git_sends_its_type(client: DataMasqueClient) -> None: + config = DiscoveryConfig(name="customers", project="7-banking", yaml="x: 1", config_type=DiscoveryConfigType.file) + with requests_mock.Mocker(case_sensitive=True) as m: + m.post(f"{BASE_URL}/api/discovery-config-git/", status_code=204) + + client.push_discovery_config_to_git(config, "Add customers") + + assert m.last_request.json() == { + "config_name": "customers", + "config_type": "file", + "project": "7-banking", + "config_content": "x: 1", + "commit_message": "Add customers", + "extension": ".yml", + } + + +@pytest.mark.parametrize( + "push", + [ + lambda client: client.push_ruleset_to_git(Ruleset(name="customers", project="7-banking"), "Add customers"), + lambda client: client.push_ruleset_library_to_git(RulesetLibrary(name="emails", project="7-banking"), "Add"), + lambda client: client.push_discovery_config_to_git( + DiscoveryConfig(name="customers", project="7-banking", config_type=DiscoveryConfigType.database), "Add" + ), + lambda client: client.push_discovery_config_library_to_git( + DiscoveryConfigLibrary(name="emails", project="7-banking", yaml=""), "Add" + ), + ], +) +def test_push_without_yaml_raises(client: DataMasqueClient, push: Callable[[DataMasqueClient], None]) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + with pytest.raises(ValueError, match="yaml is empty or unset"): + push(client) + + assert m.request_history == [] + + +def test_push_raises_on_a_path_conflict(client: DataMasqueClient) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.post(f"{BASE_URL}/api/ruleset-git/", status_code=409, json={"detail": "Cannot push"}) + + with pytest.raises(DataMasqueApiError) as error: + client.push_ruleset_to_git( + Ruleset(name="customers", project="7-banking", yaml="version: '1.0'"), "Add customers" + ) + + assert error.value.response.status_code == 409 + + +@pytest.mark.parametrize( + ("get_push_path", "endpoint", "expected_params"), + [ + ( + lambda client: client.get_ruleset_git_push_path(Ruleset(name="customers", project="7-banking")), + "/api/ruleset-git/push-path/", + {"ruleset_name": ["customers"], "mask_type": ["database"], "project": ["7-banking"]}, + ), + ( + lambda client: client.get_ruleset_library_git_push_path(RulesetLibrary(name="emails", project="7-banking")), + "/api/library-git/push-path/", + {"library_name": ["emails"], "project": ["7-banking"]}, + ), + ( + lambda client: client.get_discovery_config_git_push_path( + DiscoveryConfig(name="customers", project="7-banking", config_type=DiscoveryConfigType.database) + ), + "/api/discovery-config-git/push-path/", + {"config_name": ["customers"], "config_type": ["database"], "project": ["7-banking"]}, + ), + ( + lambda client: client.get_discovery_config_library_git_push_path( + DiscoveryConfigLibrary(name="emails", project="7-banking") + ), + "/api/discovery-config-library-git/push-path/", + {"library_name": ["emails"], "project": ["7-banking"]}, + ), + ], +) +def test_get_git_push_path_returns_the_server_s_path( + client: DataMasqueClient, + get_push_path: Callable[[DataMasqueClient], str], + endpoint: str, + expected_params: dict[str, list[str]], +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(f"{BASE_URL}{endpoint}", json={"path": "masking/rulesets/Banking/customers"}) + + assert get_push_path(client) == "masking/rulesets/Banking/customers" + + assert m.last_request.qs == expected_params + + +@pytest.mark.parametrize( + ("call", "endpoint", "response"), + [ + ( + lambda client: client.pull_ruleset_from_git("rulesets/Banking/customers.yml", COMMIT_SHA), + "/api/ruleset-git/", + {"config_yaml": None}, + ), + ( + lambda client: client.get_ruleset_git_push_path(Ruleset(name="customers", project="7-banking")), + "/api/ruleset-git/push-path/", + {}, + ), + ], +) +def test_a_malformed_git_response_raises( + client: DataMasqueClient, call: Callable[[DataMasqueClient], str], endpoint: str, response: dict[str, object] +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(f"{BASE_URL}{endpoint}", json=response) + + with pytest.raises(ValidationError): + call(client) diff --git a/tests/test_model_project.py b/tests/test_model_project.py index 112b18a..17251d0 100644 --- a/tests/test_model_project.py +++ b/tests/test_model_project.py @@ -13,6 +13,7 @@ from datamasque.client.models.ruleset_library import RulesetLibrary from datamasque.client.models.runs import RunInfo, UnfinishedRun from datamasque.client.models.table_reference import TableReference +from tests.helpers import PROJECT ResourceModel = Union[ ConnectionConfig, Ruleset, RulesetLibrary, DiscoveryConfig, DiscoveryConfigLibrary, RGConfig, TableReference @@ -52,17 +53,29 @@ @pytest.mark.parametrize(("parse", "payload"), _RESOURCE_CASES) -def test_resource_parses_project_and_excludes_it_when_serializing(parse: ParseResource, payload: Payload) -> None: - model = parse({**payload, "project": "1-default"}) +def test_resource_parses_project_and_sends_it_when_serializing(parse: ParseResource, payload: Payload) -> None: + model = parse({**payload, "project": PROJECT}) - assert model.project == "1-default" + assert model.project == PROJECT assert "project" not in (model.model_extra or {}) - assert "project" not in model.model_dump(by_alias=True, mode="json") + assert model.model_dump(by_alias=True, mode="json")["project"] == PROJECT @pytest.mark.parametrize(("parse", "payload"), _RESOURCE_CASES) -def test_resource_project_defaults_to_none_when_absent(parse: ParseResource, payload: Payload) -> None: - assert parse(payload).project is None +def test_resource_project_is_optional_and_not_sent_when_unset(parse: ParseResource, payload: Payload) -> None: + model = parse(payload) + + assert model.project is None + assert "project" not in model.model_dump(by_alias=True, exclude_none=True, mode="json") + + +@pytest.mark.parametrize("library_class", [RulesetLibrary, DiscoveryConfigLibrary]) +def test_library_drops_namespace(library_class: type[Union[RulesetLibrary, DiscoveryConfigLibrary]]) -> None: + """The server still returns the deprecated `namespace`, which would override `project` if it were sent back.""" + library = library_class.model_validate({"id": "l-1", "name": "l", "project": PROJECT, "namespace": "org"}) + + assert "namespace" not in (library.model_extra or {}) + assert "namespace" not in library.model_dump(by_alias=True, mode="json") _RUN_PAYLOAD: Payload = { @@ -77,7 +90,7 @@ def test_resource_project_defaults_to_none_when_absent(parse: ParseResource, pay @pytest.mark.parametrize("run_class", [RunInfo, UnfinishedRun]) def test_run_parses_project(run_class: type[Union[RunInfo, UnfinishedRun]]) -> None: - run = run_class.model_validate({**_RUN_PAYLOAD, "project": "1-default"}) + run = run_class.model_validate({**_RUN_PAYLOAD, "project": PROJECT}) - assert run.project == "1-default" + assert run.project == PROJECT assert "project" not in (run.model_extra or {}) diff --git a/tests/test_projects.py b/tests/test_projects.py new file mode 100644 index 0000000..11c3a96 --- /dev/null +++ b/tests/test_projects.py @@ -0,0 +1,538 @@ +"""Tests for project support in the DataMasque client.""" + +from datetime import datetime +from typing import Callable, Optional + +import pytest +import requests_mock +from pydantic import BaseModel, ValidationError + +from datamasque.client import ( + DataMasqueApiError, + DataMasqueClient, + Project, + ProjectCreateRequest, + ProjectDeleteConflict, + ProjectDeleteConflictError, + ProjectDeleteConflictReason, + ProjectId, + ProjectIdentifier, + ProjectUpdateRequest, +) +from datamasque.client.exceptions import DataMasqueException +from datamasque.client.models.connection import ConnectionId, S3ConnectionConfig +from datamasque.client.models.discovery_config import DiscoveryConfig +from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary +from datamasque.client.models.project import unwrap_project_id +from datamasque.client.models.rg_config import RGConfig +from datamasque.client.models.ruleset import Ruleset, RulesetType +from datamasque.client.models.ruleset_library import RulesetLibrary +from datamasque.client.models.table_reference import TableReference +from tests.helpers import PROJECT, PROJECT_ID, active_projects_page + +CREATED_ID = "aaaaaaaa-1111-2222-3333-444444444444" + +BANKING_PROJECT = { + "id": 7, + "reference": "7-banking", + "name": "Banking", + "description": "Customer banking data", + "created": "2026-10-01T10:00:00Z", + "updated": "2026-10-01T10:00:00Z", +} + +LEGACY_PROJECT = { + "id": 8, + "reference": "8-legacy", + "name": "Legacy", + "description": "", + "created": "2026-09-01T10:00:00Z", + "updated": "2026-10-01T10:00:00Z", +} + +FILES_EXIST_MESSAGE = 'Cannot rename the project, because files already exist at "seeds/retail".' + +RENAMED_PROJECT = {**BANKING_PROJECT, "reference": "7-retail", "name": "Retail"} + +UPDATE_PROJECT_CASES = [ + pytest.param(ProjectId(7), ProjectUpdateRequest(name="Retail"), {"name": "Retail"}, id="rename-by-id"), + pytest.param("7-banking", ProjectUpdateRequest(name="Retail"), {"name": "Retail"}, id="rename-by-reference"), + pytest.param( + Project.model_validate(BANKING_PROJECT), + ProjectUpdateRequest(description=""), + {"description": ""}, + id="project-object-clears-description", + ), + pytest.param( + ProjectId(7), + ProjectUpdateRequest(name="Banking", description="Customer banking data"), + {"name": "Banking", "description": "Customer banking data"}, + id="name-and-description", + ), +] + + +CONFLICTS = [ + { + "reason": "name_in_use", + "detail": 'The destination project already contains a ruleset named "3407".', + "type": "rulesets", + "id": "0b6c2f0e-4a0d-4d8e-9a57-2f0f3c1d9e11", + "name": "3407", + }, + { + "reason": "file_exists", + "detail": 'The destination project already has a file at "projects/8-retail/seeds/names.csv".', + "type": "seeds", + "id": "5e3b8a40-7c2d-4f61-b1a9-0d4e6f2c8b77", + "name": "names.csv", + "path": "projects/8-retail/seeds/names.csv", + }, + { + "reason": "unexpected_file", + "detail": 'The project directory contains "projects/7-banking/notes.txt", which belongs to no resource.', + "path": "projects/7-banking/notes.txt", + }, +] + + +def test_create_request_rejects_unknown_fields() -> None: + with pytest.raises(ValidationError, match="archived"): + ProjectCreateRequest(name="Banking", archived=True) # type: ignore[call-arg] + + +def test_create_project(client: DataMasqueClient) -> None: + request = ProjectCreateRequest(name="Banking", description="Customer banking data") + + with requests_mock.Mocker() as m: + m.post("http://test-server/api/projects/", json=BANKING_PROJECT, status_code=201) + project = client.create_project(request) + + assert m.last_request.json() == {"name": "Banking", "description": "Customer banking data"} + assert project.id == ProjectId(7) + assert project.reference == "7-banking" + assert project.name == "Banking" + assert project.description == "Customer banking data" + assert project.created == datetime.fromisoformat("2026-10-01T10:00:00+00:00") + assert project.updated == datetime.fromisoformat("2026-10-01T10:00:00+00:00") + + +def test_create_project_returns_the_name_and_description_as_stored(client: DataMasqueClient) -> None: + """Whitespace in the name and description is sent as given, and the server's stored values are returned.""" + request = ProjectCreateRequest(name=" Banking ", description=" Customer banking data ") + + with requests_mock.Mocker() as m: + m.post("http://test-server/api/projects/", json=BANKING_PROJECT, status_code=201) + project = client.create_project(request) + + assert m.last_request.json() == {"name": " Banking ", "description": " Customer banking data "} + assert project.name == "Banking" + assert project.description == "Customer banking data" + + +def test_create_project_rejected_name_raises(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.post( + "http://test-server/api/projects/", + json={"name": ['Project name must not contain "/".']}, + status_code=400, + ) + with pytest.raises(DataMasqueApiError) as exc_info: + client.create_project(ProjectCreateRequest(name="bank/au")) + + assert exc_info.value.response.status_code == 400 + + +def test_get_project_delete_conflicts_parses_each_reason(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/7/delete-check/", json={"conflicts": CONFLICTS}) + conflicts = client.get_project_delete_conflicts(ProjectId(7), ProjectId(8)) + + assert m.last_request.qs == {"destination": ["8"]} + assert conflicts == [ + ProjectDeleteConflict( + reason=ProjectDeleteConflictReason.name_in_use, + detail='The destination project already contains a ruleset named "3407".', + type="rulesets", + id="0b6c2f0e-4a0d-4d8e-9a57-2f0f3c1d9e11", + name="3407", + ), + ProjectDeleteConflict( + reason=ProjectDeleteConflictReason.file_exists, + detail='The destination project already has a file at "projects/8-retail/seeds/names.csv".', + type="seeds", + id="5e3b8a40-7c2d-4f61-b1a9-0d4e6f2c8b77", + name="names.csv", + path="projects/8-retail/seeds/names.csv", + ), + ProjectDeleteConflict( + reason=ProjectDeleteConflictReason.unexpected_file, + detail='The project directory contains "projects/7-banking/notes.txt", which belongs to no resource.', + path="projects/7-banking/notes.txt", + ), + ] + assert conflicts[0].path is None + assert conflicts[2].type is None + assert conflicts[2].id is None + assert conflicts[2].name is None + + +def test_get_project_delete_conflicts_returns_empty_list_when_project_can_be_deleted( + client: DataMasqueClient, +) -> None: + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/7/delete-check/", json={"conflicts": []}) + conflicts = client.get_project_delete_conflicts(ProjectId(7), ProjectId(8)) + + assert conflicts == [] + + +def test_project_delete_calls_accept_projects_as_well_as_ids(client: DataMasqueClient) -> None: + project = Project.model_validate(BANKING_PROJECT) + destination = Project.model_validate(BANKING_PROJECT | {"id": 8, "reference": "8-retail", "name": "Retail"}) + + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/7/delete-check/", json={"conflicts": []}) + m.delete("http://test-server/api/projects/7/", status_code=204) + client.get_project_delete_conflicts(project, destination) + client.delete_project(project, destination) + + assert [request.qs for request in m.request_history] == [{"destination": ["8"]}, {"destination": ["8"]}] + + +def test_delete_project(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.delete("http://test-server/api/projects/7/", status_code=204) + client.delete_project(ProjectId(7), ProjectId(8)) + + assert m.last_request.method == "DELETE" + assert m.last_request.qs == {"destination": ["8"]} + + +def test_delete_project_conflict_raises_with_conflicts(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.delete( + "http://test-server/api/projects/7/", + json={"detail": 'Project "Banking" cannot be deleted.', "conflicts": CONFLICTS}, + status_code=409, + ) + with pytest.raises(ProjectDeleteConflictError) as exc_info: + client.delete_project(ProjectId(7), ProjectId(8)) + + assert exc_info.value.response.status_code == 409 + assert str(exc_info.value) == 'Project "Banking" cannot be deleted.' + assert [conflict.reason for conflict in exc_info.value.conflicts] == [ + ProjectDeleteConflictReason.name_in_use, + ProjectDeleteConflictReason.file_exists, + ProjectDeleteConflictReason.unexpected_file, + ] + assert exc_info.value.conflicts[0].name == "3407" + + +@pytest.mark.parametrize("status_code", [400, 404]) +def test_delete_project_raises_a_plain_api_error_for_a_status_other_than_409( + client: DataMasqueClient, status_code: int +) -> None: + with requests_mock.Mocker() as m: + m.delete("http://test-server/api/projects/7/", json={"detail": "Not found."}, status_code=status_code) + with pytest.raises(DataMasqueApiError) as exc_info: + client.delete_project(ProjectId(7), ProjectId(8)) + + assert not isinstance(exc_info.value, ProjectDeleteConflictError) + assert exc_info.value.response.status_code == status_code + + +def test_list_projects_follows_pages(client: DataMasqueClient) -> None: + """With more active projects than fit on a page of 100, the client follows `next` to the last page.""" + projects = [ + { + **BANKING_PROJECT, + "id": project_id, + "reference": f"{project_id}-project-{project_id}", + "name": f"Project {project_id}", + } + for project_id in range(1, 102) + ] + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/projects/?limit=100&offset=0", + json={ + "count": 101, + "next": "http://test-server/api/projects/?limit=100&offset=100", + "results": projects[:100], + }, + complete_qs=True, + ) + m.get( + "http://test-server/api/projects/?limit=100&offset=100", + json={"count": 101, "next": None, "results": projects[100:]}, + complete_qs=True, + ) + result = client.list_projects() + + assert [project.id for project in result] == list(range(1, 102)) + + +def test_list_projects_include_archived(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/projects/?include_archived=true&limit=100&offset=0", + json={"count": 2, "next": None, "results": [BANKING_PROJECT, LEGACY_PROJECT]}, + complete_qs=True, + ) + projects = client.list_projects(include_archived=True) + + assert [project.id for project in projects] == [ProjectId(7), ProjectId(8)] + + +@pytest.mark.parametrize("project", [ProjectId(7), Project.model_validate(BANKING_PROJECT), "7-banking"]) +def test_get_project(client: DataMasqueClient, project: ProjectIdentifier) -> None: + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/7/", json=BANKING_PROJECT, complete_qs=True) + result = client.get_project(project) + + assert result.id == ProjectId(7) + assert result.name == "Banking" + + +def test_get_missing_project_raises(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/99/", json={"detail": "Not found."}, status_code=404) + with pytest.raises(DataMasqueApiError) as exc_info: + client.get_project(ProjectId(99)) + + assert exc_info.value.response.status_code == 404 + + +EMPTY_PAGE = {"count": 0, "next": None, "previous": None, "results": []} + +CREATE_OR_UPDATE_WITHOUT_A_PROJECT = [ + pytest.param( + lambda: S3ConnectionConfig(name="s3", bucket="b", base_directory=""), + "create_or_update_connection", + "/api/connections/", + [], + id="connection", + ), + pytest.param( + lambda: RGConfig(name="rg", yaml="labels: []"), + "create_or_update_rg_config", + "/api/ruleset-generation-configs/", + EMPTY_PAGE, + id="rg_config", + ), + pytest.param( + lambda: RulesetLibrary(name="lib", yaml="version: '1.0'"), + "create_or_update_ruleset_library", + "/api/ruleset-libraries/", + EMPTY_PAGE, + id="ruleset_library", + ), + pytest.param( + lambda: DiscoveryConfig(name="dc", yaml="labels: []", config_type="database"), + "create_or_update_discovery_config", + "/api/discovery/configs/", + EMPTY_PAGE, + id="discovery_config", + ), + pytest.param( + lambda: DiscoveryConfigLibrary(name="dcl", yaml="labels: []"), + "create_or_update_discovery_config_library", + "/api/discovery/config-libraries/", + [], + id="discovery_config_library", + ), + pytest.param( + lambda: TableReference(name="tr", connection=ConnectionId("1"), source="customers.csv"), + "create_or_update_table_reference", + "/api/table-references/", + [], + id="table_reference", + ), +] +""" +Each `create_or_update_*` that looks up the existing resource itself: +a factory for a resource with no `project`, the method, the path it looks up and creates on, +and that path's empty listing. +""" + +RULESET_WITHOUT_A_PROJECT = pytest.param( + lambda: Ruleset(name="rs", yaml="version: '1.0'\ntasks: []", ruleset_type=RulesetType.database), + "create_or_update_ruleset", + "/api/rulesets/", + None, + id="ruleset", +) +"""`create_or_update_ruleset`, which leaves finding the existing ruleset to the server's upsert.""" + + +@pytest.mark.parametrize("make_resource, method, path, empty_listing", CREATE_OR_UPDATE_WITHOUT_A_PROJECT) +def test_create_or_update_without_a_project_uses_the_only_active_project( + client: DataMasqueClient, + make_resource: Callable[[], BaseModel], + method: str, + path: str, + empty_listing: object, +) -> None: + """The lookup is in the only active project, and the create leaves the choice of project to the server.""" + resource = make_resource() + created = resource.model_dump(mode="json", by_alias=True) | {"id": CREATED_ID, "project": PROJECT} + + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/", json=active_projects_page(1), status_code=200) + m.get(f"http://test-server{path}", json=empty_listing, status_code=200) + m.post(f"http://test-server{path}", json=created, status_code=201) + getattr(client, method)(resource) + + assert [(request.method, request.path) for request in m.request_history] == [ + ("GET", "/api/projects/"), + ("GET", path), + ("POST", path), + ] + assert m.request_history[0].qs == {"limit": ["1"]} + assert m.request_history[1].qs["project"] == [str(PROJECT_ID)] + assert "project" not in m.last_request.json() + assert resource.model_dump()["project"] == PROJECT + + +def test_create_or_update_ruleset_without_a_project_uses_the_only_active_project(client: DataMasqueClient) -> None: + """The upsert leaves the choice of project to the server, after checking that exactly one project is active.""" + ruleset = Ruleset(name="rs", yaml="version: '1.0'\ntasks: []", ruleset_type=RulesetType.database) + + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/", json=active_projects_page(1), status_code=200) + m.post( + "http://test-server/api/rulesets/?upsert=true", + json={"id": CREATED_ID, "name": "rs", "project": PROJECT}, + status_code=201, + ) + client.create_or_update_ruleset(ruleset) + + assert [(request.method, request.path) for request in m.request_history] == [ + ("GET", "/api/projects/"), + ("POST", "/api/rulesets/"), + ] + assert "project" not in m.last_request.json() + assert ruleset.project == PROJECT + + +@pytest.mark.parametrize("active_projects", [0, 2]) +@pytest.mark.parametrize( + "make_resource, method, path, empty_listing", [*CREATE_OR_UPDATE_WITHOUT_A_PROJECT, RULESET_WITHOUT_A_PROJECT] +) +def test_create_or_update_without_a_project_needs_exactly_one_active_project( + client: DataMasqueClient, + make_resource: Callable[[], BaseModel], + method: str, + path: str, + empty_listing: object, + active_projects: int, +) -> None: + """ + With no project, the server only accepts a create when exactly one project is active. + + So the client raises before looking up or writing anything, + rather than updating a same-named resource in a project the server would not have picked. + """ + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/", json=active_projects_page(active_projects), status_code=200) + with pytest.raises(DataMasqueException, match=f"has {active_projects} active projects"): + getattr(client, method)(make_resource()) + + assert [request.path for request in m.request_history] == ["/api/projects/"] + + +@pytest.mark.parametrize( + "project", + [ProjectId(7), Project.model_validate(BANKING_PROJECT), "7-banking", "7"], + ids=["id", "project", "reference", "id as a string"], +) +def test_unwrap_project_id_reads_the_id_from_each_form(project: ProjectIdentifier) -> None: + assert unwrap_project_id(project) == ProjectId(7) + + +@pytest.mark.parametrize("project", ["myproject", "", "-7"]) +def test_unwrap_project_id_rejects_a_string_that_does_not_start_with_an_id(project: str) -> None: + with pytest.raises(ValueError, match=f'"{project}" is not a project ID or reference'): + unwrap_project_id(project) + + +def test_update_request_rejects_unknown_fields() -> None: + with pytest.raises(ValidationError) as exc_info: + # mypy rejects the unknown field, which the test passes on purpose. + ProjectUpdateRequest(name="Banking", archived=True) # type: ignore[call-arg] + + assert [(error["loc"], error["msg"]) for error in exc_info.value.errors()] == [ + (("archived",), "Extra inputs are not permitted") + ] + + +@pytest.mark.parametrize(("project", "request_body", "expected_body"), UPDATE_PROJECT_CASES) +def test_update_project_sends_only_the_fields_set( + client: DataMasqueClient, + project: ProjectIdentifier, + request_body: ProjectUpdateRequest, + expected_body: dict[str, object], +) -> None: + with requests_mock.Mocker() as m: + m.patch("http://test-server/api/projects/7/", json=BANKING_PROJECT) + client.update_project(project, request_body) + + assert m.last_request.json() == expected_body + + +def test_update_project_returns_the_renamed_project(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.patch("http://test-server/api/projects/7/", json=RENAMED_PROJECT) + project = client.update_project(ProjectId(7), ProjectUpdateRequest(name="Retail")) + + assert project.name == "Retail" + assert project.reference == "7-retail" + + +def test_update_project_rejected_name_raises(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.patch( + "http://test-server/api/projects/7/", + json={"name": [FILES_EXIST_MESSAGE]}, + status_code=400, + ) + with pytest.raises(DataMasqueApiError) as exc_info: + client.update_project(ProjectId(7), ProjectUpdateRequest(name="Retail")) + + assert exc_info.value.response.status_code == 400 + assert exc_info.value.response.json() == {"name": [FILES_EXIST_MESSAGE]} + + +@pytest.mark.parametrize( + ("status_code", "detail"), + [ + (403, "You do not have permission to perform this action."), + (404, "Not found."), + ], +) +def test_update_project_refused_raises(client: DataMasqueClient, status_code: int, detail: str) -> None: + with requests_mock.Mocker() as m: + m.patch("http://test-server/api/projects/7/", json={"detail": detail}, status_code=status_code) + with pytest.raises(DataMasqueApiError) as exc_info: + client.update_project(ProjectId(7), ProjectUpdateRequest(name="Retail")) + + assert exc_info.value.response.status_code == status_code + + +@pytest.mark.parametrize( + ("body", "expected"), + [ + pytest.param({**BANKING_PROJECT, "has_git_synced_resources": True}, True, id="synced"), + pytest.param({**BANKING_PROJECT, "has_git_synced_resources": False}, False, id="not-synced"), + pytest.param(BANKING_PROJECT, None, id="flag-absent"), + ], +) +def test_get_project_reports_git_synced_resources( + client: DataMasqueClient, body: dict[str, object], expected: Optional[bool] +) -> None: + with requests_mock.Mocker() as m: + m.get("http://test-server/api/projects/7/", json=body) + project = client.get_project(ProjectId(7)) + + assert project.has_git_synced_resources is expected diff --git a/tests/test_resources.py b/tests/test_resources.py new file mode 100644 index 0000000..f755e6a --- /dev/null +++ b/tests/test_resources.py @@ -0,0 +1,265 @@ +"""Tests for listing and moving resources with the DataMasque client.""" + +from datetime import datetime +from typing import Any, Optional + +import pytest +import requests_mock + +from datamasque.client import ( + ConnectionResource, + DataMasqueApiError, + DataMasqueClient, + MaskType, + Project, + ProjectId, + Resource, + ResourceMoveConflictReason, + ResourceOrdering, + ResourceTypeName, + RulesetResource, +) + +RULESET = { + "id": "0b6c2f0e-4a0d-4d8e-9a57-2f0f3c1d9e11", + "name": "mask_customers", + "project": "7-banking", + "created": "2026-10-01T10:00:00Z", + "modified": "2026-10-02T10:00:00Z", + "mask_type": "database", +} +CONNECTION = { + "id": "5e3b8a40-7c2d-4f61-b1a9-0d4e6f2c8b77", + "name": "customers_db", + "project": "7-banking", + "created": None, + "modified": None, + "mask_type": "database", +} +BUILT_IN_SEED = { + "id": "9a1d3c55-2b7e-4c0a-8f16-3e5d7b9c1a24", + "name": "first_names.csv", + "project": None, + "created": "2026-01-01T00:00:00Z", + "modified": "2026-01-01T00:00:00Z", +} + + +def page(results: dict[str, list[dict]], next_url: Optional[str] = None) -> dict: + return { + "count": sum(len(rows) for rows in results.values()), + "next": next_url, + "previous": None, + "results": results, + } + + +def test_list_resources_groups_them_by_type_name(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/resources/", + json=page({"rulesets": [RULESET], "connections": [CONNECTION], "seeds": [BUILT_IN_SEED]}), + ) + resources = client.list_resources() + + assert sorted(resources) == ["connections", "rulesets", "seeds"] + (ruleset,) = resources["rulesets"] + assert isinstance(ruleset, RulesetResource) + assert ruleset.id == RULESET["id"] + assert ruleset.name == "mask_customers" + assert ruleset.project == "7-banking" + assert ruleset.created == datetime.fromisoformat("2026-10-01T10:00:00+00:00") + assert ruleset.mask_type is MaskType.database + (connection,) = resources["connections"] + assert isinstance(connection, ConnectionResource) + assert connection.created is None + (seed,) = resources["seeds"] + assert type(seed) is Resource + assert seed.project is None + assert seed.position is None + + +def test_list_resources_sends_its_filters(client: DataMasqueClient) -> None: + retail = Project.model_validate( + { + "id": 8, + "reference": "8-retail", + "name": "Retail", + "created": "2026-10-01T10:00:00Z", + "updated": "2026-10-01T10:00:00Z", + } + ) + with requests_mock.Mocker() as m: + m.get("http://test-server/api/resources/", json=page({"rulesets": [], "connections": []})) + client.list_resources( + projects=[ProjectId(7), retail], + types=[ResourceTypeName.rulesets, ResourceTypeName.connections], + search="customers", + ordering=ResourceOrdering.modified, + ) + + assert m.last_request.qs == { + "project": ["7", "8"], + "type": ["rulesets", "connections"], + "search": ["customers"], + "ordering": ["modified"], + "limit": ["500"], + "offset": ["0"], + } + + +@pytest.mark.parametrize( + ("kwargs", "expected"), + [ + ({}, "type"), + ({"descending": True}, "-type"), + ({"ordering": ResourceOrdering.project}, "project"), + ({"ordering": ResourceOrdering.project, "descending": True}, "-project"), + ], +) +def test_list_resources_sends_its_ordering(client: DataMasqueClient, kwargs: dict[str, Any], expected: str) -> None: + with requests_mock.Mocker() as m: + m.get("http://test-server/api/resources/", json=page({})) + client.list_resources(**kwargs) + + assert m.last_request.qs["ordering"] == [expected] + + +def test_list_resources_keeps_each_resources_position(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/resources/", + json=page({"connections": [CONNECTION | {"position": 1}], "rulesets": [RULESET | {"position": 0}]}), + ) + resources = client.list_resources(ordering=ResourceOrdering.modified, descending=True) + + in_order = sorted((resource for group in resources.values() for resource in group), key=lambda r: r.position) + assert [resource.id for resource in in_order] == [RULESET["id"], CONNECTION["id"]] + + +@pytest.mark.parametrize( + ("projects", "types", "expected"), + [ + ([], None, {type_name.value: [] for type_name in ResourceTypeName}), + ([], [ResourceTypeName.rulesets], {"rulesets": []}), + ([ProjectId(7)], [], {}), + ], +) +def test_list_resources_with_an_empty_filter_returns_no_resources_without_a_request( + client: DataMasqueClient, + projects: list[ProjectId], + types: Optional[list[ResourceTypeName]], + expected: dict[str, list[Resource]], +) -> None: + with requests_mock.Mocker() as m: + resources = client.list_resources(projects=projects, types=types) + + assert resources == expected + assert m.call_count == 0 + + +def test_list_resources_follows_every_page_and_merges_the_groups(client: DataMasqueClient) -> None: + next_url = "http://test-server/api/resources/?limit=2&offset=2" + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/resources/", + [ + {"json": page({"rulesets": [RULESET], "connections": [CONNECTION]}, next_url=next_url)}, + {"json": page({"rulesets": [RULESET | {"id": "c3d1e5f7-0000-4000-8000-000000000001"}]})}, + ], + ) + resources = client.list_resources(page_size=2) + + assert [ruleset.id for ruleset in resources["rulesets"]] == [RULESET["id"], "c3d1e5f7-0000-4000-8000-000000000001"] + assert [connection.id for connection in resources["connections"]] == [CONNECTION["id"]] + assert m.call_count == 2 + assert m.request_history[1].qs == {"limit": ["2"], "offset": ["2"]} + + +def test_list_resources_raises_for_a_caller_who_cannot_list_them(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get("http://test-server/api/resources/", status_code=403, json={"detail": "Forbidden"}) + + with pytest.raises(DataMasqueApiError) as raised: + client.list_resources() + + assert raised.value.response.status_code == 403 + + +# The move methods accept a listed `Resource` or a plain ID. +MOVING = { + ResourceTypeName.rulesets: [RulesetResource.model_validate(RULESET)], + ResourceTypeName.connections: [CONNECTION["id"]], +} +MOVE_REQUEST = { + "destination": "8", + "resources": [{"type": "rulesets", "id": RULESET["id"]}, {"type": "connections", "id": CONNECTION["id"]}], +} +NAME_CLASH = { + "reason": "name_in_use", + "detail": 'The destination project already contains a connection named "customers_db".', + "type": "connections", + "id": CONNECTION["id"], + "name": "customers_db", +} + + +def test_get_resource_move_conflicts_sends_the_resources_and_destination(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.post("http://test-server/api/resources/move-check/", json={"conflicts": [NAME_CLASH]}) + conflicts = client.get_resource_move_conflicts(MOVING, ProjectId(8)) + + assert m.last_request.json() == MOVE_REQUEST + assert len(conflicts) == 1 + assert conflicts[0].reason is ResourceMoveConflictReason.name_in_use + assert conflicts[0].type is ResourceTypeName.connections + assert conflicts[0].name == "customers_db" + + +def test_move_resources_returns_the_count_moved_and_the_conflicts(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.post("http://test-server/api/resources/move/", json={"moved": 1, "conflicts": [NAME_CLASH]}) + result = client.move_resources(MOVING, ProjectId(8)) + + assert m.last_request.json() == MOVE_REQUEST + assert result.moved == 1 + assert [conflict.id for conflict in result.conflicts] == [CONNECTION["id"]] + + +def test_move_resources_moves_what_list_resources_returned_to_a_project(client: DataMasqueClient) -> None: + retail = Project.model_validate( + { + "id": 8, + "reference": "8-retail", + "name": "Retail", + "created": "2026-10-01T10:00:00Z", + "updated": "2026-10-01T10:00:00Z", + } + ) + with requests_mock.Mocker() as m: + m.get("http://test-server/api/resources/", json=page({"rulesets": [RULESET], "connections": [CONNECTION]})) + m.post("http://test-server/api/resources/move/", json={"moved": 2, "conflicts": []}) + client.move_resources(client.list_resources(), retail) + + assert m.last_request.json() == MOVE_REQUEST + + +def test_move_resources_takes_a_listed_resources_project_as_the_destination(client: DataMasqueClient) -> None: + listed = RulesetResource.model_validate(RULESET) + with requests_mock.Mocker() as m: + m.post("http://test-server/api/resources/move/", json={"moved": 2, "conflicts": []}) + client.move_resources(MOVING, listed.project) + + assert m.last_request.json()["destination"] == "7" + + +@pytest.mark.parametrize("resources", [{}, {ResourceTypeName.rulesets: []}]) +def test_moving_no_resources_raises_without_a_request(client: DataMasqueClient, resources: dict) -> None: + with requests_mock.Mocker() as m: + with pytest.raises(ValueError, match="at least one resource"): + client.move_resources(resources, ProjectId(8)) + + with pytest.raises(ValueError, match="at least one resource"): + client.get_resource_move_conflicts(resources, ProjectId(8)) + + assert m.call_count == 0 diff --git a/tests/test_rg_configs.py b/tests/test_rg_configs.py index d393b84..82381b0 100644 --- a/tests/test_rg_configs.py +++ b/tests/test_rg_configs.py @@ -14,6 +14,7 @@ unwrap_rg_config_id, ) from datamasque.client.models.status import ValidationErrorType, ValidationStatus +from tests.helpers import PROJECT, PROJECT_ID CONFIG_ID_1 = "aaaaaaaa-1111-2222-3333-444444444444" CONFIG_ID_2 = "bbbbbbbb-1111-2222-3333-444444444444" @@ -31,6 +32,7 @@ def sample_config_list_response() -> dict[str, Any]: { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -38,6 +40,7 @@ def sample_config_list_response() -> dict[str, Any]: { "id": CONFIG_ID_2, "name": "another_config", + "project": PROJECT, "archived": False, "created": "2025-02-01T12:00:00Z", "modified": "2025-02-02T12:00:00Z", @@ -51,6 +54,7 @@ def sample_config_detail_response() -> dict[str, Any]: return { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "config_yaml": CONFIG_YAML, "archived": False, "created": "2025-01-01T12:00:00Z", @@ -60,7 +64,7 @@ def sample_config_detail_response() -> dict[str, Any]: @pytest.fixture def rg_config() -> RGConfig: - return RGConfig(name="test_config", yaml=CONFIG_YAML) + return RGConfig(name="test_config", project=PROJECT, yaml=CONFIG_YAML) def test_list_rg_configs(client: DataMasqueClient, sample_config_list_response: dict[str, Any]) -> None: @@ -88,6 +92,7 @@ def test_list_rg_configs_pagination(client: DataMasqueClient) -> None: { "id": CONFIG_ID_1, "name": "c1", + "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -95,6 +100,7 @@ def test_list_rg_configs_pagination(client: DataMasqueClient) -> None: { "id": CONFIG_ID_2, "name": "c2", + "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -109,6 +115,7 @@ def test_list_rg_configs_pagination(client: DataMasqueClient) -> None: { "id": "cccccccc-1111-2222-3333-444444444444", "name": "c3", + "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -159,6 +166,7 @@ def test_get_rg_config_by_name_found(client: DataMasqueClient, sample_config_det { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -177,11 +185,11 @@ def test_get_rg_config_by_name_found(client: DataMasqueClient, sample_config_det json=sample_config_detail_response, status_code=200, ) - config = client.get_rg_config_by_name("my_config") + config = client.get_rg_config_by_name("my_config", PROJECT_ID) assert config is not None assert config.name == "my_config" - assert "name_exact=my_config" in m.request_history[0].url + assert m.request_history[0].qs == {"name_exact": ["my_config"], "project": [str(PROJECT_ID)], "limit": ["1"]} def test_get_rg_config_by_name_not_found(client: DataMasqueClient) -> None: @@ -193,7 +201,7 @@ def test_get_rg_config_by_name_not_found(client: DataMasqueClient) -> None: json=empty_response, status_code=200, ) - config = client.get_rg_config_by_name("nonexistent") + config = client.get_rg_config_by_name("nonexistent", PROJECT_ID) assert config is None @@ -206,6 +214,7 @@ def test_get_rg_config_by_name_raises_when_server_omits_id(client: DataMasqueCli "results": [ { "name": "my_config", + "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -220,13 +229,14 @@ def test_get_rg_config_by_name_raises_when_server_omits_id(client: DataMasqueCli status_code=200, ) with pytest.raises(DataMasqueApiError, match="without an `id`"): - client.get_rg_config_by_name("my_config") + client.get_rg_config_by_name("my_config", PROJECT_ID) def test_create_rg_config(client: DataMasqueClient, rg_config: RGConfig) -> None: create_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": CONFIG_YAML, "is_valid": "valid", "archived": False, @@ -250,6 +260,7 @@ def test_create_rg_config(client: DataMasqueClient, rg_config: RGConfig) -> None request_body = m.last_request.json() assert request_body["name"] == "test_config" + assert request_body["project"] == PROJECT assert request_body["config_yaml"] == CONFIG_YAML # Server-managed read-only fields must not be echoed back in the request body. assert "id" not in request_body @@ -264,6 +275,7 @@ def test_create_and_update_rg_config_populate_validation_errors(client: DataMasq invalid_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": CONFIG_YAML, "is_valid": "invalid", "validation_errors": [ @@ -312,6 +324,7 @@ def test_update_rg_config(client: DataMasqueClient, rg_config: RGConfig) -> None update_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": CONFIG_YAML, "archived": False, "created": "2025-06-01T10:00:00Z", @@ -344,7 +357,7 @@ def test_update_rg_config_no_id_raises(client: DataMasqueClient, rg_config: RGCo def test_update_rg_config_no_yaml_raises(client: DataMasqueClient) -> None: """A config from a list response carries no YAML, so a PUT would silently blank the server's copy.""" - listed = RGConfig(name="test_config") + listed = RGConfig(name="test_config", project=PROJECT) listed.id = RGConfigId(CONFIG_ID_1) with pytest.raises(ValueError, match="yaml is None"): @@ -356,6 +369,7 @@ def test_create_or_update_rg_config_create(client: DataMasqueClient, rg_config: create_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": CONFIG_YAML, "archived": False, "created": "2025-06-01T10:00:00Z", @@ -369,7 +383,9 @@ def test_create_or_update_rg_config_create(client: DataMasqueClient, rg_config: assert result.id == RGConfigId(CONFIG_ID_1) assert m.request_history[0].method == "GET" + assert m.request_history[0].qs["project"] == [PROJECT] assert m.request_history[1].method == "POST" + assert m.request_history[1].json()["project"] == PROJECT def test_create_or_update_rg_config_update(client: DataMasqueClient, rg_config: RGConfig) -> None: @@ -381,6 +397,7 @@ def test_create_or_update_rg_config_update(client: DataMasqueClient, rg_config: { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "archived": False, "created": "2025-06-01T10:00:00Z", "modified": "2025-06-01T10:00:00Z", @@ -390,6 +407,7 @@ def test_create_or_update_rg_config_update(client: DataMasqueClient, rg_config: update_response = { "id": CONFIG_ID_1, "name": "test_config", + "project": PROJECT, "config_yaml": CONFIG_YAML, "archived": False, "created": "2025-06-01T10:00:00Z", @@ -433,6 +451,7 @@ def test_delete_rg_config_by_name(client: DataMasqueClient) -> None: { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -443,10 +462,10 @@ def test_delete_rg_config_by_name(client: DataMasqueClient) -> None: with requests_mock.Mocker() as m: m.get("http://test-server/api/ruleset-generation-configs/", json=list_response, status_code=200) m.delete(f"http://test-server/api/ruleset-generation-configs/{CONFIG_ID_1}/", status_code=204) - client.delete_rg_config_by_name_if_exists("my_config") + client.delete_rg_config_by_name_if_exists("my_config", PROJECT_ID) assert [r.method for r in m.request_history] == ["GET", "DELETE"] - assert "name_exact=my_config" in m.request_history[0].url + assert m.request_history[0].qs == {"name_exact": ["my_config"], "project": [str(PROJECT_ID)], "limit": ["1"]} def test_delete_rg_config_by_name_not_found(client: DataMasqueClient) -> None: @@ -454,7 +473,7 @@ def test_delete_rg_config_by_name_not_found(client: DataMasqueClient) -> None: with requests_mock.Mocker() as m: m.get("http://test-server/api/ruleset-generation-configs/", json=empty_response, status_code=200) - client.delete_rg_config_by_name_if_exists("nonexistent") + client.delete_rg_config_by_name_if_exists("nonexistent", PROJECT_ID) assert m.call_count == 1 assert m.request_history[0].method == "GET" @@ -479,6 +498,7 @@ def test_rg_config_parses_validation_fields() -> None: { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "config_yaml": "labels: []", "is_valid": "invalid", "validation_errors": [{"message": "bad shape on line 3", "line_number": 3, "column_number": 1}], @@ -497,6 +517,7 @@ def test_rg_config_validation_fields_optional() -> None: { "id": CONFIG_ID_1, "name": "my_config", + "project": PROJECT, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", } @@ -512,11 +533,11 @@ def test_unwrap_rg_config_id_passes_through_strings() -> None: def test_unwrap_rg_config_id_extracts_id_from_model() -> None: - config = RGConfig(name="x", id=RGConfigId(CONFIG_ID_1)) + config = RGConfig(name="x", project=PROJECT, id=RGConfigId(CONFIG_ID_1)) assert unwrap_rg_config_id(config) == CONFIG_ID_1 def test_unwrap_rg_config_id_raises_without_id() -> None: - config = RGConfig(name="x") + config = RGConfig(name="x", project=PROJECT) with pytest.raises(ValueError, match="id is None"): unwrap_rg_config_id(config) diff --git a/tests/test_ruleset_library.py b/tests/test_ruleset_library.py index b3fec81..29b558a 100644 --- a/tests/test_ruleset_library.py +++ b/tests/test_ruleset_library.py @@ -15,6 +15,7 @@ ValidationStatus, ) from datamasque.client.models.status import ValidationErrorDetails +from tests.helpers import PROJECT, PROJECT_ID LIBRARY_ID_1 = "aaaaaaaa-1111-2222-3333-444444444444" LIBRARY_ID_2 = "bbbbbbbb-1111-2222-3333-444444444444" @@ -31,7 +32,7 @@ def sample_library_list_response(): { "id": LIBRARY_ID_1, "name": "my_library", - "namespace": "org", + "project": PROJECT, "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -39,7 +40,7 @@ def sample_library_list_response(): { "id": LIBRARY_ID_2, "name": "another_library", - "namespace": "", + "project": PROJECT, "is_valid": "invalid", "created": "2025-02-01T12:00:00Z", "modified": "2025-02-02T12:00:00Z", @@ -54,7 +55,7 @@ def sample_library_detail_response(): return { "id": LIBRARY_ID_1, "name": "my_library", - "namespace": "org", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions:\n - name: my_func", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", @@ -66,7 +67,7 @@ def sample_library_detail_response(): def ruleset_library(): return RulesetLibrary( name="test_library", - namespace="test_ns", + project=PROJECT, yaml="version: '1.0'\nfunctions: []", ) @@ -83,7 +84,7 @@ def test_list_ruleset_libraries(client: DataMasqueClient, sample_library_list_re assert len(libraries) == 2 assert libraries[0].id == RulesetLibraryId(LIBRARY_ID_1) assert libraries[0].name == "my_library" - assert libraries[0].namespace == "org" + assert libraries[0].project == PROJECT assert libraries[0].yaml is None assert libraries[0].is_valid is ValidationStatus.valid assert libraries[1].id == RulesetLibraryId(LIBRARY_ID_2) @@ -100,7 +101,7 @@ def test_list_ruleset_libraries_pagination(client: DataMasqueClient) -> None: { "id": LIBRARY_ID_1, "name": "lib1", - "namespace": "", + "project": PROJECT, "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -108,7 +109,7 @@ def test_list_ruleset_libraries_pagination(client: DataMasqueClient) -> None: { "id": LIBRARY_ID_2, "name": "lib2", - "namespace": "", + "project": PROJECT, "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -123,7 +124,7 @@ def test_list_ruleset_libraries_pagination(client: DataMasqueClient) -> None: { "id": "cccccccc-1111-2222-3333-444444444444", "name": "lib3", - "namespace": "", + "project": PROJECT, "is_valid": "unknown", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -169,7 +170,7 @@ def test_get_ruleset_library(client: DataMasqueClient, sample_library_detail_res assert library.id == RulesetLibraryId(LIBRARY_ID_1) assert library.name == "my_library" - assert library.namespace == "org" + assert library.project == PROJECT assert library.yaml == "version: '1.0'\nfunctions:\n - name: my_func" assert library.is_valid is ValidationStatus.valid @@ -185,7 +186,7 @@ def test_get_ruleset_library_by_name_found( { "id": LIBRARY_ID_1, "name": "my_library", - "namespace": "org", + "project": PROJECT, "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -204,13 +205,12 @@ def test_get_ruleset_library_by_name_found( json=sample_library_detail_response, status_code=200, ) - library = client.get_ruleset_library_by_name("my_library", "org") + library = client.get_ruleset_library_by_name("my_library", PROJECT_ID) assert library is not None assert library.name == "my_library" assert library.yaml == "version: '1.0'\nfunctions:\n - name: my_func" - assert "name_exact=my_library" in m.request_history[0].url - assert "namespace_exact=org" in m.request_history[0].url + assert m.request_history[0].qs == {"name_exact": ["my_library"], "project": [str(PROJECT_ID)], "limit": ["1"]} def test_get_ruleset_library_by_name_raises_when_server_omits_id(client: DataMasqueClient) -> None: @@ -222,7 +222,7 @@ def test_get_ruleset_library_by_name_raises_when_server_omits_id(client: DataMas "results": [ { "name": "my_library", - "namespace": "org", + "project": PROJECT, "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -237,7 +237,7 @@ def test_get_ruleset_library_by_name_raises_when_server_omits_id(client: DataMas status_code=200, ) with pytest.raises(DataMasqueApiError, match="without an `id`"): - client.get_ruleset_library_by_name("my_library", "org") + client.get_ruleset_library_by_name("my_library", PROJECT_ID) def test_get_ruleset_library_by_name_not_found(client: DataMasqueClient) -> None: @@ -249,7 +249,7 @@ def test_get_ruleset_library_by_name_not_found(client: DataMasqueClient) -> None json=empty_response, status_code=200, ) - library = client.get_ruleset_library_by_name("nonexistent") + library = client.get_ruleset_library_by_name("nonexistent", PROJECT_ID) assert library is None @@ -258,7 +258,7 @@ def test_create_ruleset_library(client: DataMasqueClient, ruleset_library: Rules create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "unknown", "created": "2025-06-01T10:00:00Z", @@ -281,17 +281,35 @@ def test_create_ruleset_library(client: DataMasqueClient, ruleset_library: Rules request_body = m.last_request.json() assert request_body["name"] == "test_library" - assert request_body["namespace"] == "test_ns" + assert request_body["project"] == PROJECT assert request_body["config_yaml"] == "version: '1.0'\nfunctions: []" +def test_update_ruleset_library_does_not_send_namespace_back( + client: DataMasqueClient, sample_library_detail_response: dict[str, Any] +) -> None: + """A `namespace` the server returns is not sent back, because the server lets it override `project`.""" + with requests_mock.Mocker() as m: + m.get( + f"http://test-server/api/ruleset-libraries/{LIBRARY_ID_1}/", + json={**sample_library_detail_response, "namespace": "org"}, + ) + m.put(f"http://test-server/api/ruleset-libraries/{LIBRARY_ID_1}/", json=sample_library_detail_response) + library = client.get_ruleset_library(RulesetLibraryId(LIBRARY_ID_1)) + client.update_ruleset_library(library) + + request_body = m.last_request.json() + assert "namespace" not in request_body + assert request_body["project"] == PROJECT + + def test_update_ruleset_library(client: DataMasqueClient, ruleset_library: RulesetLibrary) -> None: ruleset_library.id = RulesetLibraryId(LIBRARY_ID_1) update_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "unknown", "created": "2025-06-01T10:00:00Z", @@ -329,7 +347,7 @@ def test_create_or_update_ruleset_library_create( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "unknown", "created": "2025-06-01T10:00:00Z", @@ -368,26 +386,17 @@ def test_create_or_update_ruleset_library_update( { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "is_valid": "valid", "created": "2025-06-01T10:00:00Z", "modified": "2025-06-01T10:00:00Z", }, ], } - detail_response = { - "id": LIBRARY_ID_1, - "name": "test_library", - "namespace": "test_ns", - "config_yaml": "version: '1.0'\nfunctions: []", - "is_valid": "valid", - "created": "2025-06-01T10:00:00Z", - "modified": "2025-06-01T10:00:00Z", - } update_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "unknown", "created": "2025-06-01T10:00:00Z", @@ -400,11 +409,6 @@ def test_create_or_update_ruleset_library_update( json=list_response, status_code=200, ) - m.get( - f"http://test-server/api/ruleset-libraries/{LIBRARY_ID_1}/", - json=detail_response, - status_code=200, - ) m.put( f"http://test-server/api/ruleset-libraries/{LIBRARY_ID_1}/", json=update_response, @@ -413,10 +417,8 @@ def test_create_or_update_ruleset_library_update( result = client.create_or_update_ruleset_library(ruleset_library) assert result.id == RulesetLibraryId(LIBRARY_ID_1) - # Should have called GET (list), GET (detail), then PUT (update) - assert m.request_history[0].method == "GET" - assert m.request_history[1].method == "GET" - assert m.request_history[2].method == "PUT" + assert [request.method for request in m.request_history] == ["GET", "PUT"] + assert m.request_history[0].qs == {"name_exact": ["test_library"], "project": [PROJECT], "limit": ["1"]} def test_delete_ruleset_library_by_id(client: DataMasqueClient) -> None: @@ -453,31 +455,35 @@ def test_delete_ruleset_library_by_id_force(client: DataMasqueClient) -> None: def test_delete_ruleset_library_by_name(client: DataMasqueClient, sample_library_list_response: dict[str, Any]) -> None: with requests_mock.Mocker() as m: + # The server filters by name, so only the matching library is listed. + only_match = { + **sample_library_list_response, + "count": 1, + "results": sample_library_list_response["results"][:1], + } m.get( "http://test-server/api/ruleset-libraries/", - json=sample_library_list_response, + json=only_match, status_code=200, ) m.delete( f"http://test-server/api/ruleset-libraries/{LIBRARY_ID_1}/", status_code=204, ) - client.delete_ruleset_library_by_name_if_exists("my_library", "org") + client.delete_ruleset_library_by_name_if_exists("my_library", PROJECT_ID) - assert m.request_history[0].method == "GET" - assert m.request_history[1].method == "DELETE" + assert m.request_history[0].qs["project"] == [str(PROJECT_ID)] + assert [request.method for request in m.request_history] == ["GET", "DELETE"] -def test_delete_ruleset_library_by_name_not_found( - client: DataMasqueClient, sample_library_list_response: dict[str, Any] -) -> None: +def test_delete_ruleset_library_by_name_not_found(client: DataMasqueClient) -> None: with requests_mock.Mocker() as m: m.get( "http://test-server/api/ruleset-libraries/", - json=sample_library_list_response, + json={"count": 0, "next": None, "previous": None, "results": []}, status_code=200, ) - client.delete_ruleset_library_by_name_if_exists("nonexistent") + client.delete_ruleset_library_by_name_if_exists("nonexistent", PROJECT_ID) # Only the list call should have been made, no DELETE assert m.call_count == 1 @@ -488,7 +494,7 @@ def test_validate_ruleset_library(client: DataMasqueClient) -> None: validate_response = { "id": LIBRARY_ID_1, "name": "my_library", - "namespace": "org", + "project": PROJECT, "config_yaml": "version: '1.0'", "is_valid": "unknown", "created": "2025-01-01T12:00:00Z", @@ -517,6 +523,7 @@ def test_list_rulesets_using_library(client: DataMasqueClient) -> None: { "id": "eeeeeeee-1111-2222-3333-444444444444", "name": "ruleset_a", + "project": PROJECT, "mask_type": "database", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", @@ -525,6 +532,7 @@ def test_list_rulesets_using_library(client: DataMasqueClient) -> None: { "id": "ffffffff-1111-2222-3333-444444444444", "name": "ruleset_b", + "project": PROJECT, "mask_type": "file", "is_valid": "unknown", "created": "2025-02-01T12:00:00Z", @@ -575,6 +583,7 @@ def test_list_rulesets_using_library_pagination(client: DataMasqueClient) -> Non { "id": "aaa", "name": "r1", + "project": PROJECT, "mask_type": "database", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", @@ -583,6 +592,7 @@ def test_list_rulesets_using_library_pagination(client: DataMasqueClient) -> Non { "id": "bbb", "name": "r2", + "project": PROJECT, "mask_type": "database", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", @@ -598,6 +608,7 @@ def test_list_rulesets_using_library_pagination(client: DataMasqueClient) -> Non { "id": "ccc", "name": "r3", + "project": PROJECT, "mask_type": "file", "is_valid": "unknown", "created": "2025-01-01T12:00:00Z", @@ -626,7 +637,7 @@ def test_create_ruleset_library_populates_validation_errors( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "invalid", "validation_errors": [{"message": "Invalid function definition"}], @@ -654,7 +665,7 @@ def test_update_ruleset_library_populates_validation_errors( update_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "invalid", "validation_errors": [{"message": "Invalid function definition"}], @@ -684,7 +695,7 @@ def test_create_ruleset_library_does_not_send_read_only_fields( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "invalid", "validation_errors": [{"message": "Invalid function definition"}], @@ -715,7 +726,7 @@ def test_create_ruleset_library_collapses_git_snapshot( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "valid", "git_branch": "main", @@ -752,7 +763,7 @@ def test_create_and_update_ruleset_library_populate_unlicensed_feature_warnings( response = { "id": LIBRARY_ID_1, "name": "test_library", - "namespace": "test_ns", + "project": PROJECT, "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "valid", "unlicensed_feature_warnings": returned, diff --git a/tests/test_rulesets.py b/tests/test_rulesets.py index f304439..b0a989d 100644 --- a/tests/test_rulesets.py +++ b/tests/test_rulesets.py @@ -8,6 +8,7 @@ from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.ruleset import RulesetType from datamasque.client.models.status import ValidationErrorType, ValidationStatus +from tests.helpers import PROJECT, PROJECT_ID def test_list_rulesets(client, existing_rulesets_json): @@ -34,7 +35,7 @@ def test_create_or_update_ruleset_create(client, ruleset): # Test creating a new ruleset with upsert m.post( "http://test-server/api/rulesets/?upsert=true", - json={"id": "2", "name": "test_ruleset", "is_valid": "in_progress"}, + json={"id": "2", "name": "test_ruleset", "project": PROJECT, "is_valid": "in_progress"}, status_code=201, ) @@ -46,6 +47,7 @@ def test_create_or_update_ruleset_create(client, ruleset): sent = m.last_request.json() assert sent["config_yaml"] == "version: '1.0'\ntasks: []" assert sent["mask_type"] == "database" + assert sent["project"] == PROJECT def test_create_or_update_ruleset_create_fail(client, ruleset): @@ -64,7 +66,7 @@ def test_create_or_update_ruleset_update(client, ruleset): # Test updating an existing ruleset with upsert (returns 200 status for update) m.post( "http://test-server/api/rulesets/?upsert=true", - json={"id": "1", "name": "test_ruleset", "is_valid": "valid"}, + json={"id": "1", "name": "test_ruleset", "project": PROJECT, "is_valid": "valid"}, status_code=200, ) @@ -94,41 +96,31 @@ def test_delete_ruleset_by_id(client): def test_delete_ruleset_by_name(client, existing_rulesets_json): + """The server filters by name, type and project, so a same-named ruleset elsewhere is never returned.""" with requests_mock.Mocker() as m: m.get( - "http://test-server/api/v2/rulesets/", - json=existing_rulesets_json, + f"http://test-server/api/v2/rulesets/?name_exact=file_masking_ruleset&mask_type=file&project={PROJECT_ID}", + json=[existing_rulesets_json[1]], status_code=200, + complete_qs=True, ) m.delete("http://test-server/api/rulesets/2/", status_code=204) - client.delete_ruleset_by_name_if_exists("file_masking_ruleset", RulesetType.file) + client.delete_ruleset_by_name_if_exists("file_masking_ruleset", RulesetType.file, PROJECT_ID) assert m.call_count == 2 assert m.request_history[0].method == "GET" assert m.request_history[1].method == "DELETE" -def test_delete_ruleset_by_name_skips_other_type(client, existing_rulesets_json): - with requests_mock.Mocker() as m: - m.get( - "http://test-server/api/v2/rulesets/", - json=existing_rulesets_json, - status_code=200, - ) - client.delete_ruleset_by_name_if_exists("file_masking_ruleset", RulesetType.database) - - assert m.call_count == 1 - assert m.request_history[0].method == "GET" - - -def test_delete_ruleset_that_does_not_exist(client, existing_rulesets_json): +def test_delete_ruleset_that_does_not_exist(client): with requests_mock.Mocker() as m: m.get( - "http://test-server/api/v2/rulesets/", - json=existing_rulesets_json, + f"http://test-server/api/v2/rulesets/?name_exact=not_a_ruleset&mask_type=database&project={PROJECT_ID}", + json=[], status_code=200, + complete_qs=True, ) - client.delete_ruleset_by_name_if_exists("not_a_ruleset", RulesetType.database) + client.delete_ruleset_by_name_if_exists("not_a_ruleset", RulesetType.database, PROJECT_ID) assert m.call_count == 1 assert m.request_history[0].method == "GET" @@ -142,6 +134,7 @@ def test_create_or_update_ruleset_populates_validation_errors(client, ruleset): json={ "id": "2", "name": "test_ruleset", + "project": PROJECT, "is_valid": "invalid", "validation_errors": [ { @@ -169,7 +162,7 @@ def test_create_or_update_ruleset_validation_errors_empty_when_valid(client, rul with requests_mock.Mocker() as m: m.post( "http://test-server/api/rulesets/?upsert=true", - json={"id": "2", "name": "test_ruleset", "is_valid": "valid"}, + json={"id": "2", "name": "test_ruleset", "project": PROJECT, "is_valid": "valid"}, status_code=201, ) result = client.create_or_update_ruleset(ruleset) @@ -195,7 +188,13 @@ def test_create_or_update_ruleset_populates_unlicensed_feature_warnings(client, with requests_mock.Mocker() as m: m.post( "http://test-server/api/rulesets/?upsert=true", - json={"id": "2", "name": "test_ruleset", "is_valid": "valid", "unlicensed_feature_warnings": returned}, + json={ + "id": "2", + "name": "test_ruleset", + "project": PROJECT, + "is_valid": "valid", + "unlicensed_feature_warnings": returned, + }, status_code=201, ) result = client.create_or_update_ruleset(ruleset) @@ -208,7 +207,7 @@ def test_create_or_update_ruleset_unlicensed_feature_warnings_default_to_none(cl with requests_mock.Mocker() as m: m.post( "http://test-server/api/rulesets/?upsert=true", - json={"id": "2", "name": "test_ruleset", "is_valid": "in_progress"}, + json={"id": "2", "name": "test_ruleset", "project": PROJECT, "is_valid": "in_progress"}, status_code=201, ) result = client.create_or_update_ruleset(ruleset) @@ -225,6 +224,7 @@ def test_list_rulesets_unlicensed_feature_warnings(client): { "id": "1", "name": "unstructured_ruleset", + "project": PROJECT, "mask_type": "database", "is_valid": "valid", "unlicensed_feature_warnings": [ @@ -234,6 +234,7 @@ def test_list_rulesets_unlicensed_feature_warnings(client): { "id": "2", "name": "licensed_ruleset", + "project": PROJECT, "mask_type": "database", "is_valid": "valid", "unlicensed_feature_warnings": [], @@ -241,6 +242,7 @@ def test_list_rulesets_unlicensed_feature_warnings(client): { "id": "3", "name": "unvalidated_ruleset", + "project": PROJECT, "mask_type": "database", "is_valid": "in_progress", "unlicensed_feature_warnings": None, @@ -265,6 +267,7 @@ def test_create_or_update_ruleset_does_not_send_read_only_fields(client, ruleset json={ "id": "2", "name": "test_ruleset", + "project": PROJECT, "is_valid": "invalid", "validation_errors": [{"message": "bad", "validation_error_type": "ruleset"}], "unlicensed_feature_warnings": [ @@ -283,6 +286,7 @@ def test_create_or_update_ruleset_does_not_send_read_only_fields(client, ruleset assert read_only_field not in body # Input fields are still present. assert body["config_yaml"] == "version: '1.0'\ntasks: []" + assert body["project"] == PROJECT def test_create_or_update_ruleset_collapses_git_snapshot(client, ruleset): @@ -293,6 +297,7 @@ def test_create_or_update_ruleset_collapses_git_snapshot(client, ruleset): json={ "id": "2", "name": "test_ruleset", + "project": PROJECT, "is_valid": "valid", "git_branch": "main", "git_commit_sha": "abc123", @@ -318,6 +323,7 @@ def test_create_or_update_ruleset_git_none_when_not_synced(client, ruleset): json={ "id": "2", "name": "test_ruleset", + "project": PROJECT, "is_valid": "valid", "git_branch": None, "git_commit_sha": None, @@ -340,6 +346,7 @@ def test_list_rulesets_collapses_git_snapshot(client): { "id": "1", "name": "synced_ruleset", + "project": PROJECT, "mask_type": "database", "config_yaml": "version: '1.0'", "is_valid": "valid", @@ -351,6 +358,7 @@ def test_list_rulesets_collapses_git_snapshot(client): { "id": "2", "name": "local_ruleset", + "project": PROJECT, "mask_type": "database", "config_yaml": "version: '1.0'", "is_valid": "valid", diff --git a/tests/test_runs.py b/tests/test_runs.py index 295409a..85f1aff 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -3,8 +3,10 @@ import pytest import requests_mock +from datamasque.client import DataMasqueClient from datamasque.client.exceptions import ( DataMasqueApiError, + DataMasqueException, FailedToStartError, InvalidLibraryError, InvalidRulesetError, @@ -21,7 +23,7 @@ UnfinishedRun, ) from datamasque.client.models.status import MaskingRunStatus -from tests.helpers import fake +from tests.helpers import PROJECT, fake def test_get_run_log(client): @@ -92,78 +94,101 @@ def test_unfinished_run_str_without_destination(): assert str(run) == '"source_db": Run ID 42 in status `queued`, ruleset "my_ruleset"' -def test_get_unfinished_runs(client): +QUEUED_SOURCE_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a01") +QUEUED_DESTINATION_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a02") +RUNNING_SOURCE_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a03") +RUNNING_DESTINATION_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a04") +VALIDATING_SOURCE_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a05") +VALIDATING_DESTINATION_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a06") +CANCELLING_SOURCE_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a07") + +UNFINISHED_RUNS_URL = ( + "http://test-server/api/runs/?connection_ruleset_name=&ruleset_name=&run_status={}&limit=1&offset=0" +) + + +def unfinished_run_json( + run_id: int, status: str, source: tuple[str, str], destination: tuple[str, str] +) -> dict[str, object]: + """ + Builds an unfinished-run list entry from `(id, name)` pairs. + + An empty ID is sent as `None`, and an empty destination name means none. + """ + return { + "id": run_id, + "source_connection": source[0] or None, + "source_connection_name": source[1], + "destination_connection": destination[0] or None, + "destination_connection_name": destination[1], + "ruleset_name": f"ruleset_{run_id}", + "project": PROJECT, + "status": status, + } + + +def test_get_unfinished_runs(client: DataMasqueClient) -> None: + runs = { + "queued": unfinished_run_json( + 1, "queued", (QUEUED_SOURCE_ID, "queued_src"), (QUEUED_DESTINATION_ID, "queued_dst") + ), + "running": unfinished_run_json( + 2, "running", (RUNNING_SOURCE_ID, "running_src"), (RUNNING_DESTINATION_ID, "running_dst") + ), + "validating": unfinished_run_json( + 3, "validating", (VALIDATING_SOURCE_ID, "validating_src"), (VALIDATING_DESTINATION_ID, "validating_dst") + ), + "cancelling": unfinished_run_json(4, "cancelling", (CANCELLING_SOURCE_ID, "cancelling_src"), ("", "")), + } with requests_mock.Mocker() as m: - m.get( - "http://test-server/api/runs/?connection_ruleset_name=&ruleset_name=&run_status=queued&limit=1&offset=0", - json={ - "results": [ - { - "source_connection_name": "queued_src", - "destination_connection_name": "queued_dst", - "id": 1, - "ruleset_name": "ruleset_1", - "project": "1-default", - "status": "queued", - } - ] - }, - status_code=200, - ) - m.get( - "http://test-server/api/runs/?connection_ruleset_name=&ruleset_name=&run_status=running&limit=1&offset=0", - json={ - "results": [ - { - "source_connection_name": "running_src", - "destination_connection_name": "running_dst", - "id": 2, - "ruleset_name": "ruleset_2", - "project": "1-default", - "status": "running", - } - ] - }, - status_code=200, - ) - m.get( - "http://test-server/api/runs/?connection_ruleset_name=&ruleset_name=&run_status=validating&limit=1&offset=0", - json={ - "results": [ - { - "source_connection_name": "validating_src", - "destination_connection_name": "validating_dst", - "id": 3, - "ruleset_name": "ruleset_3", - "project": "1-default", - "status": "validating", - } - ] - }, - status_code=200, - ) - m.get( - "http://test-server/api/runs/?connection_ruleset_name=&ruleset_name=&run_status=cancelling&limit=1&offset=0", - json={ - "results": [ - { - "source_connection_name": "cancelling_src", - "destination_connection_name": "", - "id": 4, - "ruleset_name": "ruleset_4", - "project": "1-default", - "status": "cancelling", - } - ] - }, - status_code=200, - ) + for status, run in runs.items(): + m.get(UNFINISHED_RUNS_URL.format(status), json={"results": [run]}, status_code=200) + + ur = client.get_unfinished_runs() + + # 3 statuses have both source and destination keys, cancelling has only source (empty destination) + assert {connection_id: run.id for connection_id, run in ur.items()} == { + QUEUED_SOURCE_ID: 1, + QUEUED_DESTINATION_ID: 1, + RUNNING_SOURCE_ID: 2, + RUNNING_DESTINATION_ID: 2, + VALIDATING_SOURCE_ID: 3, + VALIDATING_DESTINATION_ID: 3, + CANCELLING_SOURCE_ID: 4, + } + + +def test_get_unfinished_runs_keeps_same_named_connections_apart(client: DataMasqueClient) -> None: + """Two connections with the same name in different projects each keep their own run.""" + runs = { + "queued": unfinished_run_json(1, "queued", (QUEUED_SOURCE_ID, "orders_db"), ("", "")), + "running": unfinished_run_json(2, "running", (RUNNING_SOURCE_ID, "orders_db"), ("", "")), + } + with requests_mock.Mocker() as m: + for status in ("queued", "running", "validating", "cancelling"): + results = [runs[status]] if status in runs else [] + m.get(UNFINISHED_RUNS_URL.format(status), json={"results": results}, status_code=200) ur = client.get_unfinished_runs() - # 3 statuses have both source and destination keys, cancelling has only source (empty destination) - assert len(ur) == 7 - for run in ur.values(): - assert isinstance(run, UnfinishedRun) + + assert {connection_id: run.id for connection_id, run in ur.items()} == {QUEUED_SOURCE_ID: 1, RUNNING_SOURCE_ID: 2} + + +@pytest.mark.parametrize( + "source, destination, missing", + [ + (("", "orders_db"), ("", ""), "source"), + ((QUEUED_SOURCE_ID, "orders_db"), ("", "orders_copy"), "destination"), + ], +) +def test_get_unfinished_runs_without_connection_id_raises( + client: DataMasqueClient, source: tuple[str, str], destination: tuple[str, str], missing: str +) -> None: + run = unfinished_run_json(1, "queued", source, destination) + with requests_mock.Mocker() as m: + m.get(UNFINISHED_RUNS_URL.format("queued"), json={"results": [run]}, status_code=200) + with pytest.raises(DataMasqueException, match=f"without a {missing} connection ID"): + client.get_unfinished_runs() def test_start_masking_run(client): @@ -360,6 +385,7 @@ def test_masking_run_request_requires_ruleset(): def test_masking_run_request_accepts_connection_config_and_ruleset_objects(): """Callers may pass full `ConnectionConfig` / `Ruleset` objects; their IDs are extracted at construction.""" connection = DatabaseConnectionConfig( + project=PROJECT, id=ConnectionId("conn-uuid"), name="prod_db", db_type=DatabaseType.postgres, @@ -369,6 +395,7 @@ def test_masking_run_request_accepts_connection_config_and_ruleset_objects(): user="masker", ) dest = DatabaseConnectionConfig( + project=PROJECT, id=ConnectionId("dest-uuid"), name="staging_db", db_type=DatabaseType.postgres, @@ -378,6 +405,7 @@ def test_masking_run_request_accepts_connection_config_and_ruleset_objects(): user="masker", ) ruleset = Ruleset( + project=PROJECT, id=RulesetId("rs-uuid"), name="my_ruleset", yaml="version: '1.0'", @@ -395,6 +423,7 @@ def test_masking_run_request_accepts_connection_config_and_ruleset_objects(): def test_masking_run_request_rejects_unpersisted_connection(): """A `ConnectionConfig` without an `id` means the caller hasn't created it yet — raise at construction.""" connection = DatabaseConnectionConfig( + project=PROJECT, name="not_yet_created", db_type=DatabaseType.postgres, host="localhost", @@ -408,7 +437,7 @@ def test_masking_run_request_rejects_unpersisted_connection(): def test_masking_run_request_rejects_unpersisted_ruleset(): """Same check on the ruleset side.""" - ruleset = Ruleset(name="fresh_ruleset", yaml="version: '1.0'", ruleset_type=RulesetType.database) + ruleset = Ruleset(project=PROJECT, name="fresh_ruleset", yaml="version: '1.0'", ruleset_type=RulesetType.database) with pytest.raises(ValueError, match="has not been created yet"): MaskingRunRequest(connection="conn-1", ruleset=ruleset) diff --git a/tests/test_table_references.py b/tests/test_table_references.py index 1a7968f..a2af76d 100644 --- a/tests/test_table_references.py +++ b/tests/test_table_references.py @@ -15,7 +15,7 @@ TableReferenceOptions, ) from datamasque.client.exceptions import DataMasqueException -from tests.helpers import file_connection_config +from tests.helpers import PROJECT, PROJECT_ID, file_connection_config TABLE_REFERENCE_ID_1 = "aaaaaaaa-1111-2222-3333-444444444444" TABLE_REFERENCE_ID_2 = "bbbbbbbb-1111-2222-3333-444444444444" @@ -38,6 +38,7 @@ def sample_table_reference_list_response() -> list[dict[str, object]]: { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -47,6 +48,7 @@ def sample_table_reference_list_response() -> list[dict[str, object]]: { "id": TABLE_REFERENCE_ID_2, "name": "account_identities", + "project": PROJECT, "connection": CONNECTION_ID_2, "source": "OPS.ACCOUNTS", "options": { @@ -66,6 +68,7 @@ def sample_table_reference_list_response() -> list[dict[str, object]]: def table_reference() -> TableReference: return TableReference( name="customer_identities", + project=PROJECT, connection=ConnectionId(CONNECTION_ID_1), source="identities/customers.csv", ) @@ -73,7 +76,7 @@ def table_reference() -> TableReference: def test_name_connection_and_source_are_required_on_the_model() -> None: with pytest.raises(ValidationError, match="source"): - TableReference(name="customer_identities", connection=ConnectionId(CONNECTION_ID_1)) + TableReference(name="customer_identities", project=PROJECT, connection=ConnectionId(CONNECTION_ID_1)) def test_options_default_to_the_servers_defaults() -> None: @@ -90,7 +93,9 @@ def test_connection_accepts_a_connection_config() -> None: connection = file_connection_config() connection.id = ConnectionId(CONNECTION_ID_1) - reference = TableReference(name="customer_identities", connection=connection, source="customers.csv") + reference = TableReference( + name="customer_identities", project=PROJECT, connection=connection, source="customers.csv" + ) assert reference.connection == ConnectionId(CONNECTION_ID_1) assert reference.connection_id == ConnectionId(CONNECTION_ID_1) @@ -98,7 +103,9 @@ def test_connection_accepts_a_connection_config() -> None: def test_connection_config_without_an_id_is_rejected() -> None: with pytest.raises(ValidationError, match="id is None"): - TableReference(name="customer_identities", connection=file_connection_config(), source="customers.csv") + TableReference( + name="customer_identities", project=PROJECT, connection=file_connection_config(), source="customers.csv" + ) def test_list_table_references( @@ -130,6 +137,21 @@ def test_list_table_references( assert references[1].options.null_string == "NULL" +def test_list_table_references_in_a_project( + client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, object]] +) -> None: + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/table-references/", + json=sample_table_reference_list_response, + status_code=200, + ) + references = client.list_table_references(PROJECT_ID) + + assert m.last_request.qs == {"project": [str(PROJECT_ID)]} + assert [reference.project for reference in references] == [PROJECT, PROJECT] + + def test_list_table_references_empty(client: DataMasqueClient) -> None: with requests_mock.Mocker() as m: m.get( @@ -168,11 +190,12 @@ def test_get_table_reference_by_name_found( json=sample_table_reference_list_response, status_code=200, ) - reference = client.get_table_reference_by_name("account_identities") + reference = client.get_table_reference_by_name("account_identities", PROJECT_ID) assert reference is not None assert reference.id == TableReferenceId(TABLE_REFERENCE_ID_2) assert m.call_count == 1 + assert m.last_request.qs == {"project": [str(PROJECT_ID)]} def test_get_table_reference_by_name_not_found( @@ -184,7 +207,7 @@ def test_get_table_reference_by_name_not_found( json=sample_table_reference_list_response, status_code=200, ) - reference = client.get_table_reference_by_name("nonexistent") + reference = client.get_table_reference_by_name("nonexistent", PROJECT_ID) assert reference is None @@ -193,6 +216,7 @@ def test_create_table_reference(client: DataMasqueClient, table_reference: Table create_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -218,6 +242,7 @@ def test_create_table_reference(client: DataMasqueClient, table_reference: Table request_body = m.last_request.json() assert request_body["name"] == "customer_identities" + assert request_body["project"] == PROJECT assert request_body["connection"] == CONNECTION_ID_1 assert request_body["source"] == "identities/customers.csv" assert "options" not in request_body @@ -225,9 +250,32 @@ def test_create_table_reference(client: DataMasqueClient, table_reference: Table assert not read_only_fields & request_body.keys() +def test_create_table_reference_without_a_project(client: DataMasqueClient) -> None: + """No `project` is sent, so the server uses its default project, which is read back.""" + reference = TableReference( + name="customer_identities", connection=ConnectionId(CONNECTION_ID_1), source="identities/customers.csv" + ) + create_response = { + "id": TABLE_REFERENCE_ID_1, + "name": "customer_identities", + "project": PROJECT, + "connection": CONNECTION_ID_1, + "source": "identities/customers.csv", + "options": DEFAULT_OPTIONS_JSON, + } + + with requests_mock.Mocker() as m: + m.post("http://test-server/api/table-references/", json=create_response, status_code=201) + client.create_table_reference(reference) + + assert "project" not in m.last_request.json() + assert reference.project == PROJECT + + def test_create_table_reference_sends_options(client: DataMasqueClient) -> None: reference = TableReference( name="customer_identities", + project=PROJECT, connection=ConnectionId(CONNECTION_ID_1), source="identities/customers.csv", options=TableReferenceOptions(format=TableReferenceFormat.parquet, delimiter=";", null_string="NULL"), @@ -235,6 +283,7 @@ def test_create_table_reference_sends_options(client: DataMasqueClient) -> None: create_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": { @@ -266,6 +315,7 @@ def test_create_table_reference_sends_null_string_default(client: DataMasqueClie """`null_string`'s default of `None` must be sent explicitly, not dropped from the payload.""" reference = TableReference( name="customer_identities", + project=PROJECT, connection=ConnectionId(CONNECTION_ID_1), source="identities/customers.csv", options=TableReferenceOptions(), @@ -273,6 +323,7 @@ def test_create_table_reference_sends_null_string_default(client: DataMasqueClie create_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -300,6 +351,7 @@ def test_update_table_reference(client: DataMasqueClient, table_reference: Table update_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers_v2.csv", "options": DEFAULT_OPTIONS_JSON, @@ -333,6 +385,7 @@ def test_update_table_reference_without_options_sends_none( update_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": {**DEFAULT_OPTIONS_JSON, "delimiter": ";"}, @@ -363,6 +416,7 @@ def test_create_or_update_table_reference_create(client: DataMasqueClient, table create_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -385,7 +439,9 @@ def test_create_or_update_table_reference_create(client: DataMasqueClient, table assert result.id == TableReferenceId(TABLE_REFERENCE_ID_1) assert m.request_history[0].method == "GET" + assert m.request_history[0].qs == {"project": [PROJECT]} assert m.request_history[1].method == "POST" + assert m.request_history[1].json()["project"] == PROJECT def test_create_or_update_table_reference_update( @@ -396,6 +452,7 @@ def test_create_or_update_table_reference_update( update_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -428,6 +485,7 @@ def test_create_or_update_table_reference_raises_when_server_omits_id( list_response_without_id = [ { "name": "customer_identities", + "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -479,10 +537,11 @@ def test_delete_table_reference_by_name( f"http://test-server/api/table-references/{TABLE_REFERENCE_ID_2}/", status_code=204, ) - client.delete_table_reference_by_name_if_exists("account_identities") + client.delete_table_reference_by_name_if_exists("account_identities", PROJECT_ID) assert m.call_count == 2 assert m.request_history[0].method == "GET" + assert m.request_history[0].qs == {"project": [str(PROJECT_ID)]} assert m.request_history[1].method == "DELETE" assert TABLE_REFERENCE_ID_2 in m.request_history[1].url @@ -496,7 +555,7 @@ def test_delete_table_reference_by_name_not_found( json=sample_table_reference_list_response, status_code=200, ) - client.delete_table_reference_by_name_if_exists("nonexistent") + client.delete_table_reference_by_name_if_exists("nonexistent", PROJECT_ID) assert m.call_count == 1 assert m.request_history[0].method == "GET" diff --git a/uv.lock b/uv.lock index 7164d65..c712118 100644 --- a/uv.lock +++ b/uv.lock @@ -419,7 +419,7 @@ toml = [ [[package]] name = "datamasque-python" -version = "1.5.0" +version = "1.7.0" source = { editable = "." } dependencies = [ { name = "pydantic" },