diff --git a/HISTORY.rst b/HISTORY.rst index 7dea79c..191e613 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,39 +2,8 @@ History ======= -1.7.0 (2026-11-06) ------------------- - -* Added project models and management APIs. -* Added resource management APIs. -* Added ``default_project`` to ``DataMasqueInstanceConfig``. -* ``ConnectionConfig``, ``Ruleset``, ``RulesetLibrary``, ``DiscoveryConfig``, ``DiscoveryConfigLibrary``, ``RGConfig`` - and ``TableReference`` send their ``project`` to the server when it is set. - It takes a project's ``-`` reference, such as ``Project.reference``, or its ID as a string. - Left unset, a create goes into the client's ``default_project``, else the server's default project. - Each create sets the model's ``project`` to the project the server stored it in. -* **Breaking:** names are unique per project, so every by-name helper takes a required ``project``: - ``get_*_by_name``, ``delete_*_by_name_if_exists`` and ``get_file_of_type_by_name``. - These ``project`` parameters take a ``Project`` or its ``ProjectId``. - ``create_or_update_*`` updates an existing resource only when it has the same name in the same project. - If the model has no ``project``, it looks in the client's ``default_project``, else the server's only active project, - and raises ``DataMasqueException`` when there is no active project or more than one. -* **Breaking:** ``upload_file`` and ``upload_file_if_not_exists`` take a required ``project``. -* **Breaking:** ``namespace`` is removed from ``RulesetLibrary`` and ``DiscoveryConfigLibrary``, - and the client drops it from server responses so that it is never sent back. - The library by-name helpers take ``project`` in its place. -* **Breaking:** ``get_unfinished_runs`` returns its runs keyed by ``ConnectionId`` instead of connection name, - because two connections in different projects can share a name. -* ``list_connections``, ``list_table_references`` and ``list_files_of_type`` take an optional ``project`` filter. -* ``DataMasqueFile`` gains a read-only ``project`` field, which is ``None`` for built-in seeds. -* ``get_generated_rulesets`` sets each ruleset's ``project`` to the connection's project. -* Added Git APIs for rulesets, ruleset libraries, discovery configs and discovery config libraries. -* Added bundle export and import APIs. - -Requires server version 3.26.21 - -1.6.0 ------ +1.6.0 (unreleased) +------------------ * Added a read-only ``project`` field to ``ConnectionConfig``, ``Ruleset``, ``RulesetLibrary``, ``DiscoveryConfig``, ``DiscoveryConfigLibrary``, ``RGConfig`` and ``TableReference``: diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index 5399310..2122c6d 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -23,21 +23,10 @@ InvalidDiscoveryConfigError, InvalidLibraryError, InvalidRulesetError, - ProjectDeleteConflictError, RGConfigNotFoundError, RunNotCancellableError, ) -from datamasque.client.git import GitExtension from datamasque.client.ifm import DataMasqueIfmClient -from datamasque.client.models.bundle import ( - BundleImportCounts, - BundleImportItemResults, - BundleImportResult, - BundleImportStatus, - LibraryImportResult, - RulesetImportResult, - SeedFileImportResult, -) from datamasque.client.models.connection import ( AzureConnectionConfig, CassandraConnectionConfig, @@ -111,7 +100,7 @@ SnowflakeKeyFile, SslZipFile, ) -from datamasque.client.models.git import GitFile, GitFileCommit, GitSnapshot +from datamasque.client.models.git import GitSnapshot from datamasque.client.models.ifm import ( DataMasqueIfmInstanceConfig, IfmLog, @@ -126,27 +115,6 @@ RulesetPlanUpdateRequest, ) from datamasque.client.models.license import LicenseInfo, SwitchableLicenseMetadata -from datamasque.client.models.project import ( - Project, - ProjectCreateRequest, - ProjectDeleteConflict, - ProjectDeleteConflictReason, - ProjectId, - ProjectIdentifier, - ProjectUpdateRequest, -) -from datamasque.client.models.resource import ( - ConnectionResource, - DiscoveryConfigResource, - Resource, - ResourceMoveConflict, - ResourceMoveConflictReason, - ResourceMoveResult, - ResourceOrdering, - ResourcesByType, - RulesetResource, -) -from datamasque.client.models.resource_type import ResourceTypeName from datamasque.client.models.rg_config import RGConfig, RGConfigId from datamasque.client.models.ruleset import Ruleset, RulesetId, RulesetType from datamasque.client.models.ruleset_library import RulesetLibrary, RulesetLibraryId @@ -214,17 +182,12 @@ "AzureConnectionConfig", "BooleanPreview", "BooleanStatistics", - "BundleImportCounts", - "BundleImportItemResults", - "BundleImportResult", - "BundleImportStatus", "CassandraConnectionConfig", "ColumnKind", "ColumnPreview", "CommonStatistics", "ConnectionConfig", "ConnectionId", - "ConnectionResource", "ConstraintColumns", "DataMasqueApiError", "DataMasqueClient", @@ -245,7 +208,6 @@ "DiscoveryConfigLibrary", "DiscoveryConfigLibraryId", "DiscoveryConfigNotFoundError", - "DiscoveryConfigResource", "DiscoveryConfigType", "DiscoveryMatch", "CosmosDbConnectionConfig", @@ -269,9 +231,6 @@ "FirstCharsStatistics", "ForeignKeyRef", "GcsConnectionConfig", - "GitExtension", - "GitFile", - "GitFileCommit", "GitSnapshot", "HashColumnsTableConfig", "IfmAuthError", @@ -289,7 +248,6 @@ "LengthEntry", "LengthUnit", "LengthsStatistics", - "LibraryImportResult", "LicenseInfo", "LicenseLock", "Locator", @@ -311,30 +269,14 @@ "PatternComposition", "PatternEntry", "PatternsStatistics", - "Project", - "ProjectCreateRequest", - "ProjectDeleteConflict", - "ProjectDeleteConflictError", - "ProjectDeleteConflictReason", - "ProjectId", - "ProjectIdentifier", - "ProjectUpdateRequest", "RGConfig", "RGConfigId", "RGConfigNotFoundError", "ReferencingForeignKey", - "Resource", - "ResourceMoveConflict", - "ResourceMoveConflictReason", - "ResourceMoveResult", - "ResourceOrdering", - "ResourceTypeName", - "ResourcesByType", "Ruleset", "RulesetGenerationRequest", "RulesetGenerationWithRGConfigRequest", "RulesetId", - "RulesetImportResult", "RulesetLibrary", "RulesetLibraryId", "RulesetPlan", @@ -342,7 +284,6 @@ "RulesetPlanOptions", "RulesetPlanPartialUpdateRequest", "RulesetPlanUpdateRequest", - "RulesetResource", "RulesetType", "RunConnectionRef", "RunId", @@ -359,7 +300,6 @@ "SchemaDiscoveryRequest", "SchemaDiscoveryResult", "SeedFile", - "SeedFileImportResult", "SelectedColumns", "SelectedData", "SelectedFileData", diff --git a/datamasque/client/base.py b/datamasque/client/base.py index d45ebb7..673356d 100644 --- a/datamasque/client/base.py +++ b/datamasque/client/base.py @@ -17,13 +17,10 @@ from datamasque.client.exceptions import ( DataMasqueApiError, - DataMasqueException, DataMasqueNotReadyError, DataMasqueTransportError, ) from datamasque.client.models.dm_instance import DataMasqueInstanceConfig -from datamasque.client.models.pagination import Page -from datamasque.client.models.project import Project, ProjectId from datamasque.client.spcs import install_spcs_gateway_auth logger = logging.getLogger(__name__) @@ -133,7 +130,6 @@ class BaseClient: password: Optional[str] verify_ssl: bool token_source: Optional[Callable[[], str]] - default_project: Optional[str] def __init__(self, connection_config: DataMasqueInstanceConfig) -> None: self.base_url = connection_config.base_url @@ -141,7 +137,6 @@ def __init__(self, connection_config: DataMasqueInstanceConfig) -> None: self.password = connection_config.password self.verify_ssl = connection_config.verify_ssl self.token_source = connection_config.token_source - self.default_project = connection_config.default_project self._session = _build_session(self.verify_ssl) self._timeout = (connection_config.connect_timeout_seconds, connection_config.read_timeout_seconds) if connection_config.spcs_pat: @@ -318,40 +313,6 @@ def _delete_if_exists(self, path: str, *, params: Optional[dict] = None) -> None self._raise_for_status(response) - def _resolve_project(self, project: Optional[Union[ProjectId, str]]) -> Union[ProjectId, str]: - """ - Returns `project`, else the client's `default_project`, else the ID of the only active project. - - The last is the project the server puts a create into when it is sent no `project`. - With no active project, or more than one, the server rejects such a create, - so this raises `DataMasqueException` instead. - """ - - if project is not None: - return project - if self.default_project is not None: - return self.default_project - - response = self.make_request("GET", "/api/projects/", params={"limit": 1}) - page = Page[Project].model_validate(response.json()) - if page.count != 1: - raise DataMasqueException( - f"No `project` is set and the server has {page.count} active projects. Set `project` to choose one." - ) - - return page.results[0].id - - def _default_project_field(self, project: Optional[Union[ProjectId, str]]) -> dict[str, str]: - """ - Returns the `project` field a create sends when the resource names no `project`: the client's `default_project`. - - Empty when the resource names one, which its own fields carry, or when there is no `default_project`. - """ - - if project is None and self.default_project is not None: - return {"project": self.default_project} - return {} - def _iter_paginated( self, path: str, @@ -360,14 +321,8 @@ def _iter_paginated( params: Optional[dict] = None, page_size: int = 100, ) -> Iterator[_T]: - """Iterate every `T` across all pages of an admin-server list endpoint.""" - - for results in self._iter_pages(path, params=params, page_size=page_size): - yield from (model.model_validate(item) for item in results) - - def _iter_pages(self, path: str, *, params: Optional[dict] = None, page_size: int = 100) -> Iterator[Any]: """ - Iterate the `results` of each page of an admin-server list endpoint. + Iterate every `T` across all pages of an admin-server list endpoint. Opts into pagination by sending `limit`/`offset` on the first request, then follows the absolute `next` URL returned by the server. @@ -383,7 +338,7 @@ def _iter_pages(self, path: str, *, params: Optional[dict] = None, page_size: in while url: response = self.make_request("GET", url, params=current_params) data = response.json() - yield data["results"] + yield from (model.model_validate(item) for item in data["results"]) url = data.get("next") # The `next` URL is absolute and already contains the pagination cursor; # do not re-send our initial params alongside it. diff --git a/datamasque/client/bundles.py b/datamasque/client/bundles.py deleted file mode 100644 index 889629a..0000000 --- a/datamasque/client/bundles.py +++ /dev/null @@ -1,134 +0,0 @@ -import logging -from collections.abc import Sequence -from typing import Optional, Union - -from datamasque.client.base import BaseClient, FileOrContent, UploadFile, read_file_or_content -from datamasque.client.models.bundle import BundleImportResult -from datamasque.client.models.project import ProjectIdentifier, unwrap_project_id -from datamasque.client.models.ruleset import Ruleset, RulesetId, unwrap_ruleset_id - -logger = logging.getLogger(__name__) - - -class BundleClient(BaseClient): - """ - Export and import bundle API methods. Mixed into `DataMasqueClient`. - - The server accepts these only with the login token that `POST /api/auth/token/login/` returns. - They fail with a 403 for a client whose `token_source` returns a user's long-lived API token, - the `api_token` that `GET /api/users/me/` gives. - """ - - def export_bundle( - self, - rulesets: Optional[Sequence[Union[Ruleset, RulesetId]]] = None, - *, - include_seed_files: bool = False, - ) -> bytes: - """ - Returns a Version 2 export bundle, a Zip archive that files every item under its project's name. - - The bundle holds `rulesets`, given as `Ruleset`s or their IDs, and the libraries they use; - or, when `rulesets` is not given, every ruleset and library. - With `include_seed_files`, it also holds the custom seed files those rulesets and libraries refer to. - Pass the bundle to `import_bundle`, on this server or another, - to put each item in the project of the same name. - For example, `Path("export.zip").write_bytes(client.export_bundle())` saves it to a file. - - Raises: - TypeError: `rulesets` is a single `Ruleset` or ID, not a sequence of them. - ValueError: `rulesets` is empty, or a `Ruleset` in it has not been created on the server. - DataMasqueApiError: the server rejects the request, - for example with a 403 when the client authenticates with a long-lived API token. - """ - - if isinstance(rulesets, (str, Ruleset)): - raise TypeError("Pass `rulesets` as a sequence, such as `[ruleset]`") - if rulesets is not None and not rulesets: - raise ValueError("Pass at least one ruleset, or omit `rulesets` to export every ruleset") - - data: dict[str, object] = {"include_seed_files": include_seed_files} - if rulesets is not None: - data["ruleset_ids"] = [unwrap_ruleset_id(ruleset) for ruleset in rulesets] - - response = self.make_request("POST", "/api/export/v2/", data=data) - return response.content - - def import_bundle( - self, - bundle: FileOrContent, - *, - overwrite_rulesets: Optional[bool] = None, - overwrite_libraries: bool = False, - overwrite_seed_files: bool = False, - fallback_project: Optional[ProjectIdentifier] = None, - ) -> BundleImportResult: - """ - Imports an export bundle, in the Version 2 or deprecated Version 1 format. - - Returns what the import did with each item. - - `bundle` is a path to the Zip archive (as `str` or `Path`), its `bytes`, - or a binary file-like object, such as a file opened with `"rb"`. - Each item goes into the project the bundle files it under, - which is created when it does not exist, if the caller is an administrator. - Items that name no project, such as a Version 1 bundle's rulesets and seed files, - go into `fallback_project`, given as a `Project`, its `ProjectId` or its `-` form, - or else the server's earliest-created active project; - the result's `fallback_project` names the project they went into. - The client's `default_project` does not apply. - - With `overwrite_rulesets` unset, a ruleset whose ID exists is left unchanged, - and a ruleset whose name is in use under another ID is imported under a new name; - `True` overwrites the existing ruleset and `False` leaves it unchanged. - Whatever the options, a ruleset or library whose ID is not on the server is left unchanged - when its name is in use by one with identical content. - `BundleImportStatus` lists every outcome. - `overwrite_libraries` and `overwrite_seed_files` overwrite existing libraries and seed files when `True`, - and leave them unchanged when `False`. - - The import can succeed for some items and fail for others: - check each type's `metadata.error`, or each item's `status`. - - Raises: - ValueError: `fallback_project` is a string that is not a project reference in the form `-`. - DataMasqueApiError: the server rejects the import, and imports nothing. - For example, a 400 when the bundle is not a valid export, `fallback_project` is not an active project, - or the bundle names a project that cannot be created with that name; - or a 403 when the bundle names a project that does not exist and the caller is not an administrator, - or when the client authenticates with a long-lived API token. - """ - - file_name, content = read_file_or_content(bundle, "bundle.zip") - content.seek(0) - - data: dict[str, object] = { - "overwrite_libraries": overwrite_libraries, - "overwrite_seed_files": overwrite_seed_files, - } - if overwrite_rulesets is not None: - data["overwrite_rulesets"] = overwrite_rulesets - if fallback_project is not None: - data["fallback_project"] = str(unwrap_project_id(fallback_project)) - - response = self.make_request( - "POST", - "/api/import/v2/", - data=data, - files=[ - UploadFile( - field_name="zip_archive", - filename=file_name, - content=content, - content_type="application/zip", - ), - ], - ) - result = BundleImportResult.model_validate(response.json()) - result.is_deprecated_bundle = response.headers.get("Deprecation", "").lower() == "true" - if result.is_deprecated_bundle: - logger.warning("Imported a deprecated Version 1 bundle; export a Version 2 bundle to keep each project") - if result.fallback_project is not None: - logger.info('Items that named no project went into project "%s"', result.fallback_project) - - return result diff --git a/datamasque/client/connections.py b/datamasque/client/connections.py index 43e7652..e41e46b 100644 --- a/datamasque/client/connections.py +++ b/datamasque/client/connections.py @@ -1,10 +1,8 @@ import logging -from typing import Optional, Union from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueException from datamasque.client.models.connection import ConnectionConfig, ConnectionId, validate_connection -from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id logger = logging.getLogger(__name__) @@ -12,39 +10,25 @@ class ConnectionClient(BaseClient): """Connection-related API methods. Mixed into `DataMasqueClient`.""" - def list_connections(self, project: Optional[ProjectIdentifier] = None) -> list[ConnectionConfig]: + def list_connections(self) -> list[ConnectionConfig]: """ - Lists all configured connections, or only those in `project` when it is given. + Lists all configured connections. - `project` is a `Project`, its `ProjectId` or its `-` form. Note that database passwords and connection strings are returned encrypted over the API and so are `None` on the returned `ConnectionConfig` objects. """ - return self._list_connections(unwrap_project_id(project) if project is not None else None) - - def _list_connections(self, project: Optional[Union[ProjectId, str]]) -> list[ConnectionConfig]: - """Lists the connections in `project`, given as an ID or `-` reference, or in every project.""" - - params = {"project": project} if project is not None else None - response = self.make_request("GET", "/api/connections/", params=params) + response = self.make_request("GET", "/api/connections/") return [validate_connection(payload) for payload in response.json()] def create_or_update_connection(self, connection_config: ConnectionConfig) -> ConnectionConfig: - """ - Creates or updates the connection in DM, and sets the `id` field on the given `connection_config`. - - An existing connection with the same name in the same project is updated. - With no `project` set, the client's `default_project` is used, else the server's only active project, - and `DataMasqueException` is raised when it has no active project or more than one. - """ + """Creates or updates the connection in DM, and sets the `id` field on the given `connection_config`.""" connection_id = connection_config.id + all_connections = self.list_connections() connections_matching_name = [ - connection - for connection in self._list_connections(self._resolve_project(connection_config.project)) - if connection.name == connection_config.name + connection for connection in all_connections if connection.name == connection_config.name ] if connections_matching_name: connection_id = connections_matching_name[0].id @@ -53,7 +37,6 @@ def create_or_update_connection(self, connection_config: ConnectionConfig) -> Co "version": "1.0", } | connection_config.model_dump(exclude_none=True, by_alias=True, mode="json") if connection_id is None: - data |= self._default_project_field(connection_config.project) response = self.make_request("POST", "/api/connections/", data=data) else: response = self.make_request("PUT", f"/api/connections/{connection_id}/", data=data) @@ -62,7 +45,6 @@ def create_or_update_connection(self, connection_config: ConnectionConfig) -> Co server_connection_id = ConnectionId(connection_data["id"]) logger.debug("%s creation successful", type(connection_config).__name__) connection_config.id = server_connection_id - connection_config.project = connection_data.get("project") return connection_config def delete_connection_by_id_if_exists(self, connection_id: ConnectionId) -> None: @@ -70,17 +52,11 @@ def delete_connection_by_id_if_exists(self, connection_id: ConnectionId) -> None self._delete_if_exists(f"/api/connections/{connection_id}/") - def delete_connection_by_name_if_exists(self, connection_name: str, project: ProjectIdentifier) -> None: - """ - Deletes the connection with the given name in the given project. - - `project` is a `Project`, its `ProjectId` or its `-` form. - No-op if the connection does not exist. - """ + def delete_connection_by_name_if_exists(self, connection_name: str) -> None: + """Deletes the connection with the given name. No-op if the connection does not exist.""" - connections_matching_name = [ - connection for connection in self.list_connections(project) if connection.name == connection_name - ] + all_connections = self.list_connections() + connections_matching_name = [connection for connection in all_connections if connection.name == connection_name] for connection in connections_matching_name: if connection.id is None: raise DataMasqueException(f'Server returned a connection named "{connection.name}" without an `id`.') diff --git a/datamasque/client/discovery.py b/datamasque/client/discovery.py index 9145298..2faaaad 100644 --- a/datamasque/client/discovery.py +++ b/datamasque/client/discovery.py @@ -15,7 +15,7 @@ InvalidDiscoveryConfigError, RGConfigNotFoundError, ) -from datamasque.client.models.connection import ConnectionId, validate_connection +from datamasque.client.models.connection import ConnectionId from datamasque.client.models.data_selection import ( SelectedColumns, SelectedData, @@ -247,7 +247,6 @@ def get_generated_rulesets(self, connection_id: ConnectionId) -> list[Ruleset]: and `DataMasqueException` if it failed or was cancelled. Note that the ruleset(s) have autogenerated names, which you may want to customize before uploading. - Their `project` is the connection's project. """ status = self.get_async_ruleset_generation_task_status(connection_id) @@ -274,8 +273,6 @@ def get_generated_rulesets(self, connection_id: ConnectionId) -> list[Ruleset]: # (which carries the generated ruleset inline as `generated_ruleset`). # `requests` follows the 303 transparently, so we distinguish by the presence of # a `Content-Disposition: attachment` header, which Django's `FileResponse` sets on the ZIP response. - project = validate_connection(self.make_request("GET", f"/api/connections/{connection_id}/").json()).project - response = self.make_request( method="GET", path=f"/api/async-generate-ruleset/{connection_id}/download-rulesets/", @@ -288,9 +285,7 @@ def get_generated_rulesets(self, connection_id: ConnectionId) -> list[Ruleset]: if file_info.filename.endswith((".yml", ".yaml")): with zip_file.open(file_info) as file: yaml_content = file.read().decode("utf-8") - rulesets.append( - Ruleset(name=Path(file_info.filename).stem, project=project, yaml=yaml_content) - ) + rulesets.append(Ruleset(name=Path(file_info.filename).stem, yaml=yaml_content)) if not rulesets: raise DataMasqueException( @@ -307,7 +302,7 @@ def get_generated_rulesets(self, connection_id: ConnectionId) -> list[Ruleset]: f"but no ruleset was returned on the task-status record." ) - return [Ruleset(name="generated_ruleset", project=project, yaml=generated)] + return [Ruleset(name="generated_ruleset", yaml=generated)] def start_schema_discovery_run(self, discovery_config: SchemaDiscoveryRequest) -> RunId: """ diff --git a/datamasque/client/discovery_config_libraries.py b/datamasque/client/discovery_config_libraries.py index c0e978b..bd1e1fb 100644 --- a/datamasque/client/discovery_config_libraries.py +++ b/datamasque/client/discovery_config_libraries.py @@ -1,10 +1,9 @@ import logging -from typing import Optional, Union +from typing import Optional from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary, DiscoveryConfigLibraryId -from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id logger = logging.getLogger(__name__) @@ -31,18 +30,22 @@ def get_discovery_config_library(self, library_id: DiscoveryConfigLibraryId) -> response = self.make_request("GET", f"/api/discovery/config-libraries/{library_id}/") return DiscoveryConfigLibrary.model_validate(response.json()) - def _get_discovery_config_library_id_by_name( - self, name: str, project: Union[ProjectId, str] - ) -> Optional[DiscoveryConfigLibraryId]: - """Returns the ID of the library with the given name in the given project, or `None` if there is none.""" + def _get_discovery_config_library_id_by_name(self, name: str, namespace: str) -> Optional[DiscoveryConfigLibraryId]: + """ + Returns the ID of the library with the given name and namespace, or `None` if there is none. + + The listing is filtered by name to keep the response small; + the namespace is matched client-side, + since the API's namespace filter cannot select the default (empty) namespace. + """ response = self.make_request( "GET", "/api/discovery/config-libraries/", - params={"name_exact": name, "project": project}, + params={"name_exact": name}, ) entries = [DiscoveryConfigLibrary.model_validate(item) for item in response.json()] - matches = [entry for entry in entries if entry.name == name] + matches = [entry for entry in entries if entry.name == name and entry.namespace == namespace] if not matches: return None @@ -54,17 +57,15 @@ def _get_discovery_config_library_id_by_name( return matches[0].id - def get_discovery_config_library_by_name( - self, name: str, project: ProjectIdentifier - ) -> Optional[DiscoveryConfigLibrary]: + def get_discovery_config_library_by_name(self, name: str, namespace: str = "") -> Optional[DiscoveryConfigLibrary]: """ - Looks for a discovery config library matching the given name (case-sensitive, exact match) in the given project. + Looks for a discovery config library matching the given name and namespace (case-sensitive, exact match). - `project` is a `Project`, its `ProjectId` or its `-` form. + Library names are unique within a namespace. Returns it (with full YAML content) if found, otherwise `None`. """ - library_id = self._get_discovery_config_library_id_by_name(name, unwrap_project_id(project)) + library_id = self._get_discovery_config_library_id_by_name(name, namespace) if library_id is None: return None @@ -81,13 +82,10 @@ def create_discovery_config_library(self, library: DiscoveryConfigLibrary) -> Di if not library.yaml: raise ValueError("Cannot create a discovery config library without YAML content (yaml is empty)") - data = library.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( - library.project - ) + data = library.model_dump(exclude_none=True, by_alias=True, mode="json") response = self.make_request("POST", "/api/discovery/config-libraries/", data=data) created = DiscoveryConfigLibrary.model_validate(response.json()) library.id = created.id - library.project = created.project library.is_valid = created.is_valid library.validation_error = created.validation_error library.usage_count = created.usage_count @@ -125,14 +123,12 @@ def update_discovery_config_library(self, library: DiscoveryConfigLibrary) -> Di def create_or_update_discovery_config_library(self, library: DiscoveryConfigLibrary) -> DiscoveryConfigLibrary: """ - Creates the library, or updates the existing one with the same name in the same project. + Creates the library, or updates the existing one with the same name and namespace. - With no `project` set, the client's `default_project` is used, else the server's only active project, - and `DataMasqueException` is raised when it has no active project or more than one. Sets the library's `id` property. """ - library_id = self._get_discovery_config_library_id_by_name(library.name, self._resolve_project(library.project)) + library_id = self._get_discovery_config_library_id_by_name(library.name, library.namespace) if library_id is not None: library.id = library_id return self.update_discovery_config_library(library) @@ -155,15 +151,15 @@ def delete_discovery_config_library_by_id_if_exists( self._delete_if_exists(f"/api/discovery/config-libraries/{library_id}/", params=params) def delete_discovery_config_library_by_name_if_exists( - self, name: str, project: ProjectIdentifier, *, force: bool = False + self, name: str, namespace: str = "", *, force: bool = False ) -> None: """ - Deletes the discovery config library with the given name in the given project. + Deletes the discovery config library with the given name and namespace. - `project` is a `Project`, its `ProjectId` or its `-` form. + Library names are unique within a namespace. No-op if no such library exists. """ - library_id = self._get_discovery_config_library_id_by_name(name, unwrap_project_id(project)) + library_id = self._get_discovery_config_library_id_by_name(name, namespace) if library_id is not None: self.delete_discovery_config_library_by_id_if_exists(library_id, force=force) diff --git a/datamasque/client/discovery_configs.py b/datamasque/client/discovery_configs.py index 69f5cac..d19f900 100644 --- a/datamasque/client/discovery_configs.py +++ b/datamasque/client/discovery_configs.py @@ -1,11 +1,10 @@ import logging -from typing import Iterator, Optional, Union +from typing import Iterator, Optional from datamasque.client.base import BaseClient -from datamasque.client.exceptions import DataMasqueApiError +from datamasque.client.exceptions import DataMasqueApiError, DataMasqueException from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType from datamasque.client.models.pagination import Page -from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id logger = logging.getLogger(__name__) @@ -35,14 +34,14 @@ def get_discovery_config(self, config_id: DiscoveryConfigId) -> DiscoveryConfig: return DiscoveryConfig.model_validate(response.json()) def _get_discovery_config_id_by_name( - self, name: str, config_type: DiscoveryConfigType, project: Union[ProjectId, str] + self, name: str, config_type: DiscoveryConfigType ) -> Optional[DiscoveryConfigId]: - """Return the id of the config matching name, type and project via a single list request, or `None`.""" + """Return the id of the config matching name and type via a single list request, or `None`.""" response = self.make_request( "GET", "/api/discovery/configs/", - params={"name_exact": name, "config_type": config_type.value, "project": project, "limit": 1}, + params={"name_exact": name, "config_type": config_type.value, "limit": 1}, ) page = Page[DiscoveryConfig].model_validate(response.json()) if not page.results: @@ -57,18 +56,15 @@ def _get_discovery_config_id_by_name( return config_id - def get_discovery_config_by_name( - self, name: str, config_type: DiscoveryConfigType, project: ProjectIdentifier - ) -> Optional[DiscoveryConfig]: + def get_discovery_config_by_name(self, name: str, config_type: DiscoveryConfigType) -> Optional[DiscoveryConfig]: """ - Looks for a discovery config matching the given name (case-sensitive, exact match), type and project. + Looks for a discovery config matching the given name and type (case-sensitive, exact match). - Config names are unique per type and project, so both are required to identify a single config. - `project` is a `Project`, its `ProjectId` or its `-` form. + Config names are unique per type, so a type is required to identify a single config. Returns it if found, otherwise `None`. """ - config_id = self._get_discovery_config_id_by_name(name, config_type, unwrap_project_id(project)) + config_id = self._get_discovery_config_id_by_name(name, config_type) if config_id is None: return None @@ -86,13 +82,10 @@ def create_discovery_config(self, config: DiscoveryConfig) -> DiscoveryConfig: if not config.yaml: raise ValueError("Cannot create a discovery config without YAML content (yaml is empty)") - data = config.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( - config.project - ) + data = config.model_dump(exclude_none=True, by_alias=True, mode="json") response = self.make_request("POST", "/api/discovery/configs/", data=data) created = DiscoveryConfig.model_validate(response.json()) config.id = created.id - config.project = created.project config.is_valid = created.is_valid config.validation_error = created.validation_error config.validation_error_details = created.validation_error_details @@ -130,16 +123,12 @@ def update_discovery_config(self, config: DiscoveryConfig) -> DiscoveryConfig: def create_or_update_discovery_config(self, config: DiscoveryConfig) -> DiscoveryConfig: """ - Creates the config, or updates the existing one with the same name and type in the same project. + Creates the config if it doesn't exist, or updates it if one with the same name already exists. - With no `project` set, the client's `default_project` is used, else the server's only active project, - and `DataMasqueException` is raised when it has no active project or more than one. Sets the config's `id` property. """ - existing_id = self._get_discovery_config_id_by_name( - config.name, config.config_type, self._resolve_project(config.project) - ) + existing_id = self._get_discovery_config_id_by_name(config.name, config.config_type) if existing_id is not None: config.id = existing_id return self.update_discovery_config(config) @@ -155,20 +144,24 @@ def delete_discovery_config_by_id_if_exists(self, config_id: DiscoveryConfigId) self._delete_if_exists(f"/api/discovery/configs/{config_id}/") - def delete_discovery_config_by_name_if_exists( - self, name: str, config_type: DiscoveryConfigType, project: ProjectIdentifier - ) -> None: + def delete_discovery_config_by_name_if_exists(self, name: str, config_type: DiscoveryConfigType) -> None: """ - Deletes the discovery config with the given name and type in the given project. + Deletes the discovery config with the given name and type. - Config names are unique per type and project, so both are required to identify a single config. - `project` is a `Project`, its `ProjectId` or its `-` form. + Config names are unique per type, so a type is required to identify a single config. No-op if no such config exists. """ - config_id = self._get_discovery_config_id_by_name(name, config_type, unwrap_project_id(project)) - if config_id is not None: - self.delete_discovery_config_by_id_if_exists(config_id) + matching = [ + config + for config in self.list_discovery_configs() + if config.name == name and config.config_type is config_type + ] + for config in matching: + if config.id is None: + raise DataMasqueException(f'Server returned a discovery config named "{config.name}" without an `id`.') + + self.delete_discovery_config_by_id_if_exists(config.id) def get_default_discovery_config_yaml(self) -> str: """Returns the server's built-in default discovery configuration as a YAML string.""" diff --git a/datamasque/client/dmclient.py b/datamasque/client/dmclient.py index 6acaa69..7e8cc7f 100644 --- a/datamasque/client/dmclient.py +++ b/datamasque/client/dmclient.py @@ -1,14 +1,10 @@ from datamasque.client.base import FileOrContent, UploadFile -from datamasque.client.bundles import BundleClient from datamasque.client.connections import ConnectionClient from datamasque.client.discovery import DiscoveryClient from datamasque.client.discovery_config_libraries import DiscoveryConfigLibraryClient from datamasque.client.discovery_configs import DiscoveryConfigClient from datamasque.client.files import FileClient -from datamasque.client.git import GitClient from datamasque.client.license import LicenseClient -from datamasque.client.projects import ProjectClient -from datamasque.client.resources import ResourceClient from datamasque.client.rg_configs import RGConfigClient from datamasque.client.ruleset_libraries import RulesetLibraryClient from datamasque.client.rulesets import RulesetClient @@ -22,18 +18,14 @@ class DataMasqueClient( LicenseClient, - ProjectClient, - ResourceClient, ConnectionClient, RulesetClient, RulesetLibraryClient, FileClient, - BundleClient, RunClient, DiscoveryClient, DiscoveryConfigClient, DiscoveryConfigLibraryClient, - GitClient, TableReferenceClient, RGConfigClient, UserClient, diff --git a/datamasque/client/exceptions.py b/datamasque/client/exceptions.py index 747fd64..9943b8b 100644 --- a/datamasque/client/exceptions.py +++ b/datamasque/client/exceptions.py @@ -1,7 +1,5 @@ from requests import Response -from datamasque.client.models.project import ProjectDeleteConflict - class DataMasqueException(Exception): """Generic exception base class.""" @@ -68,21 +66,6 @@ class RGConfigNotFoundError(DataMasqueApiError): """ -class ProjectDeleteConflictError(DataMasqueApiError): - """ - Raised when `delete_project` cannot move the project's resources to the destination project. - - The server deletes and moves nothing in this case. - - Attributes: - conflicts: Every reason the resources cannot move. - """ - - def __init__(self, message: str, *, response: Response, conflicts: list[ProjectDeleteConflict]) -> None: - super().__init__(message, response=response) - self.conflicts = conflicts - - class DataMasqueTransportError(DataMasqueException): """ Raised when a request to the DataMasque server fails before any response is received. diff --git a/datamasque/client/files.py b/datamasque/client/files.py index 641b4bc..89d1a67 100644 --- a/datamasque/client/files.py +++ b/datamasque/client/files.py @@ -3,7 +3,6 @@ from datamasque.client.base import BaseClient, UploadFile, read_file_or_content from datamasque.client.models.files import DataMasqueFile -from datamasque.client.models.project import ProjectIdentifier, unwrap_project_id FileTypeT = TypeVar("FileTypeT", bound=DataMasqueFile) @@ -16,15 +15,13 @@ def upload_file( file_type: Type[FileTypeT], file_name: str, file_path_or_content: Union[str, bytes, Path], - project: ProjectIdentifier, ) -> FileTypeT: """ - Uploads a file of the given type to the DataMasque server, into the given project. + Uploads a file of the given type to the DataMasque server. `file_type` must be a concrete subclass of `DataMasqueFile` (`SeedFile`, `OracleWalletFile`, `SslZipFile`, `SnowflakeKeyFile`). `file_path_or_content` may be a path (as `str` or `Path`), raw `bytes`, or a file-like object. - `project` is a `Project`, its `ProjectId` or its `-` form. """ name, content = read_file_or_content(file_path_or_content, file_name) @@ -33,7 +30,7 @@ def upload_file( response = self.make_request( "POST", file_type.get_url(), - data={"name": file_name, "project": unwrap_project_id(project), **file_type.get_extra_form_data()}, + data={"name": file_name, **file_type.get_extra_form_data()}, files=[ UploadFile( field_name=file_type.get_content_param_name(), @@ -59,50 +56,37 @@ def delete_file_if_exists(self, file: DataMasqueFile) -> None: # file.get_url() ends with a slash so no need to insert one before the id self._delete_if_exists(f"{file.get_url()}{file.id}/") - def list_files_of_type( - self, file_type: Type[FileTypeT], project: Optional[ProjectIdentifier] = None - ) -> list[FileTypeT]: - """ - Returns all files of the given type (a concrete subclass of `DataMasqueFile`). - - Only files in `project` are returned when it is given. - `project` is a `Project`, its `ProjectId` or its `-` form. - """ + def list_files_of_type(self, file_type: Type[FileTypeT]) -> list[FileTypeT]: + """Returns all files of the given type (a concrete subclass of `DataMasqueFile`).""" - params = {"project": unwrap_project_id(project)} if project is not None else None - response = self.make_request("GET", file_type.get_url(), params=params) + response = self.make_request("GET", file_type.get_url()) return [file_type.model_validate(file) for file in response.json()] - def get_file_of_type_by_name( - self, file_type: Type[FileTypeT], name: str, project: ProjectIdentifier - ) -> Optional[FileTypeT]: + def get_file_of_type_by_name(self, file_type: Type[FileTypeT], name: str) -> Optional[FileTypeT]: """ - Looks for a file of the given type (a concrete subclass of `DataMasqueFile`) by name in the given project. + Looks for a file of the given type (a concrete subclass of `DataMasqueFile`) with the given `name`. Returns it if found, otherwise `None`. """ - matching_files = [f for f in self.list_files_of_type(file_type, project) if f.name == name] + matching_files = [f for f in self.list_files_of_type(file_type) if f.name == name] return matching_files[0] if matching_files else None - def upload_file_if_not_exists( - self, file_type: Type[FileTypeT], file_path: Union[str, Path], project: ProjectIdentifier - ) -> Optional[FileTypeT]: + def upload_file_if_not_exists(self, file_type: Type[FileTypeT], file_path: Union[str, Path]) -> Optional[FileTypeT]: """ - Upload a file only if one with the same name doesn't already exist in the project. + Upload a file only if one with the same name doesn't already exist. Args: file_type: A concrete subclass of `DataMasqueFile` (e.g., SeedFile, OracleWalletFile). file_path: Path to the file to upload. - project: A `Project`, its `ProjectId` or its `-` form. Returns: The uploaded file object if a new file was uploaded, or None if a file - with the same name already exists in the project. + with the same name already exists. """ file_path = Path(file_path) - if self.get_file_of_type_by_name(file_type, file_path.name, project) is not None: + if self.get_file_of_type_by_name(file_type, file_path.name) is not None: return None - return self.upload_file(file_type, file_path.name, file_path, project) + return self.upload_file(file_type, file_path.name, file_path) diff --git a/datamasque/client/git.py b/datamasque/client/git.py deleted file mode 100644 index fc238d9..0000000 --- a/datamasque/client/git.py +++ /dev/null @@ -1,463 +0,0 @@ -from dataclasses import dataclass -from pathlib import PurePosixPath -from typing import Literal, Optional, Union - -from pydantic import BaseModel - -from datamasque.client.base import BaseClient -from datamasque.client.models.discovery_config import DiscoveryConfig -from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary -from datamasque.client.models.git import GitFile -from datamasque.client.models.ruleset import Ruleset -from datamasque.client.models.ruleset_library import RulesetLibrary - -GitExtension = Literal[".yml", ".yaml"] -"""The extensions the server accepts for a pushed file.""" - -DEFAULT_GIT_EXTENSION: GitExtension = ".yml" - - -@dataclass(frozen=True) -class _GitResourceType: - """The endpoint and request field names one resource type's git operations use.""" - - endpoint: str - name_field: str - content_field: str - folder_param: str - """The pull parameter naming the file's folder, as the file list shows it.""" - - -_RULESET = _GitResourceType("/api/ruleset-git/", "ruleset_name", "ruleset_content", "folder") -_RULESET_LIBRARY = _GitResourceType("/api/library-git/", "library_name", "library_content", "namespace") -_DISCOVERY_CONFIG = _GitResourceType("/api/discovery-config-git/", "config_name", "config_content", "folder") -_DISCOVERY_CONFIG_LIBRARY = _GitResourceType( - "/api/discovery-config-library-git/", "library_name", "library_content", "namespace" -) - - -class _GitPull(BaseModel): - """The response to a pull.""" - - config_yaml: str - - -class _GitPushPath(BaseModel): - """The response to a push-path request.""" - - path: str - - -def _branch_param(branch: Optional[str]) -> dict[str, str]: - return {} if branch is None else {"branch": branch} - - -def _require_yaml(yaml: Optional[str], description: str, getter: str) -> str: - if not yaml: - raise ValueError( - f"Cannot push a {description} to git without YAML content (yaml is empty or unset); " - f"fetch the {description} with its YAML using `{getter}` first" - ) - - return yaml - - -class GitClient(BaseClient): - """ - Git push, pull and file listing for rulesets, ruleset libraries, discovery configs and their libraries. - - Mixed into `DataMasqueClient`. - The server reads and writes the repository in its Git settings with the user's SSH key, - so both must be set up for the user the client authenticates as. - Each `branch` parameter defaults to the branch in the Git settings. - - A push writes `//` under the Git directory path, - where `` is the project's `name`, not its `reference`, - and the type folders are `rulesets/`, `libraries/`, `discovery-configs/` and `discovery-config-libraries/`. - A resource pushed before the server's upgrade to projects keeps the file it had then, - until it is renamed or moved to another project: - a ruleset at the top of the Git directory path, - a discovery config directly in `discovery-configs/`, - a library of the `Default` project directly in its type folder, - and a library of another namespace in the namespace's folder, - such as `libraries/bank/au/` for the namespace that the upgrade made the project `bank_au`. - - The methods give paths in two forms: - - - `get_*_git_push_path` returns the file's path in the repository without its extension, - starting with the Git directory path. - - `list_*_git_files` returns each file's path with its extension, - relative to the Git directory path for rulesets and to the type folder for the other types. - The matching `pull_*_from_git` method takes a path in this form. - - For example, to restore a ruleset to the commit before its latest one and push it back: - - .. code-block:: python - - banking = next(project for project in client.iter_projects() if project.name == "Banking") - file = next(f for f in client.list_ruleset_git_files() if f.path == "rulesets/Banking/customers.yml") - yaml = client.pull_ruleset_from_git(file.path, file.commits[1].commit_sha) - ruleset = client.create_or_update_ruleset(Ruleset(name="customers", project=banking.reference, yaml=yaml)) - client.push_ruleset_to_git(ruleset, "Restore the previous customers ruleset") - """ - - def list_ruleset_git_files(self, *, branch: Optional[str] = None) -> list[GitFile]: - """ - Lists the ruleset files in the git repository. - - Paths are relative to the Git directory path: - `rulesets//customers.yml` for a project's ruleset, - and `customers.yml` for one pushed before the server's upgrade to projects. - """ - - return self._list_git_files(_RULESET, branch) - - def pull_ruleset_from_git(self, path: str, commit_sha: str, *, branch: Optional[str] = None) -> str: - """ - Returns the YAML of a ruleset file at `commit_sha`. - - `path` is the file's path as `list_ruleset_git_files` lists it. - Saves nothing: set it as a ruleset's `yaml` and save that ruleset to keep it. - """ - - return self._pull_from_git(_RULESET, path, commit_sha, branch) - - def push_ruleset_to_git( - self, - ruleset: Ruleset, - commit_message: str, - *, - branch: Optional[str] = None, - extension: GitExtension = DEFAULT_GIT_EXTENSION, - ) -> None: - """ - Commits the ruleset's `yaml` to the git repository and pushes it. - - The file is the ruleset's name with `extension` (`.yml` or `.yaml`) in its project's folder, - unless the saved ruleset kept an older file (see `GitClient`). - A push with the other extension writes a second file and leaves the first. - With no `project` set, the project is the saved ruleset's when it has an `id`, - else the client's `default_project`, else the server's only active project. - The ruleset's `id`, else its name, type and project, identify the saved ruleset the push is for. - With an `id`, the ruleset's name, type and project must match the saved ruleset's. - The push sends the given ruleset's `yaml`, not the saved one, so save the ruleset first. - The server records the pushed commit as the saved ruleset's git provenance when their YAML matches, - but the given ruleset is not updated. - - Raises: - ValueError: The ruleset's `yaml` is empty. - DataMasqueApiError: With status 409 when another ruleset uses or would write the file, - and with status 400 when nothing changed, the repository has a file where the path needs a folder, - or no project is set and the server does not have exactly one active project. - """ - - self._push_to_git( - _RULESET, - self._ruleset_git_target(ruleset), - _require_yaml(ruleset.yaml, "ruleset", "list_rulesets"), - commit_message, - branch=branch, - extension=extension, - ) - - def get_ruleset_git_push_path(self, ruleset: Ruleset) -> str: - """ - Returns the path `push_ruleset_to_git` would write the ruleset to. - - The path is relative to the repository and has no extension. - """ - - return self._get_git_push_path(_RULESET, self._ruleset_git_target(ruleset)) - - def list_ruleset_library_git_files(self, *, branch: Optional[str] = None) -> list[GitFile]: - """ - Lists the ruleset library files in the git repository. - - Paths are relative to `libraries/` under the Git directory path: - `/emails.yml` for a project's library, - `emails.yml` for a library of the `Default` project pushed before the server's upgrade to projects, - and `/emails.yml` for a library in a namespace whose project the upgrade named differently, - such as `bank/au/emails.yml` in the project `bank_au`. - """ - - return self._list_git_files(_RULESET_LIBRARY, branch) - - def pull_ruleset_library_from_git(self, path: str, commit_sha: str, *, branch: Optional[str] = None) -> str: - """ - Returns the YAML of a library file at `commit_sha`. - - `path` is the file's path as `list_ruleset_library_git_files` lists it. - Saves nothing: set it as a library's `yaml` and save that library to keep it. - """ - - return self._pull_from_git(_RULESET_LIBRARY, path, commit_sha, branch) - - def push_ruleset_library_to_git( - self, - library: RulesetLibrary, - commit_message: str, - *, - branch: Optional[str] = None, - extension: GitExtension = DEFAULT_GIT_EXTENSION, - ) -> None: - """ - Commits the library's `yaml` to the git repository and pushes it. - - The file is the library's name with `extension` (`.yml` or `.yaml`) in its project's folder, - unless the saved library kept an older file (see `GitClient`). - A push with the other extension writes a second file and leaves the first. - With no `project` set, the project is the saved library's when it has an `id`, - else the client's `default_project`, else the server's only active project. - The library's `id`, else its name and project, identify the saved library the push is for. - With an `id`, the library's name and project must match the saved library's. - The push sends the given library's `yaml`, not the saved one, so save the library first. - The server records the pushed commit as the saved library's git provenance when their YAML matches, - but the given library is not updated. - - Raises: - ValueError: The library's `yaml` is empty or unset. - DataMasqueApiError: With status 409 when another library uses or would write the file, - and with status 400 when nothing changed, the repository has a file where the path needs a folder, - or no project is set and the server does not have exactly one active project. - """ - - self._push_to_git( - _RULESET_LIBRARY, - self._library_git_target(_RULESET_LIBRARY, library), - _require_yaml(library.yaml, "ruleset library", "get_ruleset_library"), - commit_message, - branch=branch, - extension=extension, - ) - - def get_ruleset_library_git_push_path(self, library: RulesetLibrary) -> str: - """ - Returns the path `push_ruleset_library_to_git` would write the library to. - - The path is relative to the repository and has no extension. - """ - - return self._get_git_push_path(_RULESET_LIBRARY, self._library_git_target(_RULESET_LIBRARY, library)) - - def list_discovery_config_git_files(self, *, branch: Optional[str] = None) -> list[GitFile]: - """ - Lists the discovery config files in the git repository. - - Paths are relative to `discovery-configs/` under the Git directory path: - `/customers.yml` for a project's config, - and `customers.yml` for one pushed before the server's upgrade to projects. - """ - - return self._list_git_files(_DISCOVERY_CONFIG, branch) - - def pull_discovery_config_from_git(self, path: str, commit_sha: str, *, branch: Optional[str] = None) -> str: - """ - Returns the YAML of a config file at `commit_sha`. - - `path` is the file's path as `list_discovery_config_git_files` lists it. - Saves nothing: set it as a config's `yaml` and save that config to keep it. - """ - - return self._pull_from_git(_DISCOVERY_CONFIG, path, commit_sha, branch) - - def push_discovery_config_to_git( - self, - config: DiscoveryConfig, - commit_message: str, - *, - branch: Optional[str] = None, - extension: GitExtension = DEFAULT_GIT_EXTENSION, - ) -> None: - """ - Commits the discovery config's `yaml` to the git repository and pushes it. - - The file is the config's name with `extension` (`.yml` or `.yaml`) in its project's folder, - unless the saved config kept an older file (see `GitClient`). - A push with the other extension writes a second file and leaves the first. - With no `project` set, the project is the saved config's when it has an `id`, - else the client's `default_project`, else the server's only active project. - The config's `id`, else its name, type and project, identify the saved config the push is for. - With an `id`, the config's name, type and project must match the saved config's. - The push sends the given config's `yaml`, not the saved one, so save the config first. - The server records the pushed commit on the saved config when their YAML matches, - but `DiscoveryConfig` does not expose it. - - Raises: - ValueError: The config's `yaml` is empty or unset. - DataMasqueApiError: With status 409 when another config uses or would write the file, - and with status 400 when nothing changed, the repository has a file where the path needs a folder, - or no project is set and the server does not have exactly one active project. - """ - - self._push_to_git( - _DISCOVERY_CONFIG, - self._discovery_config_git_target(config), - _require_yaml(config.yaml, "discovery config", "get_discovery_config"), - commit_message, - branch=branch, - extension=extension, - ) - - def get_discovery_config_git_push_path(self, config: DiscoveryConfig) -> str: - """ - Returns the path `push_discovery_config_to_git` would write the config to. - - The path is relative to the repository and has no extension. - """ - - return self._get_git_push_path(_DISCOVERY_CONFIG, self._discovery_config_git_target(config)) - - def list_discovery_config_library_git_files(self, *, branch: Optional[str] = None) -> list[GitFile]: - """ - Lists the discovery config library files in the git repository. - - Paths are relative to `discovery-config-libraries/` under the Git directory path: - `/emails.yml` for a project's library, - `emails.yml` for a library of the `Default` project pushed before the server's upgrade to projects, - and `/emails.yml` for a library in a namespace whose project the upgrade named differently, - such as `bank/au/emails.yml` in the project `bank_au`. - """ - - return self._list_git_files(_DISCOVERY_CONFIG_LIBRARY, branch) - - def pull_discovery_config_library_from_git( - self, path: str, commit_sha: str, *, branch: Optional[str] = None - ) -> str: - """ - Returns the YAML of a library file at `commit_sha`. - - `path` is the file's path as `list_discovery_config_library_git_files` lists it. - Saves nothing: set it as a library's `yaml` and save that library to keep it. - """ - - return self._pull_from_git(_DISCOVERY_CONFIG_LIBRARY, path, commit_sha, branch) - - def push_discovery_config_library_to_git( - self, - library: DiscoveryConfigLibrary, - commit_message: str, - *, - branch: Optional[str] = None, - extension: GitExtension = DEFAULT_GIT_EXTENSION, - ) -> None: - """ - Commits the discovery config library's `yaml` to the git repository and pushes it. - - The file is the library's name with `extension` (`.yml` or `.yaml`) in its project's folder, - unless the saved library kept an older file (see `GitClient`). - A push with the other extension writes a second file and leaves the first. - With no `project` set, the project is the saved library's when it has an `id`, - else the client's `default_project`, else the server's only active project. - The library's `id`, else its name and project, identify the saved library the push is for. - With an `id`, the library's name and project must match the saved library's. - The push sends the given library's `yaml`, not the saved one, so save the library first. - The server records the pushed commit on the saved library when their YAML matches, - but `DiscoveryConfigLibrary` does not expose it. - - Raises: - ValueError: The library's `yaml` is empty or unset. - DataMasqueApiError: With status 409 when another library uses or would write the file, - and with status 400 when nothing changed, the repository has a file where the path needs a folder, - or no project is set and the server does not have exactly one active project. - """ - - self._push_to_git( - _DISCOVERY_CONFIG_LIBRARY, - self._library_git_target(_DISCOVERY_CONFIG_LIBRARY, library), - _require_yaml(library.yaml, "discovery config library", "get_discovery_config_library"), - commit_message, - branch=branch, - extension=extension, - ) - - def get_discovery_config_library_git_push_path(self, library: DiscoveryConfigLibrary) -> str: - """ - Returns the path `push_discovery_config_library_to_git` would write the library to. - - The path is relative to the repository and has no extension. - """ - - return self._get_git_push_path( - _DISCOVERY_CONFIG_LIBRARY, - self._library_git_target(_DISCOVERY_CONFIG_LIBRARY, library), - ) - - def _ruleset_git_target(self, ruleset: Ruleset) -> dict[str, str]: - return self._git_target( - _RULESET, ruleset.name, ruleset.project, ruleset.id, mask_type=ruleset.ruleset_type.value - ) - - def _discovery_config_git_target(self, config: DiscoveryConfig) -> dict[str, str]: - return self._git_target( - _DISCOVERY_CONFIG, config.name, config.project, config.id, config_type=config.config_type.value - ) - - def _library_git_target( - self, resource_type: _GitResourceType, library: Union[RulesetLibrary, DiscoveryConfigLibrary] - ) -> dict[str, str]: - return self._git_target(resource_type, library.name, library.project, library.id) - - def _git_target( - self, - resource_type: _GitResourceType, - name: str, - project: Optional[str], - resource_id: Optional[str], - **type_fields: str, - ) -> dict[str, str]: - """ - Returns the fields that pick the file a push writes, as the push and push-path requests take them. - - An unset `project` is left for the server to take from the saved resource when there is an `id`, - and is otherwise the client's `default_project` when there is one. - """ - - fields: dict[str, str] = {resource_type.name_field: name, **type_fields} - if resource_id is not None: - fields["id"] = resource_id - if project is not None: - fields["project"] = project - elif resource_id is None: - fields |= self._default_project_field(None) - - return fields - - def _list_git_files(self, resource_type: _GitResourceType, branch: Optional[str]) -> list[GitFile]: - response = self.make_request("GET", f"{resource_type.endpoint}files/", params=_branch_param(branch)) - return [GitFile(path=path, commits=commits) for path, commits in response.json().items()] - - def _pull_from_git(self, resource_type: _GitResourceType, path: str, commit_sha: str, branch: Optional[str]) -> str: - file_path = PurePosixPath(path) - folder = file_path.parent - params = { - resource_type.name_field: file_path.stem, - "extension": file_path.suffix, - resource_type.folder_param: "" if folder == PurePosixPath(".") else folder.as_posix(), - "commit_id": commit_sha, - **_branch_param(branch), - } - response = self.make_request("GET", resource_type.endpoint, params=params) - return _GitPull.model_validate(response.json()).config_yaml - - def _push_to_git( - self, - resource_type: _GitResourceType, - target: dict[str, str], - content: str, - commit_message: str, - *, - branch: Optional[str], - extension: GitExtension, - ) -> None: - data = { - **target, - resource_type.content_field: content, - "commit_message": commit_message, - "extension": extension, - **_branch_param(branch), - } - self.make_request("POST", resource_type.endpoint, data=data) - - def _get_git_push_path(self, resource_type: _GitResourceType, target: dict[str, str]) -> str: - response = self.make_request("GET", f"{resource_type.endpoint}push-path/", params=target) - return _GitPushPath.model_validate(response.json()).path diff --git a/datamasque/client/models/bundle.py b/datamasque/client/models/bundle.py deleted file mode 100644 index 0504b5d..0000000 --- a/datamasque/client/models/bundle.py +++ /dev/null @@ -1,165 +0,0 @@ -"""Export and import bundle models for the DataMasque API.""" - -from enum import Enum -from typing import Generic, Optional, TypeVar - -from pydantic import BaseModel, ConfigDict - -from datamasque.client.models.ruleset import RulesetId, RulesetType -from datamasque.client.models.ruleset_library import RulesetLibraryId - - -class BundleImportStatus(str, Enum): - """What an import did with one item of a bundle.""" - - created = "CREATED" - """No item with that ID or name existed, so it was created. Counted in `created`.""" - created_duplicate = "CREATED_DUPLICATE" - """ - A ruleset with that name existed and `overwrite_rulesets` was not set, - so it was created under a new name, given in `imported_name`. - Counted in `created`. - """ - not_created = "NOT_CREATED" - """ - The item was left unchanged, because an item with that name already has identical content, - or because a ruleset with that ID exists and `overwrite_rulesets` was not set. - Not counted in any count but `processed`. - """ - restored = "RESTORED" - """ - An item with that ID, or with that name in the same project, had been deleted, - so it was brought back with the bundle's name and content. - Counted in `restored`. - """ - updated = "UPDATED" - """ - An existing item was overwritten, because the import's `overwrite_*` option for its type was `True`. - Counted in `updated`. - """ - skipped = "SKIPPED" - """ - An existing item was left unchanged, because the import's `overwrite_*` option for its type was `False`. - Counted in `skipped`. - """ - error = "ERROR" - """The item could not be imported; `message` says why. Counted in `error`.""" - - -class BundleImportCounts(BaseModel): - """How many items of one type an import processed, and what it did with them.""" - - model_config = ConfigDict(extra="allow") - - processed: int - """The number of items of the type in the bundle.""" - created: int - """The number of `CREATED` and `CREATED_DUPLICATE` items.""" - restored: int - """The number of `RESTORED` items.""" - error: int - """The number of `ERROR` items.""" - updated: Optional[int] = None - """The number of `UPDATED` items, or `None` for rulesets imported without `overwrite_rulesets`.""" - skipped: Optional[int] = None - """The number of `SKIPPED` items, or `None` for rulesets imported without `overwrite_rulesets`.""" - - -class RulesetImportResult(BaseModel): - """What an import did with one ruleset of a bundle.""" - - model_config = ConfigDict(extra="allow") - - exported_name: str - exported_id: Optional[RulesetId] = None - """`None` for a hand-built Zip archive without the `manifest.json` that an export writes.""" - exported_project: Optional[str] = None - """The name of the project the bundle files the ruleset under, or `None` for a Version 1 bundle.""" - imported_name: Optional[str] = None - """`None` on error.""" - imported_id: Optional[RulesetId] = None - """`None` on error.""" - imported_project: Optional[str] = None - """The name of the project the ruleset is in, or `None` on error.""" - ruleset_type: RulesetType - status: BundleImportStatus - message: str - """A description of the outcome, which can change between DataMasque versions; check `status` instead.""" - - -class LibraryImportResult(BaseModel): - """What an import did with one library of a bundle.""" - - model_config = ConfigDict(extra="allow") - - exported_name: str - exported_namespace: str - """ - The folder a Version 1 bundle files the library under, which names its project; - `""` for a library in no folder, and for every library in a Version 2 bundle. - """ - exported_project: Optional[str] = None - """The name of the project the bundle files the library under, or `None` for a Version 1 bundle.""" - exported_id: Optional[RulesetLibraryId] = None - """`None` for a hand-built Zip archive without the `manifest.json` that an export writes.""" - imported_name: Optional[str] = None - """`None` on error.""" - imported_id: Optional[RulesetLibraryId] = None - """`None` on error.""" - imported_project: Optional[str] = None - """The name of the project the library is in, or `None` on error.""" - status: BundleImportStatus - message: str - """A description of the outcome, which can change between DataMasque versions; check `status` instead.""" - - -class SeedFileImportResult(BaseModel): - """What an import did with one seed file of a bundle.""" - - model_config = ConfigDict(extra="allow") - - exported_name: str - exported_project: Optional[str] = None - """The name of the project the bundle files the seed file under, or `None` for a Version 1 bundle.""" - imported_name: Optional[str] = None - """`None` on error.""" - imported_project: Optional[str] = None - """The name of the project the seed file is in, or `None` on error.""" - status: BundleImportStatus - message: str - """A description of the outcome, which can change between DataMasque versions; check `status` instead.""" - - -ItemResultT = TypeVar("ItemResultT", RulesetImportResult, LibraryImportResult, SeedFileImportResult) - - -class BundleImportItemResults(BaseModel, Generic[ItemResultT]): - """The counts and per-item results of one item type in an import.""" - - model_config = ConfigDict(extra="allow") - - metadata: BundleImportCounts - data: list[ItemResultT] - - -class BundleImportResult(BaseModel): - """The outcome of `import_bundle`, grouped by item type.""" - - model_config = ConfigDict(extra="allow") - - rulesets: BundleImportItemResults[RulesetImportResult] - libraries: Optional[BundleImportItemResults[LibraryImportResult]] = None - """`None` when the bundle has no libraries.""" - seed_files: Optional[BundleImportItemResults[SeedFileImportResult]] = None - """`None` when the bundle has no seed files.""" - fallback_project: Optional[str] = None - """ - The name of the project that took the resources that named no project, such as a Version 1 bundle's rulesets. - `None` when the import created, restored or updated no such resources. - """ - is_deprecated_bundle: bool = False - """ - Whether the bundle is in the deprecated Version 1 format, - or is a hand-built Zip archive without `manifest.json`, which is read as Version 1. - `import_bundle` sets it from the response's `Deprecation` header, which the response body does not carry. - """ diff --git a/datamasque/client/models/connection.py b/datamasque/client/models/connection.py index cde6bd0..5bec97b 100644 --- a/datamasque/client/models/connection.py +++ b/datamasque/client/models/connection.py @@ -121,15 +121,13 @@ class ConnectionConfig(BaseModel): model_config = ConfigDict(extra="allow", populate_by_name=True) name: str - project: Optional[str] = None - """ - The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, - or the project's ID as a string. - Left as `None`, a create goes into the client's `default_project`, else the server's default project. - """ id: Optional[ConnectionId] = None # Server-populated and read-only, so it is excluded when serializing a config back to the API. license_lock: Optional[LicenseLock] = Field(default=None, exclude=True) + # Its project's `-` reference, e.g. `1-default`; + # `None` until it is created on the server. + # Server-populated, so excluded when serializing. + project: Optional[str] = Field(default=None, exclude=True) # Whether the connection's protected fields are currently locked under the license edit lock. target_frozen: bool = Field(default=False, exclude=True) # The protected fields currently locked, as server config keys (e.g. `schema`, not `db_schema`); diff --git a/datamasque/client/models/discovery_config.py b/datamasque/client/models/discovery_config.py index 3d767bb..086257f 100644 --- a/datamasque/client/models/discovery_config.py +++ b/datamasque/client/models/discovery_config.py @@ -40,17 +40,13 @@ class DiscoveryConfig(BaseModel): model_config = ConfigDict(extra="allow", populate_by_name=True) name: str - project: Optional[str] = None - """ - The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, - or the project's ID as a string. - Left as `None`, a create goes into the client's `default_project`, else the server's default project. - """ yaml: Optional[str] = Field(default=None, alias="config_yaml") config_type: DiscoveryConfigType # Server-populated read-only fields, excluded from request bodies. id: Optional[DiscoveryConfigId] = Field(default=None, exclude=True) + project: Optional[str] = Field(default=None, exclude=True) + """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True) """Validation status; may be `in_progress` briefly after creating a large config.""" validation_error: Optional[str] = Field(default=None, exclude=True) diff --git a/datamasque/client/models/discovery_config_library.py b/datamasque/client/models/discovery_config_library.py index 06058bb..629160f 100644 --- a/datamasque/client/models/discovery_config_library.py +++ b/datamasque/client/models/discovery_config_library.py @@ -1,9 +1,8 @@ from datetime import datetime from typing import NewType, Optional -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field -from datamasque.client.models.project import drop_namespace from datamasque.client.models.status import ValidationStatus DiscoveryConfigLibraryId = NewType("DiscoveryConfigLibraryId", str) @@ -11,24 +10,21 @@ class DiscoveryConfigLibrary(BaseModel): """ - Represents a named, persisted YAML discovery config library. + Represents a named, namespaced, persisted YAML discovery config library. A library is untyped: the same library may be imported by both database and - file discovery configs, and its name is unique within its project. + file discovery configs, and its name is unique within a namespace. """ model_config = ConfigDict(extra="allow", populate_by_name=True) name: str - project: Optional[str] = None - """ - The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, - or the project's ID as a string. - Left as `None`, a create goes into the client's `default_project`, else the server's default project. - """ + namespace: str = "" yaml: Optional[str] = Field(default=None, alias="config_yaml") # Server-populated read-only fields, excluded from request bodies. id: Optional[DiscoveryConfigLibraryId] = Field(default=None, exclude=True) + project: Optional[str] = Field(default=None, exclude=True) + """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True) """Validation status; libraries are validated synchronously on create/update.""" validation_error: Optional[str] = Field(default=None, exclude=True) @@ -37,5 +33,3 @@ class DiscoveryConfigLibrary(BaseModel): """Number of active discovery configs that import this library.""" created: Optional[datetime] = Field(default=None, exclude=True) modified: Optional[datetime] = Field(default=None, exclude=True) - - _drop_namespace = model_validator(mode="before")(drop_namespace) diff --git a/datamasque/client/models/dm_instance.py b/datamasque/client/models/dm_instance.py index 0f82152..7b30a98 100644 --- a/datamasque/client/models/dm_instance.py +++ b/datamasque/client/models/dm_instance.py @@ -49,12 +49,6 @@ class DataMasqueInstanceConfig(BaseModel): """How long a request may take to connect, including the TLS handshake.""" read_timeout_seconds: float = DEFAULT_READ_TIMEOUT_SECONDS """How long a request may wait for the server between bytes of its response.""" - default_project: Optional[str] = None - """The project a create goes into when the resource names none, as a `-` reference (e.g. `1-default`) - or a bare project id. - - Applies to connections, rulesets, discovery configs, RG configs, table references and libraries. - Leave unset to use the server's only active project.""" @model_validator(mode="after") def _validate_auth_source(self) -> "DataMasqueInstanceConfig": diff --git a/datamasque/client/models/files.py b/datamasque/client/models/files.py index 6b8dc65..13c9846 100644 --- a/datamasque/client/models/files.py +++ b/datamasque/client/models/files.py @@ -16,8 +16,6 @@ class DataMasqueFile(BaseModel): created_date: datetime modified_date: Optional[datetime] = None id: Optional[FileId] = None - project: Optional[str] = None - """Its project's `-` reference, or `None` for a built-in seed, which belongs to no project.""" @model_validator(mode="before") @classmethod diff --git a/datamasque/client/models/git.py b/datamasque/client/models/git.py index c409f6c..df8e71a 100644 --- a/datamasque/client/models/git.py +++ b/datamasque/client/models/git.py @@ -58,32 +58,3 @@ def _collapse_git_fields(cls, data: Any) -> Any: data["git"] = snapshot return data - - -class GitFileCommit(BaseModel): - """A commit that changed a file in the git repository.""" - - model_config = ConfigDict(extra="allow", populate_by_name=True) - - commit_sha: str = Field(alias="commit") - """The commit's SHA, which the `pull_*_from_git` methods take.""" - date: datetime - """When the commit was authored.""" - message: str - """The commit message.""" - - -class GitFile(BaseModel): - """A YAML file in the git repository, as a `list_*_git_files` method lists it.""" - - path: str - """ - The file's path with its extension, - relative to the Git directory path for rulesets and to the type folder for the other types. - Pass it to the matching `pull_*_from_git` method. - """ - commits: list[GitFileCommit] - """ - The file's most recent commits, newest first (the server caps how many). - The `pull_*_from_git` methods also take the SHA of an older commit, such as one from `git log`. - """ diff --git a/datamasque/client/models/project.py b/datamasque/client/models/project.py deleted file mode 100644 index bbe4eae..0000000 --- a/datamasque/client/models/project.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Project models for the DataMasque API.""" - -from datetime import datetime -from enum import Enum -from typing import NewType, Optional, Union - -from pydantic import BaseModel, ConfigDict - -from datamasque.client.models.resource_type import ResourceTypeName - -ProjectId = NewType("ProjectId", int) - - -def drop_namespace(cls: type, data: object) -> object: - """ - Drops the deprecated `namespace` field that the server still returns on libraries. - - The models allow extra fields, so a kept `namespace` would be sent back on update, - and the server lets `namespace` override `project`. - """ - - if isinstance(data, dict) and "namespace" in data: - data = {key: value for key, value in data.items() if key != "namespace"} - - return data - - -class Project(BaseModel): - """A project: the top-level grouping that connections, rulesets and other resources belong to.""" - - model_config = ConfigDict(extra="allow") - - id: ProjectId - reference: str - """The `-` string other resources use to refer to this project.""" - name: str - description: str = "" - created: datetime - updated: datetime - has_git_synced_resources: Optional[bool] = None - """ - Whether any library, discovery config library, ruleset or discovery config in the project - that is not deleted is synced with git. - - A resource edited since its last push is not counted. - Only `get_project` sets it; other calls leave it `None`. - After a rename, a git-synced resource's next push writes a new copy under the new name and the old copy stays. - DataMasque does not move files in the repository. - """ - - -# A project, given as a `Project`, its `ProjectId`, or its `-` form such as a listed resource's `project`. -ProjectIdentifier = Union[ProjectId, Project, str] - - -def unwrap_project_id(project: ProjectIdentifier) -> ProjectId: - """ - Return the ID of `project`. - - Raises: - ValueError: `project` is a string that is not a project reference in the form `-`. - """ - if isinstance(project, Project): - return project.id - - if isinstance(project, str): - try: - return ProjectId(int(project.split("-")[0])) - except ValueError: - raise ValueError( - f'"{project}" is not a project ID or reference. ' - "Specify the ID as an integer, or a reference in the form `3-myproject`." - ) from None - - return project - - -class ProjectCreateRequest(BaseModel): - """ - Request body for `POST /api/projects/`. - - A name must not be ".", start with "DataMasque_", or contain "/", ".." or a control character, - and must differ, ignoring case, from the name of every other active project. - The server trims leading and trailing whitespace from `name` and `description`. - """ - - model_config = ConfigDict(extra="forbid") - - name: str - description: str = "" - - -class ProjectUpdateRequest(BaseModel): - """ - Request body for `PATCH /api/projects/{id}/`. - - A field left as `None` is not sent, so the server keeps its current value. - A new `name` follows the rules of `ProjectCreateRequest`. - """ - - model_config = ConfigDict(extra="forbid") - - name: Optional[str] = None - description: Optional[str] = None - - -class ProjectDeleteConflictReason(str, Enum): - """Why a project's resources cannot move to the destination project when the project is deleted.""" - - name_in_use = "name_in_use" - """The destination project already contains a resource of the same type and name.""" - file_exists = "file_exists" - """The destination project already has a file or directory where the resource's files would move to.""" - unexpected_file = "unexpected_file" - """The project's directory contains a file or directory that belongs to none of its resources.""" - - -class ProjectDeleteConflict(BaseModel): - """A conflict that prevents a project's resources moving to the destination project when the project is deleted.""" - - model_config = ConfigDict(extra="allow") - - reason: ProjectDeleteConflictReason - detail: str - """A message that describes the conflict.""" - type: Optional[ResourceTypeName] = None - """For `name_in_use` and `file_exists`: the type of the resource that would move.""" - id: Optional[str] = None - """For `name_in_use` and `file_exists`: the ID of the resource that would move.""" - name: Optional[str] = None - """For `name_in_use` and `file_exists`: the name of the resource that would move.""" - path: Optional[str] = None - """For `file_exists` and `unexpected_file`: the path of the file or directory.""" diff --git a/datamasque/client/models/resource.py b/datamasque/client/models/resource.py deleted file mode 100644 index 4f50baa..0000000 --- a/datamasque/client/models/resource.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Resource listing models for the DataMasque API.""" - -from collections.abc import Mapping, Sequence -from datetime import datetime -from enum import Enum -from typing import Optional, Union - -from pydantic import BaseModel, ConfigDict - -from datamasque.client.models.discovery_config import DiscoveryConfigType -from datamasque.client.models.resource_type import ResourceTypeName -from datamasque.client.models.runs import MaskType - - -class Resource(BaseModel): - """ - One resource in the resources listing: the summary the listing gives, not the resource's full model. - - Connections, rulesets and discovery configs are listed as the subclasses that add the field each carries. - """ - - model_config = ConfigDict(extra="allow") - - id: str - name: str - project: Optional[str] = None - """Its project's `-` reference, or `None` for a resource in no project, such as a built-in seed.""" - created: Optional[datetime] = None - """`None` for a type that does not record it, currently connections.""" - modified: Optional[datetime] = None - """`None` for a type that does not record it, currently connections.""" - position: Optional[int] = None - """ - Its index, counting from 0, among every resource the listing matched, in the listing's ordering. - It changes with the filters and ordering, so it does not identify the resource. - Requires server version 3.26.21, and is `None` on an older server. - """ - - -class ResourceOrdering(str, Enum): - """ - A column `list_resources` sorts by, with the columns that break ties after it. - - Ties are broken by type, then name, then project name, then ID, leaving out the column sorted by. - Types sort in the order `ResourceTypeName` declares them. - Names and project names compare ignoring case. - """ - - type = "type" - name = "name" - project = "project" - """ - Sorts by project name. - A resource in no project sorts after every project, - or before every project when `descending` is set. - """ - modified = "modified" - """A resource with no `modified` time sorts as the oldest.""" - - -class ConnectionResource(Resource): - """A connection in the resources listing.""" - - mask_type: MaskType - - -class RulesetResource(Resource): - """A ruleset in the resources listing.""" - - mask_type: MaskType - - -class DiscoveryConfigResource(Resource): - """A discovery config in the resources listing.""" - - config_type: DiscoveryConfigType - - -# The model each type's rows are listed as; a type not named here is listed as `Resource`. -RESOURCE_MODEL_BY_TYPE_NAME: dict[str, type[Resource]] = { - ResourceTypeName.connections.value: ConnectionResource, - ResourceTypeName.rulesets.value: RulesetResource, - ResourceTypeName.discovery_configs.value: DiscoveryConfigResource, -} - -# Resources by type, each given as a listed `Resource` or its ID, in the shape `list_resources` returns. -ResourcesByType = Mapping[ResourceTypeName, Sequence[Union[Resource, str]]] - - -class ResourceMoveConflictReason(str, Enum): - """Why a resource cannot move to the destination project.""" - - already_in_destination = "already_in_destination" - """The resource is already in the destination project.""" - name_in_use = "name_in_use" - """The destination project already contains a resource of the same type and name.""" - name_in_selection = "name_in_selection" - """Another resource of the same type and name is moving too, from another project.""" - file_exists = "file_exists" - """The destination project already has a file or directory where the resource's files would move to.""" - - -class ResourceMoveConflict(BaseModel): - """A conflict that keeps a resource from moving to the destination project.""" - - model_config = ConfigDict(extra="allow") - - reason: ResourceMoveConflictReason - detail: str - """A message that describes the conflict.""" - type: ResourceTypeName - id: str - name: str - path: Optional[str] = None - """For `file_exists`: the path of the file or directory, relative to the DataMasque files directory.""" - - -class ResourceMoveResult(BaseModel): - """The outcome of `move_resources`: every resource asked for either moved or is listed in `conflicts`.""" - - model_config = ConfigDict(extra="allow") - - moved: int - """The number of resources that moved.""" - conflicts: list[ResourceMoveConflict] - """The resources that did not move, and why.""" diff --git a/datamasque/client/models/resource_type.py b/datamasque/client/models/resource_type.py deleted file mode 100644 index edefcd3..0000000 --- a/datamasque/client/models/resource_type.py +++ /dev/null @@ -1,19 +0,0 @@ -"""The types of resource that belong to a project.""" - -from enum import Enum - - -class ResourceTypeName(str, Enum): - """A type of resource that belongs to a project, named as the resources listing groups it.""" - - connections = "connections" - connection_filesets = "connection_filesets" - oracle_wallets = "oracle_wallets" - snowflake_keys = "snowflake_keys" - table_references = "table_references" - rulesets = "rulesets" - ruleset_libraries = "ruleset_libraries" - discovery_configs = "discovery_configs" - discovery_config_libraries = "discovery_config_libraries" - ruleset_generation_configs = "ruleset_generation_configs" - seeds = "seeds" diff --git a/datamasque/client/models/rg_config.py b/datamasque/client/models/rg_config.py index 441306b..1403937 100644 --- a/datamasque/client/models/rg_config.py +++ b/datamasque/client/models/rg_config.py @@ -39,16 +39,12 @@ class RGConfig(BaseModel): model_config = ConfigDict(extra="allow", populate_by_name=True) name: str - project: Optional[str] = None - """ - The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, - or the project's ID as a string. - Left as `None`, a create goes into the client's `default_project`, else the server's default project. - """ yaml: Optional[str] = Field(default=None, alias="config_yaml") # Server-populated read-only fields, excluded from request bodies. id: Optional[RGConfigId] = Field(default=None, exclude=True) + project: Optional[str] = Field(default=None, exclude=True) + """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True) validation_errors: list[ValidationErrorDetails] = Field(default_factory=list, exclude=True) created: Optional[datetime] = Field(default=None, exclude=True) diff --git a/datamasque/client/models/ruleset.py b/datamasque/client/models/ruleset.py index a42be76..d14c2d4 100644 --- a/datamasque/client/models/ruleset.py +++ b/datamasque/client/models/ruleset.py @@ -38,17 +38,13 @@ class Ruleset(GitTrackedEntity): """Represents a ruleset.""" name: str - project: Optional[str] = None - """ - The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, - or the project's ID as a string. - Left as `None`, a create goes into the client's `default_project`, else the server's default project. - """ yaml: str = Field(default="", alias="config_yaml") ruleset_type: RulesetType = Field(default=RulesetType.database, alias="mask_type") # Server-populated read-only fields, excluded from request bodies. id: Optional[RulesetId] = Field(default=None, exclude=True) + project: Optional[str] = Field(default=None, exclude=True) + """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True) validation_errors: list[ValidationErrorDetails] = Field(default_factory=list, exclude=True) """Validation errors surfaced by the server; empty when valid.""" diff --git a/datamasque/client/models/ruleset_library.py b/datamasque/client/models/ruleset_library.py index dd54b20..68929c2 100644 --- a/datamasque/client/models/ruleset_library.py +++ b/datamasque/client/models/ruleset_library.py @@ -1,10 +1,9 @@ from datetime import datetime from typing import NewType, Optional -from pydantic import Field, model_validator +from pydantic import Field from datamasque.client.models.git import GitTrackedEntity -from datamasque.client.models.project import drop_namespace from datamasque.client.models.status import ValidationErrorDetails, ValidationStatus RulesetLibraryId = NewType("RulesetLibraryId", str) @@ -14,16 +13,13 @@ class RulesetLibrary(GitTrackedEntity): """Represents a ruleset library.""" name: str - project: Optional[str] = None - """ - The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, - or the project's ID as a string. - Left as `None`, a create goes into the client's `default_project`, else the server's default project. - """ + namespace: str = "" yaml: Optional[str] = Field(default=None, alias="config_yaml") # Server-populated read-only fields, excluded from request bodies. id: Optional[RulesetLibraryId] = Field(default=None, exclude=True) + project: Optional[str] = Field(default=None, exclude=True) + """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" is_valid: Optional[ValidationStatus] = Field(default=None, exclude=True) validation_errors: list[ValidationErrorDetails] = Field(default_factory=list, exclude=True) """Validation errors surfaced by the server; empty when valid.""" @@ -36,5 +32,3 @@ class RulesetLibrary(GitTrackedEntity): """ created: Optional[datetime] = Field(default=None, exclude=True) modified: Optional[datetime] = Field(default=None, exclude=True) - - _drop_namespace = model_validator(mode="before")(drop_namespace) diff --git a/datamasque/client/models/runs.py b/datamasque/client/models/runs.py index 6c1bd3d..09bae55 100644 --- a/datamasque/client/models/runs.py +++ b/datamasque/client/models/runs.py @@ -146,8 +146,7 @@ class RunInfo(BaseModel): """ Its project's `-` reference, e.g. `1-default`. - The project of the run's ruleset, else its discovery config, else its source connection. - The run moves when that resource moves to another project. + Taken from the ruleset's project when the run starts, and never changes after that. """ @model_validator(mode="before") diff --git a/datamasque/client/models/table_reference.py b/datamasque/client/models/table_reference.py index 5b6c9b5..e03bb38 100644 --- a/datamasque/client/models/table_reference.py +++ b/datamasque/client/models/table_reference.py @@ -73,12 +73,6 @@ class TableReference(BaseModel): model_config = ConfigDict(extra="allow", populate_by_name=True) name: str - project: Optional[str] = None - """ - The project it belongs to: the `-` reference a GET returns, such as `Project.reference`, - or the project's ID as a string. - Left as `None`, a create goes into the client's `default_project`, else the server's default project. - """ connection: Union[ConnectionId, ConnectionConfig] """The connection the data lives in. Accepts a `ConnectionId` or a `ConnectionConfig` (its `id` is extracted).""" source: str @@ -87,6 +81,8 @@ class TableReference(BaseModel): # Server-populated read-only fields, excluded from request bodies. id: Optional[TableReferenceId] = Field(default=None, exclude=True) + project: Optional[str] = Field(default=None, exclude=True) + """Its project's `-` reference, e.g. `1-default`; `None` until it is created on the server.""" created: Optional[datetime] = Field(default=None, exclude=True) modified: Optional[datetime] = Field(default=None, exclude=True) diff --git a/datamasque/client/projects.py b/datamasque/client/projects.py deleted file mode 100644 index ef64906..0000000 --- a/datamasque/client/projects.py +++ /dev/null @@ -1,148 +0,0 @@ -import logging -from typing import Iterator - -from requests import Response - -from datamasque.client.base import BaseClient -from datamasque.client.exceptions import ProjectDeleteConflictError -from datamasque.client.models.project import ( - Project, - ProjectCreateRequest, - ProjectDeleteConflict, - ProjectIdentifier, - ProjectUpdateRequest, - unwrap_project_id, -) - -logger = logging.getLogger(__name__) - - -def _get_conflicts(response: Response) -> list[ProjectDeleteConflict]: - return [ProjectDeleteConflict.model_validate(conflict) for conflict in response.json()["conflicts"]] - - -class ProjectClient(BaseClient): - """Project API methods. Mixed into `DataMasqueClient`.""" - - def iter_projects(self, *, include_archived: bool = False) -> Iterator[Project]: - """ - Lazily iterate projects via the paginated endpoint. - - Archived projects are skipped unless `include_archived` is set. - """ - - params = {"include_archived": "true"} if include_archived else None - return self._iter_paginated("/api/projects/", model=Project, params=params) - - def list_projects(self, *, include_archived: bool = False) -> list[Project]: - """ - Lists projects. - - Archived projects are left out unless `include_archived` is set. - """ - - return list(self.iter_projects(include_archived=include_archived)) - - def get_project(self, project: ProjectIdentifier) -> Project: - """ - Retrieves a single project, given as a `Project`, its `ProjectId` or its `-` form. - - Archived projects can be retrieved too. - """ - - response = self.make_request("GET", f"/api/projects/{unwrap_project_id(project)}/") - return Project.model_validate(response.json()) - - def create_project(self, request: ProjectCreateRequest) -> Project: - """ - Creates a new project on the server, and returns it as the server stored it. - - Raises: - DataMasqueApiError: the server rejects the name (see `ProjectCreateRequest`), or any other non-2xx response. - """ - - data = request.model_dump(mode="json") - response = self.make_request("POST", "/api/projects/", data=data) - project = Project.model_validate(response.json()) - logger.info('Creation of project "%s" successful', project.name) - return project - - def update_project(self, project: ProjectIdentifier, request: ProjectUpdateRequest) -> Project: - """ - Renames a project or changes its description, and returns the updated project. - - Only admin users can, and API tokens cannot. - Renaming invalidates rulesets, discovery configs and generator configs that import a library of the project - by the old project name. Imports without a project name stay valid. - It also moves the project's seed and credential files, keeping connections working. - Seed and table references in rulesets that name the old project are not rewritten. - - Raises: - DataMasqueApiError: the server rejects the name (see `ProjectCreateRequest`), - a file already exists where one of the project's files would move, - the project does not exist or is deleted, or any other non-2xx response. - """ - - project_id = unwrap_project_id(project) - data = request.model_dump(mode="json", exclude_none=True) - response = self.make_request("PATCH", f"/api/projects/{project_id}/", data=data) - updated_project = Project.model_validate(response.json()) - logger.info("Update of project %s successful", project_id) - return updated_project - - def get_project_delete_conflicts( - self, - project: ProjectIdentifier, - destination: ProjectIdentifier, - ) -> list[ProjectDeleteConflict]: - """ - Returns the conflicts that would prevent `delete_project` moving the project's resources to the destination. - - An empty list means `delete_project` can delete the project with this destination, - unless the resources in either project change first. - - Raises: - DataMasqueApiError: the destination does not exist, is deleted, or is the project; - the project does not exist or is already deleted; or any other non-2xx response. - """ - - response = self.make_request( - "GET", - f"/api/projects/{unwrap_project_id(project)}/delete-check/", - params={"destination": unwrap_project_id(destination)}, - ) - return _get_conflicts(response) - - def delete_project( - self, - project: ProjectIdentifier, - destination: ProjectIdentifier, - ) -> None: - """ - Deletes a project, after moving every resource in it to the destination project. - - When the resources cannot move, the server changes nothing. - `get_project_delete_conflicts` lists the same conflicts without trying to delete the project. - - Raises: - ProjectDeleteConflictError: the resources cannot move to the destination; its `conflicts` says why. - DataMasqueApiError: the destination does not exist, is deleted, or is the project; - the project does not exist or is already deleted; or any other non-2xx response. - """ - - project_id = unwrap_project_id(project) - response = self.make_request( - "DELETE", - f"/api/projects/{project_id}/", - params={"destination": unwrap_project_id(destination)}, - require_status_check=False, - ) - if response.status_code == 409: - raise ProjectDeleteConflictError( - str(response.json()["detail"]), - response=response, - conflicts=_get_conflicts(response), - ) - - self._raise_for_status(response) - logger.info("Deletion of project %s successful", project_id) diff --git a/datamasque/client/resources.py b/datamasque/client/resources.py deleted file mode 100644 index 380b61c..0000000 --- a/datamasque/client/resources.py +++ /dev/null @@ -1,141 +0,0 @@ -import logging -from collections import defaultdict -from collections.abc import Sequence -from typing import Optional - -from datamasque.client.base import BaseClient -from datamasque.client.models.project import ProjectIdentifier, unwrap_project_id -from datamasque.client.models.resource import ( - RESOURCE_MODEL_BY_TYPE_NAME, - Resource, - ResourceMoveConflict, - ResourceMoveResult, - ResourceOrdering, - ResourcesByType, -) -from datamasque.client.models.resource_type import ResourceTypeName - -logger = logging.getLogger(__name__) - - -def _move_request(resources: ResourcesByType, destination: ProjectIdentifier) -> dict[str, object]: - keys = [ - {"type": type_name.value, "id": resource.id if isinstance(resource, Resource) else resource} - for type_name, group in resources.items() - for resource in group - ] - if not keys: - raise ValueError("Specify at least one resource to move") - - # The API takes the destination as a string: an id or `-`. - return {"destination": str(unwrap_project_id(destination)), "resources": keys} - - -class ResourceClient(BaseClient): - """Resource listing and move API methods. Mixed into `DataMasqueClient`.""" - - def list_resources( - self, - *, - projects: Optional[Sequence[ProjectIdentifier]] = None, - types: Optional[Sequence[ResourceTypeName]] = None, - search: Optional[str] = None, - ordering: ResourceOrdering = ResourceOrdering.type, - descending: bool = False, - page_size: int = 500, - ) -> dict[ResourceTypeName, list[Resource]]: - """ - Returns the resources the caller can manage, grouped by type, fetched in pages of `page_size`. - - `projects` limits them to those projects, each given as a `Project`, its `ProjectId` or its `-` form; - `types` to those types; and `search` to resources whose name contains every word of it, ignoring case. - Each group's resources come in `ordering`, descending when `descending` is set. - Columns that break ties are always sorted ascending. - Sort the resources of every group together by `position` to get the whole listing in order. - `ordering` and `descending` require server version 3.26.21. - An older server ignores them, and each resource's `position` is `None`. - The groups are the requested types, or every type the server has when `types` is not given. - An empty `projects` or `types` matches no resources, without a request to the server. - Connections, rulesets and discovery configs are listed as `ConnectionResource`, `RulesetResource` - and `DiscoveryConfigResource`. - - Raises: - DataMasqueApiError: the server rejects the request, for example because a project does not exist. - """ - - # `requests` leaves out a parameter whose value is an empty list, which would remove the filter. - if projects is not None and not projects: - return {type_name: [] for type_name in types or list(ResourceTypeName)} - - if types is not None and not types: - return {} - - params: dict[str, object] = {} - if projects is not None: - params["project"] = [unwrap_project_id(project) for project in projects] - if types is not None: - params["type"] = [type_name.value for type_name in types] - if search is not None: - params["search"] = search - params["ordering"] = f"-{ordering.value}" if descending else ordering.value - - resources: defaultdict[ResourceTypeName, list[Resource]] = defaultdict(list) - for results in self._iter_pages("/api/resources/", params=params, page_size=page_size): - for type_name, rows in results.items(): - model = RESOURCE_MODEL_BY_TYPE_NAME.get(type_name, Resource) - resources[ResourceTypeName(type_name)].extend(model.model_validate(row) for row in rows) - - return dict(resources) - - def get_resource_move_conflicts( - self, - resources: ResourcesByType, - destination: ProjectIdentifier, - ) -> list[ResourceMoveConflict]: - """ - Checks whether `resources` can move to the destination project, without moving them. - - `resources` can contain up to 2,000 resources. - - Returns: - The conflicts that would keep any of `resources` from moving (if any). - - Raises: - ValueError: `resources` is empty. - DataMasqueApiError: the server rejects the request, for example because the destination does not exist - or `resources` contains more than 2,000 resources. - """ - - response = self.make_request("POST", "/api/resources/move-check/", data=_move_request(resources, destination)) - return [ResourceMoveConflict.model_validate(conflict) for conflict in response.json()["conflicts"]] - - def move_resources( - self, - resources: ResourcesByType, - destination: ProjectIdentifier, - ) -> ResourceMoveResult: - """ - Moves `resources` to the destination project, with their files and runs. - - `resources` can be the result of `list_resources`, or any part of it, and can contain up to 2,000 resources. - A resource that conflicts with the destination, or with another resource in `resources`, - stays where it is and is listed in the result's `conflicts`; every other resource moves. - Rulesets, discovery configs and RG configs whose library imports resolve differently after the move - are marked invalid. - - Raises: - ValueError: `resources` is empty. - DataMasqueApiError: the server rejects the move, for example because the destination does not exist, - `resources` contains more than 2,000 resources, - or the caller does not have permission to move one of the resource types. - """ - - response = self.make_request("POST", "/api/resources/move/", data=_move_request(resources, destination)) - result = ResourceMoveResult.model_validate(response.json()) - logger.info( - "Moved %d resources to project %s; %d could not move", - result.moved, - unwrap_project_id(destination), - len(result.conflicts), - ) - return result diff --git a/datamasque/client/rg_configs.py b/datamasque/client/rg_configs.py index cf5237d..adb6066 100644 --- a/datamasque/client/rg_configs.py +++ b/datamasque/client/rg_configs.py @@ -1,10 +1,9 @@ import logging -from typing import Iterator, Optional, Union +from typing import Iterator, Optional from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.pagination import Page -from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id from datamasque.client.models.rg_config import RGConfig, RGConfigId logger = logging.getLogger(__name__) @@ -34,13 +33,13 @@ def get_rg_config(self, config_id: RGConfigId) -> RGConfig: response = self.make_request("GET", f"/api/ruleset-generation-configs/{config_id}/") return RGConfig.model_validate(response.json()) - def _get_rg_config_id_by_name(self, name: str, project: Union[ProjectId, str]) -> Optional[RGConfigId]: - """Return the id of the config matching the name and project via a single list request, or `None`.""" + def _get_rg_config_id_by_name(self, name: str) -> Optional[RGConfigId]: + """Return the id of the config matching the name via a single list request, or `None`.""" response = self.make_request( "GET", "/api/ruleset-generation-configs/", - params={"name_exact": name, "project": project, "limit": 1}, + params={"name_exact": name, "limit": 1}, ) page = Page[RGConfig].model_validate(response.json()) if not page.results: @@ -55,16 +54,15 @@ def _get_rg_config_id_by_name(self, name: str, project: Union[ProjectId, str]) - return config_id - def get_rg_config_by_name(self, name: str, project: ProjectIdentifier) -> Optional[RGConfig]: + def get_rg_config_by_name(self, name: str) -> Optional[RGConfig]: """ - Looks for an RG config matching the given name (case-sensitive, exact match) in the given project. + Looks for an RG config matching the given name (case-sensitive, exact match). - RG configs are untyped and their names are unique per project. - `project` is a `Project`, its `ProjectId` or its `-` form. + RG configs are untyped and their names are unique, so the name alone identifies a single config. Returns it if found, otherwise `None`. """ - config_id = self._get_rg_config_id_by_name(name, unwrap_project_id(project)) + config_id = self._get_rg_config_id_by_name(name) if config_id is None: return None @@ -78,13 +76,10 @@ def create_rg_config(self, config: RGConfig) -> RGConfig: (`id`, `is_valid`, `validation_errors`, `created`, `modified`) and returns the config. """ - data = config.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( - config.project - ) + data = config.model_dump(exclude_none=True, by_alias=True, mode="json") response = self.make_request("POST", "/api/ruleset-generation-configs/", data=data) created = RGConfig.model_validate(response.json()) config.id = created.id - config.project = created.project config.is_valid = created.is_valid config.validation_errors = created.validation_errors config.created = created.created @@ -121,14 +116,12 @@ def update_rg_config(self, config: RGConfig) -> RGConfig: def create_or_update_rg_config(self, config: RGConfig) -> RGConfig: """ - Creates the config, or updates the existing one with the same name in the same project. + Creates the config if it doesn't exist, or updates it if one with the same name already exists. - With no `project` set, the client's `default_project` is used, else the server's only active project, - and `DataMasqueException` is raised when it has no active project or more than one. Sets the config's `id` property. """ - existing_id = self._get_rg_config_id_by_name(config.name, self._resolve_project(config.project)) + existing_id = self._get_rg_config_id_by_name(config.name) if existing_id is not None: config.id = existing_id return self.update_rg_config(config) @@ -144,15 +137,14 @@ def delete_rg_config_by_id_if_exists(self, config_id: RGConfigId) -> None: self._delete_if_exists(f"/api/ruleset-generation-configs/{config_id}/") - def delete_rg_config_by_name_if_exists(self, name: str, project: ProjectIdentifier) -> None: + def delete_rg_config_by_name_if_exists(self, name: str) -> None: """ - Deletes the RG config with the given name in the given project. + Deletes the RG config with the given name. - `project` is a `Project`, its `ProjectId` or its `-` form. No-op if no such config exists. """ - config_id = self._get_rg_config_id_by_name(name, unwrap_project_id(project)) + config_id = self._get_rg_config_id_by_name(name) if config_id is not None: self.delete_rg_config_by_id_if_exists(config_id) diff --git a/datamasque/client/ruleset_libraries.py b/datamasque/client/ruleset_libraries.py index 20003c3..ce5fddb 100644 --- a/datamasque/client/ruleset_libraries.py +++ b/datamasque/client/ruleset_libraries.py @@ -1,10 +1,9 @@ import logging -from typing import Iterator, Optional, Union +from typing import Iterator, Optional from datamasque.client.base import BaseClient -from datamasque.client.exceptions import DataMasqueApiError +from datamasque.client.exceptions import DataMasqueApiError, DataMasqueException from datamasque.client.models.pagination import Page -from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id from datamasque.client.models.ruleset import Ruleset from datamasque.client.models.ruleset_library import RulesetLibrary, RulesetLibraryId @@ -35,13 +34,17 @@ def get_ruleset_library(self, library_id: RulesetLibraryId) -> RulesetLibrary: response = self.make_request("GET", f"/api/ruleset-libraries/{library_id}/") return RulesetLibrary.model_validate(response.json()) - def _get_ruleset_library_id_by_name(self, name: str, project: Union[ProjectId, str]) -> Optional[RulesetLibraryId]: - """Return the ID of the library with the given name in the given project, or `None` if there is none.""" + def get_ruleset_library_by_name(self, name: str, namespace: str = "") -> Optional[RulesetLibrary]: + """ + Looks for a ruleset library matching the given name and namespace (case-sensitive, exact match). + + Returns it (with full YAML content) if found, otherwise None. + """ response = self.make_request( "GET", "/api/ruleset-libraries/", - params={"name_exact": name, "project": project, "limit": 1}, + params={"name_exact": name, "namespace_exact": namespace, "limit": 1}, ) page = Page[RulesetLibrary].model_validate(response.json()) if not page.results: @@ -54,20 +57,6 @@ def _get_ruleset_library_id_by_name(self, name: str, project: Union[ProjectId, s response=response, ) - return library_id - - def get_ruleset_library_by_name(self, name: str, project: ProjectIdentifier) -> Optional[RulesetLibrary]: - """ - Looks for a ruleset library matching the given name (case-sensitive, exact match) in the given project. - - `project` is a `Project`, its `ProjectId` or its `-` form. - Returns it (with full YAML content) if found, otherwise None. - """ - - library_id = self._get_ruleset_library_id_by_name(name, unwrap_project_id(project)) - if library_id is None: - return None - return self.get_ruleset_library(library_id) def create_ruleset_library(self, library: RulesetLibrary) -> RulesetLibrary: @@ -79,13 +68,10 @@ def create_ruleset_library(self, library: RulesetLibrary) -> RulesetLibrary: and returns the library. """ - data = library.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( - library.project - ) + data = library.model_dump(exclude_none=True, by_alias=True, mode="json") response = self.make_request("POST", "/api/ruleset-libraries/", data=data) created_library = RulesetLibrary.model_validate(response.json()) library.id = created_library.id - library.project = created_library.project library.is_valid = created_library.is_valid library.validation_errors = created_library.validation_errors library.unlicensed_feature_warnings = created_library.unlicensed_feature_warnings @@ -118,16 +104,14 @@ def update_ruleset_library(self, library: RulesetLibrary) -> RulesetLibrary: def create_or_update_ruleset_library(self, library: RulesetLibrary) -> RulesetLibrary: """ - Creates the library, or updates the existing one with the same name in the same project. + Creates the library if it doesn't exist, or updates it if a library with the same name already exists. - With no `project` set, the client's `default_project` is used, else the server's only active project, - and `DataMasqueException` is raised when it has no active project or more than one. Sets the library's `id` property. """ - library_id = self._get_ruleset_library_id_by_name(library.name, self._resolve_project(library.project)) - if library_id is not None: - library.id = library_id + existing = self.get_ruleset_library_by_name(library.name, library.namespace) + if existing is not None: + library.id = existing.id return self.update_ruleset_library(library) return self.create_ruleset_library(library) @@ -147,18 +131,21 @@ def delete_ruleset_library_by_id_if_exists(self, library_id: RulesetLibraryId, * self._delete_if_exists(f"/api/ruleset-libraries/{library_id}/", params=params) def delete_ruleset_library_by_name_if_exists( - self, library_name: str, project: ProjectIdentifier, *, force: bool = False + self, library_name: str, namespace: str = "", *, force: bool = False ) -> None: """ - Deletes the ruleset library with the given name in the given project. + Deletes the ruleset library with the given name and namespace. - `project` is a `Project`, its `ProjectId` or its `-` form. No-op if the library does not exist. """ - library_id = self._get_ruleset_library_id_by_name(library_name, unwrap_project_id(project)) - if library_id is not None: - self.delete_ruleset_library_by_id_if_exists(library_id, force=force) + all_libraries = self.list_ruleset_libraries() + matching = [lib for lib in all_libraries if lib.name == library_name and lib.namespace == namespace] + for lib in matching: + if lib.id is None: + raise DataMasqueException(f'Server returned a ruleset library named "{lib.name}" without an `id`.') + + self.delete_ruleset_library_by_id_if_exists(lib.id, force=force) def iter_rulesets_using_library(self, library_id: RulesetLibraryId) -> Iterator[Ruleset]: """Lazily iterate rulesets that import the given library.""" diff --git a/datamasque/client/rulesets.py b/datamasque/client/rulesets.py index 5337c93..96fc98c 100644 --- a/datamasque/client/rulesets.py +++ b/datamasque/client/rulesets.py @@ -2,7 +2,6 @@ from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueException -from datamasque.client.models.project import ProjectIdentifier, unwrap_project_id from datamasque.client.models.ruleset import Ruleset, RulesetId, RulesetType logger = logging.getLogger(__name__) @@ -19,23 +18,16 @@ def list_rulesets(self) -> list[Ruleset]: def create_or_update_ruleset(self, ruleset: Ruleset) -> Ruleset: """ - Creates the ruleset, or updates the existing one with the same name and type in the same project. + Creates or updates a ruleset. - With no `project` set, the client's `default_project` is used, else the server's only active project, - and `DataMasqueException` is raised when it has no active project or more than one. Populates the given ruleset's `id`, `is_valid`, `validation_errors`, `unlicensed_feature_warnings`, and `git` fields from the server response, and returns the same ruleset instance for convenience. """ - # The server finds the existing ruleset itself, so this only checks that an unset project resolves. - self._resolve_project(ruleset.project) - data = ruleset.model_dump(exclude_none=True, by_alias=True, mode="json") | self._default_project_field( - ruleset.project - ) + data = ruleset.model_dump(exclude_none=True, by_alias=True, mode="json") response = self.make_request("POST", "/api/rulesets/", data=data, params={"upsert": "true"}) created = Ruleset.model_validate(response.json()) ruleset.id = created.id - ruleset.project = created.project ruleset.is_valid = created.is_valid ruleset.validation_errors = created.validation_errors ruleset.unlicensed_feature_warnings = created.unlicensed_feature_warnings @@ -53,23 +45,20 @@ def delete_ruleset_by_id_if_exists(self, ruleset_id: RulesetId) -> None: self._delete_if_exists(f"/api/rulesets/{ruleset_id}/") - def delete_ruleset_by_name_if_exists( - self, ruleset_name: str, ruleset_type: RulesetType, project: ProjectIdentifier - ) -> None: + def delete_ruleset_by_name_if_exists(self, ruleset_name: str, ruleset_type: RulesetType) -> None: """ - Deletes the ruleset with the given name and type in the given project. + Deletes the ruleset with the given name and type. - Ruleset names are unique per type and project, so both are required to identify a single ruleset. - `project` is a `Project`, its `ProjectId` or its `-` form. + Ruleset names are unique per type, so a type is required to identify a single ruleset. No-op if no such ruleset exists. """ - response = self.make_request( - "GET", - "/api/v2/rulesets/", - params={"name_exact": ruleset_name, "mask_type": ruleset_type.value, "project": unwrap_project_id(project)}, - ) - for ruleset in (Ruleset.model_validate(payload) for payload in response.json()): + matching = [ + ruleset + for ruleset in self.list_rulesets() + if ruleset.name == ruleset_name and ruleset.ruleset_type is ruleset_type + ] + for ruleset in matching: if ruleset.id is None: raise DataMasqueException(f'Server returned a ruleset named "{ruleset.name}" without an `id`.') diff --git a/datamasque/client/runs.py b/datamasque/client/runs.py index 3668731..9f4c87c 100644 --- a/datamasque/client/runs.py +++ b/datamasque/client/runs.py @@ -4,13 +4,11 @@ from datamasque.client.base import BaseClient from datamasque.client.exceptions import ( - DataMasqueException, FailedToStartError, InvalidLibraryError, InvalidRulesetError, RunNotCancellableError, ) -from datamasque.client.models.connection import ConnectionId from datamasque.client.models.runs import MaskingRunRequest, RunId, RunInfo, UnfinishedRun from datamasque.client.models.status import MaskingRunStatus @@ -65,10 +63,10 @@ def get_db_discovery_result_report(self, run_id: RunId, include_selection_column return response.content return response.text - def get_unfinished_runs(self) -> dict[ConnectionId, UnfinishedRun]: - """Queries the DM instance for unfinished runs, and returns them organised by connection ID.""" + def get_unfinished_runs(self) -> dict[str, UnfinishedRun]: + """Queries the DM instance for unfinished runs, and returns them organised by connection name.""" - unfinished_runs: dict[ConnectionId, UnfinishedRun] = {} + unfinished_runs = {} for status in ( MaskingRunStatus.queued, MaskingRunStatus.running, @@ -91,22 +89,9 @@ def get_unfinished_runs(self) -> dict[ConnectionId, UnfinishedRun]: for run in data.get("results", []): unfinished_run = UnfinishedRun.model_validate(run) - source = unfinished_run.source_connection - if source.id is None: - raise DataMasqueException( - f"Server returned unfinished run {unfinished_run.id} without a source connection ID." - ) - - unfinished_runs[source.id] = unfinished_run - - destination = unfinished_run.destination_connection - if destination is not None: - if destination.id is None: - raise DataMasqueException( - f"Server returned unfinished run {unfinished_run.id} without a destination connection ID." - ) - - unfinished_runs[destination.id] = unfinished_run + unfinished_runs[unfinished_run.source_connection.name] = unfinished_run + if unfinished_run.destination_connection is not None: + unfinished_runs[unfinished_run.destination_connection.name] = unfinished_run return unfinished_runs diff --git a/datamasque/client/table_references.py b/datamasque/client/table_references.py index 04ec955..f191abf 100644 --- a/datamasque/client/table_references.py +++ b/datamasque/client/table_references.py @@ -1,9 +1,8 @@ import logging -from typing import Optional, Union +from typing import Optional from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueApiError -from datamasque.client.models.project import ProjectId, ProjectIdentifier, unwrap_project_id from datamasque.client.models.table_reference import TableReference, TableReferenceId logger = logging.getLogger(__name__) @@ -12,19 +11,16 @@ class TableReferenceClient(BaseClient): """Table reference CRUD API methods. Mixed into `DataMasqueClient`.""" - def list_table_references(self, project: Optional[ProjectIdentifier] = None) -> list[TableReference]: + def list_table_references(self) -> list[TableReference]: """ - Lists all table references, or only those in `project` when it is given. - - `project` is a `Project`, its `ProjectId` or its `-` form. + Lists all table references. Unlike most list endpoints, this one is not paginated; the server returns every table reference in a single response. Deleted table references are archived rather than removed, and are not listed. """ - params = {"project": unwrap_project_id(project)} if project is not None else None - response = self.make_request("GET", "/api/table-references/", params=params) + response = self.make_request("GET", "/api/table-references/") return [TableReference.model_validate(item) for item in response.json()] def get_table_reference(self, table_reference_id: TableReferenceId) -> TableReference: @@ -33,27 +29,26 @@ def get_table_reference(self, table_reference_id: TableReferenceId) -> TableRefe response = self.make_request("GET", f"/api/table-references/{table_reference_id}/") return TableReference.model_validate(response.json()) - def get_table_reference_by_name(self, name: str, project: ProjectIdentifier) -> Optional[TableReference]: + def get_table_reference_by_name(self, name: str) -> Optional[TableReference]: """ - Looks for a table reference with the given name (case-sensitive, exact match) in the given project. + Looks for a table reference with the given name (case-sensitive, exact match). - Names are unique across a project's live table references, so at most one can match. - `project` is a `Project`, its `ProjectId` or its `-` form. + Names are unique across live table references, so at most one can match. Returns it if found, otherwise `None`. - The endpoint takes no name filter, so the match is made client-side over the project's listing. + The endpoint takes no name filter, so the match is made client-side over the full listing. """ - matches = [reference for reference in self.list_table_references(project) if reference.name == name] + matches = [reference for reference in self.list_table_references() if reference.name == name] if not matches: return None return matches[0] - def _get_table_reference_id_by_name(self, name: str, project: Union[ProjectId, str]) -> Optional[TableReferenceId]: - """Return the ID of the table reference with the given name in the given project, or `None` if there is none.""" + def _get_table_reference_id_by_name(self, name: str) -> Optional[TableReferenceId]: + """Return the ID of the table reference with the given name, or `None` if there is none.""" - response = self.make_request("GET", "/api/table-references/", params={"project": project}) + response = self.make_request("GET", "/api/table-references/") matches = [item for item in response.json() if item.get("name") == name] if not matches: return None @@ -79,13 +74,9 @@ def create_table_reference(self, table_reference: TableReference) -> TableRefere data = table_reference.model_dump(by_alias=True, mode="json") if data.get("options") is None: data.pop("options", None) - if data.get("project") is None: - data.pop("project", None) - data |= self._default_project_field(table_reference.project) response = self.make_request("POST", "/api/table-references/", data=data) created = TableReference.model_validate(response.json()) table_reference.id = created.id - table_reference.project = created.project table_reference.options = created.options table_reference.created = created.created table_reference.modified = created.modified @@ -110,8 +101,6 @@ def update_table_reference(self, table_reference: TableReference) -> TableRefere data = table_reference.model_dump(by_alias=True, mode="json") if data.get("options") is None: data.pop("options", None) - if data.get("project") is None: - data.pop("project", None) response = self.make_request("PUT", f"/api/table-references/{table_reference.id}/", data=data) updated = TableReference.model_validate(response.json()) table_reference.options = updated.options @@ -121,16 +110,12 @@ def update_table_reference(self, table_reference: TableReference) -> TableRefere def create_or_update_table_reference(self, table_reference: TableReference) -> TableReference: """ - Creates the table reference, or updates the existing one with the same name in the same project. + Creates the table reference, or updates the existing one with the same name. - With no `project` set, the client's `default_project` is used, else the server's only active project, - and `DataMasqueException` is raised when it has no active project or more than one. Sets the table reference's `id` property. """ - existing_id = self._get_table_reference_id_by_name( - table_reference.name, self._resolve_project(table_reference.project) - ) + existing_id = self._get_table_reference_id_by_name(table_reference.name) if existing_id is not None: table_reference.id = existing_id return self.update_table_reference(table_reference) @@ -147,14 +132,13 @@ def delete_table_reference_by_id_if_exists(self, table_reference_id: TableRefere self._delete_if_exists(f"/api/table-references/{table_reference_id}/") - def delete_table_reference_by_name_if_exists(self, name: str, project: ProjectIdentifier) -> None: + def delete_table_reference_by_name_if_exists(self, name: str) -> None: """ - Deletes the table reference with the given name in the given project. + Deletes the table reference with the given name. - `project` is a `Project`, its `ProjectId` or its `-` form. No-op if no such table reference exists. """ - table_reference_id = self._get_table_reference_id_by_name(name, unwrap_project_id(project)) + table_reference_id = self._get_table_reference_id_by_name(name) if table_reference_id is not None: self.delete_table_reference_by_id_if_exists(table_reference_id) diff --git a/docs/client.models.rst b/docs/client.models.rst index 54c271f..e98789b 100644 --- a/docs/client.models.rst +++ b/docs/client.models.rst @@ -4,14 +4,6 @@ datamasque.client.models package Submodules ---------- -datamasque.client.models.bundle module --------------------------------------- - -.. automodule:: datamasque.client.models.bundle - :members: - :undoc-members: - :show-inheritance: - datamasque.client.models.connection module ------------------------------------------ @@ -52,13 +44,6 @@ datamasque.client.models.files module :undoc-members: :show-inheritance: -datamasque.client.models.git module ------------------------------------ - -.. automodule:: datamasque.client.models.git - :members: GitSnapshot, GitFile, GitFileCommit - :show-inheritance: - datamasque.client.models.ifm module ----------------------------------- @@ -75,30 +60,6 @@ datamasque.client.models.license module :undoc-members: :show-inheritance: -datamasque.client.models.project module ---------------------------------------- - -.. automodule:: datamasque.client.models.project - :members: - :undoc-members: - :show-inheritance: - -datamasque.client.models.resource module ----------------------------------------- - -.. automodule:: datamasque.client.models.resource - :members: - :undoc-members: - :show-inheritance: - -datamasque.client.models.resource_type module ---------------------------------------------- - -.. automodule:: datamasque.client.models.resource_type - :members: - :undoc-members: - :show-inheritance: - datamasque.client.models.rg_config module ----------------------------------------- diff --git a/docs/client.rst b/docs/client.rst index fabcb5f..f013c64 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -20,14 +20,6 @@ datamasque.client.base module :undoc-members: :show-inheritance: -datamasque.client.bundles module --------------------------------- - -.. automodule:: datamasque.client.bundles - :members: - :undoc-members: - :show-inheritance: - datamasque.client.connections module ------------------------------------ @@ -68,14 +60,6 @@ datamasque.client.files module :undoc-members: :show-inheritance: -datamasque.client.git module ----------------------------- - -.. automodule:: datamasque.client.git - :members: - :undoc-members: - :show-inheritance: - datamasque.client.spcs module ----------------------------- @@ -100,22 +84,6 @@ datamasque.client.license module :undoc-members: :show-inheritance: -datamasque.client.projects module ---------------------------------- - -.. automodule:: datamasque.client.projects - :members: - :undoc-members: - :show-inheritance: - -datamasque.client.resources module ----------------------------------- - -.. automodule:: datamasque.client.resources - :members: - :undoc-members: - :show-inheritance: - datamasque.client.rg_configs module ----------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 7fced60..b3183ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "datamasque-python" -version = "1.7.0" +version = "1.5.0" description = "Official Python client for the DataMasque data-masking API." authors = [ { name = "DataMasque Ltd" }, diff --git a/tests/conftest.py b/tests/conftest.py index 1052be1..317778a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,7 @@ from datamasque.client import DataMasqueClient from datamasque.client.models.dm_instance import DataMasqueInstanceConfig from datamasque.client.models.ruleset import Ruleset, RulesetType -from tests.helpers import PROJECT, database_connection_config, file_connection_config +from tests.helpers import database_connection_config, file_connection_config @pytest.fixture @@ -36,7 +36,6 @@ def existing_connection_json(): return { "id": "1", "name": "an_existing_connection", - "project": PROJECT, "mask_type": "database", "db_type": "mysql", "host": "my-host", @@ -52,7 +51,6 @@ def existing_rulesets_json(): { "id": "1", "name": "db_masking_ruleset", - "project": PROJECT, "mask_type": "database", "config_yaml": "version: '1.0'", "is_valid": "valid", @@ -60,7 +58,6 @@ def existing_rulesets_json(): { "id": "2", "name": "file_masking_ruleset", - "project": PROJECT, "mask_type": "file", "config_yaml": "version: '1.0'", "is_valid": "invalid", @@ -72,7 +69,6 @@ def existing_rulesets_json(): def ruleset(): return Ruleset( name="test_ruleset", - project=PROJECT, yaml="version: '1.0'\ntasks: []", ruleset_type=RulesetType.database, ) diff --git a/tests/helpers.py b/tests/helpers.py index 5c21f4e..2303cfc 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -9,41 +9,9 @@ S3ConnectionConfig, SnowflakeConnectionConfig, ) -from datamasque.client.models.project import Project, ProjectId fake = Faker() -PROJECT = "1-default" -"""The project reference the tests put resources in.""" - -PROJECT_ID = ProjectId(1) -"""The ID of `PROJECT`, as the client methods take it.""" - -OTHER_PROJECT = Project.model_validate( - { - "id": 2, - "reference": "2-banking", - "name": "Banking", - "description": "", - "created": "2026-10-01T10:00:00Z", - "updated": "2026-10-01T10:00:00Z", - } -) -"""A second project, for tests that pass a `Project` to a client method.""" - - -def active_projects_page(count: int) -> dict[str, object]: - """Builds the `GET /api/projects/?limit=1` response of a server with `count` active projects, `PROJECT` first.""" - project = { - "id": PROJECT_ID, - "reference": PROJECT, - "name": "Default", - "description": "", - "created": "2026-10-01T10:00:00Z", - "updated": "2026-10-01T10:00:00Z", - } - return {"count": count, "next": None, "previous": None, "results": [project] if count else []} - def parse_multipart_form(request) -> dict: # noqa: C901 """ @@ -101,7 +69,6 @@ def parse_multipart_form(request) -> dict: # noqa: C901 def database_connection_config(): return DatabaseConnectionConfig( - project=PROJECT, name=fake.word(), user=fake.user_name(), password=fake.password(), @@ -115,7 +82,6 @@ def database_connection_config(): def file_connection_config(): return S3ConnectionConfig( - project=PROJECT, name=fake.word(), base_directory=fake.uri_path(), bucket=fake.uri_page(), @@ -128,7 +94,6 @@ def sample_mounted_share_connection_json(*, id, name): return { "name": name, "id": id, - "project": PROJECT, "mask_type": "file", "type": "mounted_share_connection", "base_directory": "", @@ -153,7 +118,6 @@ def make_ok_response() -> Response: def snowflake_connection_config(): return SnowflakeConnectionConfig( - project=PROJECT, name="snowflake", database="test_db", user="snowflake_user", diff --git a/tests/test_bundles.py b/tests/test_bundles.py deleted file mode 100644 index 3939511..0000000 --- a/tests/test_bundles.py +++ /dev/null @@ -1,238 +0,0 @@ -"""Tests for exporting and importing bundles with the DataMasque client.""" - -import json -from io import BytesIO -from pathlib import Path - -import pytest -import requests_mock - -from datamasque.client import ( - BundleImportStatus, - DataMasqueApiError, - DataMasqueClient, - Ruleset, - RulesetId, - RulesetType, -) -from tests.helpers import OTHER_PROJECT, parse_multipart_form - -BUNDLE = b"PK\x03\x04 bundle" - -V2_IMPORT_RESPONSE = { - "rulesets": { - "metadata": {"processed": 1, "created": 1, "restored": 0, "error": 0}, - "data": [ - { - "exported_name": "Customers", - "exported_project": "payments", - "exported_id": "9d641e97-adf7-4f22-9089-afc3711bf222", - "imported_name": "Customers", - "imported_id": "9d641e97-adf7-4f22-9089-afc3711bf222", - "imported_project": "payments", - "ruleset_type": "database", - "status": "CREATED", - "message": 'Ruleset named "Customers" did not exist, and was created.', - } - ], - }, - "libraries": { - "metadata": {"processed": 1, "created": 0, "restored": 0, "error": 0, "updated": 0, "skipped": 1}, - "data": [ - { - "exported_name": "pii_masks", - "exported_namespace": "", - "exported_project": "payments", - "exported_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", - "imported_name": "pii_masks", - "imported_namespace": "payments", - "imported_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", - "imported_project": "payments", - "status": "SKIPPED", - "message": 'A library with ID "a1b2c3d4-e5f6-7890-abcd-ef1234567890" already exists.', - } - ], - }, -} - -V1_IMPORT_RESPONSE = { - "rulesets": { - "metadata": {"processed": 1, "created": 1, "restored": 0, "error": 0}, - "data": [ - { - "exported_name": "Customers", - "exported_project": None, - "exported_id": None, - "imported_name": "Customers", - "imported_id": "04ea20f0-ad4c-498e-881f-b0bc79d83ba7", - "imported_project": "Banking", - "ruleset_type": "file", - "status": "CREATED", - "message": 'Ruleset named "Customers" did not exist, and was created.', - } - ], - }, - "seed_files": { - "metadata": {"processed": 1, "created": 0, "restored": 0, "error": 1, "updated": 0, "skipped": 0}, - "data": [ - { - "exported_name": "names.csv", - "exported_project": None, - "imported_name": None, - "imported_project": None, - "status": "ERROR", - "message": "The seed file could not be written.", - } - ], - }, - "fallback_project": "Banking", -} - - -def test_export_bundle_exports_everything_by_default(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - m.post("http://test-server/api/export/v2/", content=BUNDLE, headers={"Content-Type": "application/zip"}) - bundle = client.export_bundle() - - assert bundle == BUNDLE - assert m.last_request.json() == {"include_seed_files": False} - - -def test_export_bundle_sends_the_rulesets_and_seed_file_option(client: DataMasqueClient) -> None: - ruleset = Ruleset(name="Customers") - ruleset.id = RulesetId("9d641e97-adf7-4f22-9089-afc3711bf222") - with requests_mock.Mocker() as m: - m.post("http://test-server/api/export/v2/", content=BUNDLE) - client.export_bundle([ruleset, RulesetId("04ea20f0-ad4c-498e-881f-b0bc79d83ba7")], include_seed_files=True) - - assert m.last_request.json() == { - "include_seed_files": True, - "ruleset_ids": ["9d641e97-adf7-4f22-9089-afc3711bf222", "04ea20f0-ad4c-498e-881f-b0bc79d83ba7"], - } - - -def test_export_bundle_rejects_an_empty_sequence_of_rulesets(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - with pytest.raises(ValueError, match="at least one ruleset"): - client.export_bundle([]) - - assert not m.called - - -def test_export_bundle_rejects_a_ruleset_not_yet_created(client: DataMasqueClient) -> None: - with pytest.raises(ValueError, match="not been created"): - client.export_bundle([Ruleset(name="Customers")]) - - -@pytest.mark.parametrize("rulesets", [RulesetId("9d641e97-adf7-4f22-9089-afc3711bf222"), Ruleset(name="Customers")]) -def test_export_bundle_rejects_a_single_ruleset(client: DataMasqueClient, rulesets: object) -> None: - with pytest.raises(TypeError, match="as a sequence"): - client.export_bundle(rulesets) # type: ignore[arg-type] - - -def test_import_bundle_uploads_the_archive_with_default_options(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - m.post("http://test-server/api/import/v2/", json=V2_IMPORT_RESPONSE, status_code=201) - result = client.import_bundle(BUNDLE) - - form = parse_multipart_form(m.last_request) - assert form == { - "overwrite_libraries": "False", - "overwrite_seed_files": "False", - "zip_archive": {"filename": "bundle.zip", "content_type": "application/zip", "content": BUNDLE}, - } - (ruleset,) = result.rulesets.data - assert ruleset.imported_project == "payments" - assert ruleset.ruleset_type is RulesetType.database - assert ruleset.status is BundleImportStatus.created - assert result.rulesets.metadata.updated is None - assert result.libraries is not None - assert result.libraries.metadata.skipped == 1 - assert result.libraries.data[0].status is BundleImportStatus.skipped - assert result.seed_files is None - assert result.fallback_project is None - assert result.is_deprecated_bundle is False - - -@pytest.mark.parametrize(("overwrite", "form_value"), [(True, "True"), (False, "False")]) -def test_import_bundle_sends_its_options( - client: DataMasqueClient, tmp_path: Path, overwrite: bool, form_value: str -) -> None: - bundle_path = tmp_path / "export.zip" - bundle_path.write_bytes(BUNDLE) - with requests_mock.Mocker() as m: - m.post("http://test-server/api/import/v2/", json=V2_IMPORT_RESPONSE) - client.import_bundle( - bundle_path, - overwrite_rulesets=overwrite, - overwrite_libraries=overwrite, - overwrite_seed_files=overwrite, - fallback_project=OTHER_PROJECT, - ) - - form = parse_multipart_form(m.last_request) - assert form["zip_archive"]["filename"] == "export.zip" - assert form["overwrite_rulesets"] == form_value - assert form["overwrite_libraries"] == form_value - assert form["overwrite_seed_files"] == form_value - assert form["fallback_project"] == "2" - - -def test_import_bundle_uploads_all_of_a_partly_read_file(client: DataMasqueClient) -> None: - bundle = BytesIO(BUNDLE) - bundle.read(4) - with requests_mock.Mocker() as m: - m.post("http://test-server/api/import/v2/", json=V2_IMPORT_RESPONSE) - client.import_bundle(bundle) - - archive = parse_multipart_form(m.last_request)["zip_archive"] - assert archive["filename"] == "bundle.zip" - assert archive["content"] == BUNDLE - - -def test_import_bundle_rejects_a_fallback_project_that_is_not_a_reference(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - with pytest.raises(ValueError, match="not a project ID or reference"): - client.import_bundle(BUNDLE, fallback_project="banking") - - assert not m.called - - -def test_import_bundle_does_not_send_the_client_default_project(client: DataMasqueClient) -> None: - client.default_project = "1-default" - with requests_mock.Mocker() as m: - m.post("http://test-server/api/import/v2/", json=V1_IMPORT_RESPONSE) - client.import_bundle(BUNDLE) - - assert "fallback_project" not in parse_multipart_form(m.last_request) - - -def test_import_bundle_reports_a_v1_bundle_and_its_fallback_project(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - m.post( - "http://test-server/api/import/v2/", - json=V1_IMPORT_RESPONSE, - status_code=201, - headers={"Deprecation": "true"}, - ) - result = client.import_bundle(BUNDLE, fallback_project="2-banking") - - assert parse_multipart_form(m.last_request)["fallback_project"] == "2" - assert result.is_deprecated_bundle is True - assert result.fallback_project == "Banking" - assert result.rulesets.data[0].exported_project is None - assert result.seed_files is not None - (seed_file,) = result.seed_files.data - assert seed_file.status is BundleImportStatus.error - assert seed_file.imported_project is None - - -def test_import_bundle_raises_when_the_server_rejects_it(client: DataMasqueClient) -> None: - detail = 'The bundle names the project "payments", which does not exist. Creating it requires an administrator.' - with requests_mock.Mocker() as m: - m.post("http://test-server/api/import/v2/", status_code=403, text=json.dumps({"detail": detail})) - with pytest.raises(DataMasqueApiError) as error: - client.import_bundle(BUNDLE) - - assert error.value.response.status_code == 403 - assert error.value.response.json() == {"detail": detail} diff --git a/tests/test_connections.py b/tests/test_connections.py index 9f22271..cbc3f7d 100644 --- a/tests/test_connections.py +++ b/tests/test_connections.py @@ -1,12 +1,10 @@ """Tests for `ConnectionClient` (CRUD + Snowflake-specific behaviour).""" from datetime import datetime, timezone -from typing import Union import pytest import requests_mock -from datamasque.client import DataMasqueClient, Project, ProjectId from datamasque.client.exceptions import DataMasqueApiError, DataMasqueException from datamasque.client.models.connection import ( AzureConnectionConfig, @@ -33,9 +31,6 @@ validate_connection, ) from tests.helpers import ( - OTHER_PROJECT, - PROJECT, - PROJECT_ID, sample_mounted_share_connection_json, snowflake_connection_config, ) @@ -101,69 +96,6 @@ def test_create_or_update_connection_create_fail(client, connection_config, exis client.create_or_update_connection(connection_config) -def test_create_or_update_connection_sends_the_project( - client: DataMasqueClient, connection_config: ConnectionConfig -) -> None: - """The lookup for an existing connection is filtered to the connection's project, and the body carries it.""" - with requests_mock.Mocker() as m: - m.get("http://test-server/api/connections/", json=[], status_code=200) - m.post("http://test-server/api/connections/", json={"id": "2"}, status_code=201) - - client.create_or_update_connection(connection_config) - - assert m.request_history[0].qs == {"project": [PROJECT]} - assert m.last_request.json()["project"] == PROJECT - - -def test_create_or_update_connection_ignores_same_name_in_another_project( - client: DataMasqueClient, connection_config: ConnectionConfig -) -> None: - """A same-named connection in another project is not updated, because the listing is filtered by project.""" - other_project_connection = sample_mounted_share_connection_json(name=connection_config.name, id="1") | { - "project": "2-banking" - } - with requests_mock.Mocker() as m: - m.get("http://test-server/api/connections/", json=[other_project_connection], status_code=200) - m.get( - f"http://test-server/api/connections/?project={PROJECT}", - json=[], - status_code=200, - complete_qs=True, - ) - m.post("http://test-server/api/connections/", json={"id": "2"}, status_code=201) - - result = client.create_or_update_connection(connection_config) - - assert result.id == ConnectionId("2") - assert [request.method for request in m.request_history] == ["GET", "POST"] - - -@pytest.mark.parametrize("project", [ProjectId(2), OTHER_PROJECT]) -def test_list_connections_filters_by_project(client: DataMasqueClient, project: Union[ProjectId, Project]) -> None: - with requests_mock.Mocker() as m: - m.get("http://test-server/api/connections/", json=[], status_code=200) - client.list_connections(project) - - assert m.last_request.qs == {"project": ["2"]} - - -def test_list_connections_keeps_same_name_in_different_projects(client: DataMasqueClient) -> None: - """Connection names are unique per project, so an unfiltered listing can hold two connections with one name.""" - connections = [ - sample_mounted_share_connection_json(name="shared", id="1"), - sample_mounted_share_connection_json(name="shared", id="2") | {"project": OTHER_PROJECT.reference}, - ] - with requests_mock.Mocker() as m: - m.get("http://test-server/api/connections/", json=connections, status_code=200) - result = client.list_connections() - - assert m.last_request.qs == {} - assert [(connection.id, connection.project) for connection in result] == [ - ("1", PROJECT), - ("2", OTHER_PROJECT.reference), - ] - - def test_list_connnections(client): with requests_mock.Mocker() as m: m.get( @@ -172,7 +104,6 @@ def test_list_connnections(client): # Realistic API data, hence why it's so verbose { "name": "s3", - "project": PROJECT, "bucket": "my-s3-bucket", "base_directory": "", "type": "s3_connection", @@ -186,7 +117,6 @@ def test_list_connnections(client): }, { "name": "azure", - "project": PROJECT, "type": "azure_blob_connection", "base_directory": "", "mask_type": "file", @@ -201,7 +131,6 @@ def test_list_connnections(client): }, { "name": "mounted_share_dest", - "project": PROJECT, "type": "mounted_share_connection", "base_directory": "dest", "mask_type": "file", @@ -220,7 +149,6 @@ def test_list_connnections(client): "db_type": "mysql", "database": "mydatabase", "name": "mysql", - "project": PROJECT, "schema": "", "is_read_only": False, "password_encrypted": "some_base64_here", @@ -235,7 +163,6 @@ def test_list_connnections(client): "version": "1.0", "mask_type": "database", "name": "db_dynamo", - "project": PROJECT, "s3_bucket_name": "my-dynamo-staging-bucket", "dynamo_append_datetime": False, "dynamo_append_suffix": "-masked", @@ -266,7 +193,6 @@ def test_list_connnections(client): "db_type": "mssql_linked", "database": "database_name", "name": "mssql-linked", - "project": PROJECT, "schema": "", "is_read_only": False, "password_encrypted": "some_base64_here", @@ -282,7 +208,6 @@ def test_list_connnections(client): "version": "1.0", "mask_type": "database", "name": "db_dynamo_2", - "project": PROJECT, "s3_bucket_name": "my-dynamo-staging-bucket-2", "dynamo_append_datetime": False, "dynamo_append_suffix": "-masked", @@ -307,7 +232,6 @@ def test_list_connnections(client): "db_type": "snowflake", "database": "icicle", "name": "snowflake", - "project": PROJECT, "schema": "snowball", "snowflake_role": "snowballs do indeed roll", "snowflake_account_id": "ABCDEF-123456", @@ -329,7 +253,6 @@ def test_list_connnections(client): "db_type": "snowflake", "database": "igloo", "name": "snowflake_minimal_with_key", - "project": PROJECT, "snowflake_account_id": "ACCOUNT-1234", "snowflake_warehouse": "clothing_store", "snowflake_private_key": "2831289a-4398-abcd-4112-fe09a1239f89", @@ -458,11 +381,10 @@ def test_delete_connection_by_name(client): status_code=200, ) - client.delete_connection_by_name_if_exists(connection_name, PROJECT_ID) + client.delete_connection_by_name_if_exists(connection_name) assert m.call_count == 3 assert m.request_history[0].method == "GET" - assert m.request_history[0].qs == {"project": [str(PROJECT_ID)]} assert m.request_history[1].method == "DELETE" assert m.request_history[2].method == "DELETE" @@ -483,7 +405,7 @@ def test_delete_connection_that_does_not_exist(client): ], status_code=200, ) - client.delete_connection_by_name_if_exists("my_connection", PROJECT_ID) + client.delete_connection_by_name_if_exists("my_connection") assert m.call_count == 1 assert m.request_history[0].method == "GET" @@ -545,7 +467,6 @@ def test_list_snowflake_connection_from_older_server_drops_external_stage(client "db_type": "snowflake", "database": "s3_db", "name": "snowflake_s3", - "project": PROJECT, "snowflake_account_id": "S3-ACCOUNT", "snowflake_warehouse": "s3_warehouse", "snowflake_storage_integration_name": "s3_integration", @@ -573,7 +494,6 @@ def test_s3_connection_model_validate(): payload = { "id": "88dabb63-aca5-4cc4-8f76-f78736a42f39", "name": "s3", - "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "data/", @@ -599,7 +519,6 @@ def test_connection_model_validate_parses_license_lock() -> None: payload = { "id": "88dabb63-aca5-4cc4-8f76-f78736a42f39", "name": "s3", - "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "data/", @@ -623,7 +542,6 @@ def test_connection_model_validate_null_license_lock() -> None: payload = { "id": "id-1", "name": "s3", - "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "", @@ -641,7 +559,6 @@ def test_connection_model_validate_parses_freeze_fields() -> None: payload = { "id": "db-1", "name": "pg", - "project": PROJECT, "mask_type": "database", "db_type": "postgres", "host": "h", @@ -665,7 +582,6 @@ def test_connection_freeze_fields_default_when_absent() -> None: payload = { "id": "db-1", "name": "pg", - "project": PROJECT, "mask_type": "database", "db_type": "postgres", "host": "h", @@ -694,7 +610,6 @@ def test_connection_freeze_fields_default_when_absent() -> None: { "id": "id-1", "name": "s3", - "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "", @@ -709,7 +624,6 @@ def test_connection_freeze_fields_default_when_absent() -> None: { "id": "db-1", "name": "pg", - "project": PROJECT, "mask_type": "database", "db_type": "postgres", "host": "h", @@ -725,7 +639,6 @@ def test_connection_freeze_fields_default_when_absent() -> None: { "id": "dyn-1", "name": "dyn", - "project": PROJECT, "mask_type": "database", "db_type": "dynamo_db", "s3_bucket_name": "bucket", @@ -740,7 +653,6 @@ def test_connection_freeze_fields_default_when_absent() -> None: { "id": "mongo-1", "name": "mongo", - "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -759,7 +671,6 @@ def test_connection_freeze_fields_default_when_absent() -> None: { "id": "sf-1", "name": "snowflake", - "project": PROJECT, "mask_type": "database", "db_type": "snowflake", "user": "snowman", @@ -802,7 +713,6 @@ def test_s3_connection_model_validate_no_iam_role(): payload = { "id": "id-1", "name": "s3", - "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "", @@ -819,7 +729,6 @@ def test_databricks_connection_model_validate(): payload = { "id": "db-id-1", "name": "databricks", - "project": PROJECT, "mask_type": "database", "db_type": "databricks", "server_hostname": "adb-1234.azuredatabricks.net", @@ -846,7 +755,6 @@ def test_databricks_connection_model_validate_blanks_encrypted_token(): payload = { "id": "db-id-2", "name": "databricks", - "project": PROJECT, "mask_type": "database", "db_type": "databricks", "server_hostname": "adb-1234.azuredatabricks.net", @@ -866,7 +774,6 @@ def test_validate_connection_dispatches_databricks(): payload = { "id": "db-id-3", "name": "databricks", - "project": PROJECT, "mask_type": "database", "db_type": "databricks", "server_hostname": "adb-1234.azuredatabricks.net", @@ -885,7 +792,6 @@ def test_azure_connection_model_validate_blanks_encrypted_connection_string(): payload = { "id": "490502e5-5bf6-4abb-b67b-c6091d40ecf0", "name": "azure", - "project": PROJECT, "mask_type": "file", "type": "azure_blob_connection", "base_directory": "", @@ -908,7 +814,6 @@ def test_gcs_connection_model_validate_blanks_encrypted_service_account_key(): payload = { "id": "0f9f5d2e-8d1a-4a1f-9f57-3f2c1b0c9a11", "name": "gcs", - "project": PROJECT, "mask_type": "file", "type": "gcs_connection", "base_directory": "inbound", @@ -930,7 +835,6 @@ def test_gcs_connection_model_validate_blanks_encrypted_service_account_key(): def test_gcs_connection_serializes_the_key_for_create(): conn = GcsConnectionConfig( - project=PROJECT, name="gcs", bucket="my-gcs-bucket", service_account_key='{"type": "service_account"}', @@ -960,7 +864,6 @@ def test_database_connection_model_validate_drops_schema_for_mysql(): payload = { "id": "f0557fb3-1c9a-4cb1-bcf4-9699cf496bf7", "name": "mysql", - "project": PROJECT, "mask_type": "database", "db_type": "mysql", "host": "my-mysql-host", @@ -983,7 +886,6 @@ def test_database_connection_model_validate_keeps_schema_for_postgres(): payload = { "id": "abc", "name": "pg", - "project": PROJECT, "mask_type": "database", "db_type": "postgres", "host": "pg-host", @@ -1002,7 +904,6 @@ def test_database_connection_model_validate_databricks_lakebase(): payload = { "id": "abc-lakebase", "name": "lakebase", - "project": PROJECT, "mask_type": "database", "db_type": "databricks_lakebase", "host": "lakebase-host", @@ -1024,7 +925,6 @@ def test_mssql_linked_connection_model_validate_includes_linked_server(): payload = { "id": "48a7af45-f63f-4e05-bf9f-7b1cc3a0e89d", "name": "mssql-linked", - "project": PROJECT, "mask_type": "database", "db_type": "mssql_linked", "host": "mssql-linked-host", @@ -1047,7 +947,6 @@ def test_dynamo_connection_model_validate_with_sse(): payload = { "id": "d7257552-0485-4806-b0fb-d72b4d268073", "name": "db_dynamo", - "project": PROJECT, "mask_type": "database", "db_type": "dynamo_db", "s3_bucket_name": "my-dynamo-staging-bucket", @@ -1076,7 +975,6 @@ def test_dynamo_connection_model_validate_without_sse_uses_default(): payload = { "id": "id-2", "name": "db_dynamo", - "project": PROJECT, "mask_type": "database", "db_type": "dynamo_db", "s3_bucket_name": "bucket", @@ -1098,7 +996,6 @@ def test_snowflake_connection_model_validate_drops_external_stage_location(stage payload = { "id": "f0557fb3-1c9a-4cb1-bcf4-9699cf496bf7", "name": "snowflake", - "project": PROJECT, "mask_type": "database", "db_type": "snowflake", "user": "snowman", @@ -1132,7 +1029,6 @@ def test_snowflake_connection_model_validate_minimal_payload(): payload = { "id": "id-3", "name": "snowflake", - "project": PROJECT, "mask_type": "database", "db_type": "snowflake", "user": "frosty", @@ -1153,7 +1049,6 @@ def test_connection_config_dispatch_picks_subclass(): s3_payload = { "id": "id-s3", "name": "s3", - "project": PROJECT, "mask_type": "file", "type": "s3_connection", "base_directory": "", @@ -1164,7 +1059,6 @@ def test_connection_config_dispatch_picks_subclass(): db_payload = { "id": "id-pg", "name": "pg", - "project": PROJECT, "mask_type": "database", "db_type": "postgres", "host": "h", @@ -1194,7 +1088,6 @@ def test_dynamo_connection_round_trip_with_iam_role_and_prefix(): payload = { "id": "id-dynamo-1", "name": "db_dynamo", - "project": PROJECT, "mask_type": "database", "db_type": "dynamo_db", "s3_bucket_name": "staging-bucket", @@ -1218,7 +1111,6 @@ def test_dynamo_connection_round_trip_with_iam_role_and_prefix(): def test_dynamo_connection_model_dump_omits_unset_iam_role_and_prefix(): """When the new optional fields are unset, `model_dump` must omit them entirely (not send `null`).""" conn = DynamoConnectionConfig( - project=PROJECT, name="db_dynamo", s3_bucket_name="bucket", dynamo_append_datetime=False, @@ -1237,7 +1129,6 @@ def test_dynamo_model_validate_defaults_to_none_when_fields_absent(): payload = { "id": "id-dynamo-2", "name": "db_dynamo", - "project": PROJECT, "mask_type": "database", "db_type": "dynamo_db", "s3_bucket_name": "bucket", @@ -1255,7 +1146,6 @@ def test_dynamo_model_validate_defaults_to_none_when_fields_absent(): def test_mongo_connection_model_dump_minimal(): """An unauthenticated, plain TCP connection sends only the required keys plus the defaulted booleans.""" conn = MongoConnectionConfig( - project=PROJECT, name="mongo", host="mongo.example", database="people", @@ -1274,7 +1164,6 @@ def test_mongo_connection_model_dump_minimal(): def test_mongo_connection_model_dump_full(): conn = MongoConnectionConfig( - project=PROJECT, name="mongo", host="mongo.example", port=27018, @@ -1306,7 +1195,6 @@ def test_mongo_connection_model_dump_full(): def test_mongo_connection_model_dump_omits_falsy_optional_flags(): """`tls`, `direct_connection`, `user`, `password`, and `replica_set` are only sent when truthy.""" conn = MongoConnectionConfig( - project=PROJECT, name="mongo", host="mongo.example", database="people", @@ -1324,7 +1212,7 @@ def test_mongo_connection_model_dump_omits_falsy_optional_flags(): def test_mongo_connection_retry_writes_defaults_true_and_is_omitted(): """retry_writes defaults to True (matching the server default), so it is left out of the payload.""" - conn = MongoConnectionConfig(project=PROJECT, name="mongo", host="mongo.example", database="people") + conn = MongoConnectionConfig(name="mongo", host="mongo.example", database="people") assert conn.retry_writes is True api_dict = conn.model_dump(exclude_none=True, by_alias=True, mode="json") assert "retry_writes" not in api_dict @@ -1332,18 +1220,14 @@ def test_mongo_connection_retry_writes_defaults_true_and_is_omitted(): def test_mongo_connection_sends_retry_writes_when_disabled(): """retry_writes is sent only when explicitly disabled (e.g. AWS DocumentDB rejects retryable writes).""" - conn = MongoConnectionConfig( - project=PROJECT, name="mongo", host="mongo.example", database="people", retry_writes=False - ) + conn = MongoConnectionConfig(name="mongo", host="mongo.example", database="people", retry_writes=False) api_dict = conn.model_dump(exclude_none=True, by_alias=True, mode="json") assert api_dict["retry_writes"] is False def test_documentdb_connection_sends_retry_writes_when_disabled(): """A DocumentDB connection inherits retry_writes and sends it when disabled.""" - conn = DocumentDbConnectionConfig( - project=PROJECT, name="docdb", host="docdb.example", database="people", retry_writes=False - ) + conn = DocumentDbConnectionConfig(name="docdb", host="docdb.example", database="people", retry_writes=False) api_dict = conn.model_dump(exclude_none=True, by_alias=True, mode="json") assert api_dict["db_type"] == "documentdb" assert api_dict["retry_writes"] is False @@ -1353,7 +1237,6 @@ def test_mongo_connection_model_validate_blanks_encrypted_password(): payload = { "id": "mongo-id-1", "name": "mongo", - "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -1385,7 +1268,6 @@ def test_mongo_connection_model_validate_defaults_when_optional_fields_missing() payload = { "id": "mongo-id-2", "name": "mongo-min", - "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -1406,7 +1288,6 @@ def test_connection_config_dispatch_picks_mongo_subclass(): payload = { "id": "mongo-id-3", "name": "mongo", - "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -1417,7 +1298,6 @@ def test_connection_config_dispatch_picks_mongo_subclass(): def test_mongo_tls_ca_fields_emitted_when_tls_enabled(): conn = MongoConnectionConfig( - project=PROJECT, name="mongo", host="mongo.example", database="people", @@ -1432,7 +1312,7 @@ def test_mongo_tls_ca_fields_emitted_when_tls_enabled(): def test_mongo_tls_ca_fields_omitted_when_unset_but_tls_enabled(): - conn = MongoConnectionConfig(project=PROJECT, name="mongo", host="mongo.example", database="people", tls=True) + conn = MongoConnectionConfig(name="mongo", host="mongo.example", database="people", tls=True) d = conn.model_dump(exclude_none=True, by_alias=True, mode="json") assert d["tls"] is True assert "tls_ca_file" not in d @@ -1442,7 +1322,6 @@ def test_mongo_tls_ca_fields_omitted_when_unset_but_tls_enabled(): def test_mongo_tls_ca_fields_omitted_when_tls_disabled(): """Even when set, the TLS-dependent fields are not sent while TLS is disabled.""" conn = MongoConnectionConfig( - project=PROJECT, name="mongo", host="mongo.example", database="people", @@ -1459,7 +1338,6 @@ def test_mongo_tls_ca_fields_roundtrip(): payload = { "id": "mongo-tls-1", "name": "mongo", - "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -1478,7 +1356,6 @@ def test_mongo_tls_ca_fields_default_when_missing(): payload = { "id": "mongo-tls-2", "name": "mongo", - "project": PROJECT, "mask_type": "database", "db_type": "mongodb", "host": "mongo.example", @@ -1491,7 +1368,6 @@ def test_mongo_tls_ca_fields_default_when_missing(): def test_documentdb_connection_model_dump(): conn = DocumentDbConnectionConfig( - project=PROJECT, name="docdb", host="dtq-documentdb.cluster.example", database="people", @@ -1515,7 +1391,6 @@ def test_connection_config_dispatch_picks_documentdb_subclass(): payload = { "id": "docdb-id-1", "name": "docdb", - "project": PROJECT, "mask_type": "database", "db_type": "documentdb", "host": "dtq-documentdb.cluster.example", @@ -1529,7 +1404,6 @@ def test_connection_config_dispatch_picks_documentdb_subclass(): def _cosmosdb_connection(**overrides) -> CosmosDbConnectionConfig: """Build a Cosmos DB config, leaving the port to the connection type's own default.""" return CosmosDbConnectionConfig( - project=PROJECT, name="cosmos", host="dtq-cosmos.mongo.cosmos.azure.com", database="people", @@ -1575,7 +1449,6 @@ def test_connection_config_dispatch_picks_cosmosdb_subclass(): payload = { "id": "cosmos-id-1", "name": "cosmos", - "project": PROJECT, "mask_type": "database", "db_type": "cosmosdb", "host": "dtq-cosmos.mongo.cosmos.azure.com", @@ -1600,7 +1473,6 @@ def test_database_connection_config_rejects_document_store_database_types( """`DatabaseConnectionConfig` is for SQL engines; each document store has its own class.""" with pytest.raises(ValueError, match=message): DatabaseConnectionConfig( - project=PROJECT, name="doc-store", host="doc-store.example", port=27017, @@ -1614,7 +1486,6 @@ def test_database_connection_config_rejects_document_store_database_types( def test_create_or_update_mongo_connection(client): """End-to-end: a Mongo connection round-trips through `create_or_update_connection`.""" conn = MongoConnectionConfig( - project=PROJECT, name="mongo", host="mongo.example", database="people", @@ -1642,7 +1513,6 @@ def test_create_or_update_mongo_connection(client): def _database_connection(database_type: DatabaseType, **extra) -> DatabaseConnectionConfig: return DatabaseConnectionConfig( - project=PROJECT, name="conn", host="mydb.abc123.ap-southeast-2.rds.amazonaws.com", port=5432, @@ -1737,7 +1607,6 @@ def test_database_connection_config_rejects_engines_with_their_own_class( ) -> None: with pytest.raises(ValueError, match=message): DatabaseConnectionConfig( - project=PROJECT, name="own-class", host="db.example", port=1, @@ -1750,7 +1619,6 @@ def test_database_connection_config_rejects_engines_with_their_own_class( def test_cassandra_connection_serializes_password_as_dbpassword(): conn = CassandraConnectionConfig( - project=PROJECT, name="cassandra", host="cassandra.example", database="people", @@ -1775,9 +1643,9 @@ def test_cassandra_connection_serializes_password_as_dbpassword(): def test_cassandra_connection_without_credentials_omits_user_and_password(): """Cassandra allows unauthenticated access; the server rejects a username sent without a password.""" - d = CassandraConnectionConfig( - project=PROJECT, name="cassandra", host="cassandra.example", database="people" - ).model_dump(exclude_none=True, by_alias=True, mode="json") + d = CassandraConnectionConfig(name="cassandra", host="cassandra.example", database="people").model_dump( + exclude_none=True, by_alias=True, mode="json" + ) assert "user" not in d assert "dbpassword" not in d @@ -1787,7 +1655,6 @@ def test_connection_config_dispatch_picks_cassandra_subclass(): payload = { "id": "cassandra-id-1", "name": "cassandra", - "project": PROJECT, "mask_type": "database", "db_type": "cassandra", "host": "cassandra.example", @@ -1811,7 +1678,6 @@ def test_connection_config_dispatch_picks_cassandra_subclass(): def test_sap_ase_connection_serializes_without_a_schema(): """ASE has no schema setting: the schema is the table owner.""" conn = SapAseConnectionConfig( - project=PROJECT, name="sap_ase", host="ase.example", database="people", @@ -1837,9 +1703,9 @@ def test_sap_ase_connection_serializes_without_a_schema(): def test_sap_ase_connection_omits_unset_tls_options(): - d = SapAseConnectionConfig( - project=PROJECT, name="sap_ase", host="ase.example", database="people", user="sa" - ).model_dump(exclude_none=True, by_alias=True, mode="json") + d = SapAseConnectionConfig(name="sap_ase", host="ase.example", database="people", user="sa").model_dump( + exclude_none=True, by_alias=True, mode="json" + ) assert "tls_server_name" not in d assert "connect_timeout" not in d @@ -1850,7 +1716,6 @@ def test_connection_config_dispatch_picks_sap_ase_subclass(): payload = { "id": "sap-ase-id-1", "name": "sap_ase", - "project": PROJECT, "mask_type": "database", "db_type": "sap_ase", "host": "ase.example", @@ -1869,7 +1734,6 @@ def test_connection_config_dispatch_picks_sap_ase_subclass(): def test_salesforce_connection_serializes_for_create(): conn = SalesforceConnectionConfig( - project=PROJECT, name="salesforce", instance_url="https://example--uat.sandbox.my.salesforce.com", login_url="https://example--uat.sandbox.my.salesforce.com", @@ -1896,7 +1760,6 @@ def test_salesforce_connection_serializes_for_create(): def test_salesforce_connection_without_passphrase_omits_it(): """An unencrypted private key needs no passphrase.""" d = SalesforceConnectionConfig( - project=PROJECT, name="salesforce", instance_url="https://example.my.salesforce.com", login_url="https://example.my.salesforce.com", @@ -1912,7 +1775,6 @@ def test_connection_config_dispatch_picks_salesforce_subclass(): payload = { "id": "salesforce-id-1", "name": "salesforce", - "project": PROJECT, "mask_type": "database", "db_type": "salesforce", "instance_url": "https://example.my.salesforce.com", diff --git a/tests/test_default_project.py b/tests/test_default_project.py deleted file mode 100644 index 899ad5b..0000000 --- a/tests/test_default_project.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Tests for the client's `default_project`, which a create uses when the resource names no project.""" - -from collections.abc import Callable -from typing import Any, Optional - -import pytest -import requests_mock - -from datamasque.client import DataMasqueClient -from datamasque.client.models.connection import ConnectionId -from datamasque.client.models.discovery_config import DiscoveryConfig -from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary -from datamasque.client.models.dm_instance import DataMasqueInstanceConfig -from datamasque.client.models.rg_config import RGConfig -from datamasque.client.models.ruleset import Ruleset, RulesetType -from datamasque.client.models.ruleset_library import RulesetLibrary -from datamasque.client.models.table_reference import TableReference -from tests.helpers import active_projects_page, database_connection_config - -BASE_URL = "http://test-server" - - -def make_client(default_project: Optional[str] = None) -> DataMasqueClient: - return DataMasqueClient( - DataMasqueInstanceConfig(base_url=BASE_URL, username="u", password="p", default_project=default_project) - ) - - -def create_ruleset(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: - m.post(f"{BASE_URL}/api/rulesets/?upsert=true", json={"id": "r-1", "name": "r", "project": "2-acme"}) - ruleset = Ruleset(name="r", yaml="version: '1.0'", ruleset_type=RulesetType.database, project=project) - return client.create_or_update_ruleset(ruleset) - - -def create_connection(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: - m.get(f"{BASE_URL}/api/connections/", json=[]) - m.post(f"{BASE_URL}/api/connections/", json={"id": "c-1", "project": "2-acme"}) - connection = database_connection_config() - connection.project = project - return client.create_or_update_connection(connection) - - -def create_discovery_config(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: - m.post( - f"{BASE_URL}/api/discovery/configs/", - json={"id": "dc-1", "name": "dc", "config_type": "database", "project": "2-acme"}, - ) - config = DiscoveryConfig(name="dc", yaml="labels: []\n", config_type="database", project=project) - return client.create_discovery_config(config) - - -def create_rg_config(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: - m.post(f"{BASE_URL}/api/ruleset-generation-configs/", json={"id": "rg-1", "name": "rg", "project": "2-acme"}) - return client.create_rg_config(RGConfig(name="rg", yaml="x: 1\n", project=project)) - - -def create_table_reference(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: - m.post( - f"{BASE_URL}/api/table-references/", - json={"id": "tr-1", "name": "tr", "connection": "c-1", "source": "people.csv", "project": "2-acme"}, - ) - table_reference = TableReference(name="tr", connection=ConnectionId("c-1"), source="people.csv", project=project) - return client.create_table_reference(table_reference) - - -def create_ruleset_library(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: - m.post(f"{BASE_URL}/api/ruleset-libraries/", json={"id": "rl-1", "name": "rl", "project": "2-acme"}) - return client.create_ruleset_library(RulesetLibrary(name="rl", yaml="version: '1.0'\n", project=project)) - - -def create_discovery_config_library(client: DataMasqueClient, project: Optional[str], m: requests_mock.Mocker) -> Any: - m.post(f"{BASE_URL}/api/discovery/config-libraries/", json={"id": "dl-1", "name": "dl", "project": "2-acme"}) - return client.create_discovery_config_library( - DiscoveryConfigLibrary(name="dl", yaml="labels: []\n", project=project) - ) - - -Create = Callable[[DataMasqueClient, Optional[str], requests_mock.Mocker], Any] - -CREATES = [ - pytest.param(create_ruleset, id="ruleset"), - pytest.param(create_connection, id="connection"), - pytest.param(create_discovery_config, id="discovery_config"), - pytest.param(create_rg_config, id="rg_config"), - pytest.param(create_table_reference, id="table_reference"), - pytest.param(create_ruleset_library, id="ruleset_library"), - pytest.param(create_discovery_config_library, id="discovery_config_library"), -] - - -@pytest.mark.parametrize("create", CREATES) -def test_a_create_sends_the_resources_project_over_the_default(create: Create) -> None: - with requests_mock.Mocker() as m: - create(make_client(default_project="1-default"), "2-acme", m) - - assert m.last_request.json()["project"] == "2-acme" - - -@pytest.mark.parametrize("create", CREATES) -def test_a_create_sends_the_default_project_when_the_resource_names_none(create: Create) -> None: - with requests_mock.Mocker() as m: - create(make_client(default_project="1-default"), None, m) - - assert m.last_request.json()["project"] == "1-default" - assert all(request.path != "/api/projects/" for request in m.request_history) - - -@pytest.mark.parametrize("create", CREATES) -def test_a_create_with_no_default_project_sends_no_project(create: Create) -> None: - with requests_mock.Mocker() as m: - m.get(f"{BASE_URL}/api/projects/", json=active_projects_page(1)) - create(make_client(), None, m) - - assert "project" not in m.last_request.json() - - -def test_a_connection_update_sends_no_project() -> None: - """An update leaves the connection in its own project, whatever the default.""" - client = make_client(default_project="1-default") - connection = database_connection_config() - connection.project = None - with requests_mock.Mocker() as m: - m.get(f"{BASE_URL}/api/connections/", json=[{**connection.model_dump(mode="json"), "id": "c-1"}]) - m.put(f"{BASE_URL}/api/connections/c-1/", json={"id": "c-1", "project": "2-acme"}) - - client.create_or_update_connection(connection) - - assert m.last_request.method == "PUT" - assert "project" not in m.last_request.json() - - -def test_an_upsert_looks_for_the_name_in_the_default_project() -> None: - with requests_mock.Mocker() as m: - create_connection(make_client(default_project="1-default"), None, m) - - lookup = m.request_history[0] - assert (lookup.method, lookup.path, lookup.qs) == ("GET", "/api/connections/", {"project": ["1-default"]}) diff --git a/tests/test_discovery.py b/tests/test_discovery.py index a42a396..83f9348 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -46,12 +46,10 @@ from datamasque.client.models.connection import ConnectionId, DatabaseConnectionConfig, DatabaseType from datamasque.client.models.data_selection import SelectedColumns, SelectedFileData, UserSelection from datamasque.client.models.status import AsyncRulesetGenerationTaskStatus -from tests.helpers import PROJECT, parse_multipart_form, sample_mounted_share_connection_json +from tests.helpers import parse_multipart_form DISCOVERY_CONFIG_ID = "aaaaaaaa-1111-2222-3333-444444444444" RG_CONFIG_ID = "bbbbbbbb-1111-2222-3333-444444444444" -CONNECTION_PROJECT = "2-banking" -"""The project of the connection the generated rulesets come from.""" def test_generate_ruleset(client): @@ -161,14 +159,6 @@ def test_poll_async_ruleset_generation(client): assert status is AsyncRulesetGenerationTaskStatus.failed -def mock_source_connection(m: requests_mock.Mocker, connection_id: ConnectionId) -> None: - """Mocks the connection lookup that `get_generated_rulesets` makes to find the rulesets' project.""" - m.get( - f"http://test-server/api/connections/{connection_id}/", - json=sample_mounted_share_connection_json(id=connection_id, name="source") | {"project": CONNECTION_PROJECT}, - ) - - def test_get_generated_rulesets_success(client): connection_id = ConnectionId("1") yaml_content_1 = b""" @@ -195,7 +185,6 @@ def test_get_generated_rulesets_success(client): """ with requests_mock.Mocker() as m: - mock_source_connection(m, connection_id) m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", json={"status": "finished"}, @@ -222,7 +211,6 @@ def test_get_generated_rulesets_success(client): assert rulesets[0].yaml == yaml_content_1.decode("utf-8") assert rulesets[1].name == "ruleset2" assert rulesets[1].yaml == yaml_content_2.decode("utf-8") - assert [ruleset.project for ruleset in rulesets] == [CONNECTION_PROJECT, CONNECTION_PROJECT] def test_get_generated_rulesets_finished_with_warnings_success(client): @@ -241,7 +229,6 @@ def test_get_generated_rulesets_finished_with_warnings_success(client): """ with requests_mock.Mocker() as m: - mock_source_connection(m, connection_id) m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", json={"status": "finished_with_warnings"}, @@ -272,7 +259,6 @@ def test_get_generated_rulesets_empty_archive_raises(client): connection_id = ConnectionId("1") with requests_mock.Mocker() as m: - mock_source_connection(m, connection_id) m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", json={"status": "finished"}, @@ -301,7 +287,6 @@ def test_get_generated_rulesets_from_selection_success(client): generated_yaml = 'version: "1.0"\ntasks:\n- type: mask_table\n table: users\n key: id\n rules: []\n' with requests_mock.Mocker() as m: - mock_source_connection(m, connection_id) # Status check and the redirect target are the same URL; both resolve to the same JSON. m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", @@ -320,14 +305,12 @@ def test_get_generated_rulesets_from_selection_success(client): assert rulesets[0].yaml == generated_yaml # The server doesn't return a name — callers set one before create_or_update_ruleset. assert rulesets[0].name == "generated_ruleset" - assert rulesets[0].project == CONNECTION_PROJECT def test_get_generated_rulesets_from_selection_empty_ruleset_raises(client): """A finished task with no `generated_ruleset` in the JSON body raises a clear error.""" connection_id = ConnectionId("1") with requests_mock.Mocker() as m: - mock_source_connection(m, connection_id) m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", json={"status": "finished", "generated_ruleset": None}, @@ -393,7 +376,6 @@ def test_get_generated_rulesets_download_fail(client): connection_id = ConnectionId("1") with requests_mock.Mocker() as m: - mock_source_connection(m, connection_id) m.get( f"http://test-server/api/async-generate-ruleset/{connection_id}/", json={"status": "finished"}, @@ -678,7 +660,6 @@ def test_schema_discovery_request_model_dump_includes_set_fields(): def test_discovery_requests_accept_connection_config_objects(): """All three discovery request models accept a full `ConnectionConfig` and extract its `id`.""" connection = DatabaseConnectionConfig( - project=PROJECT, id=ConnectionId("conn-uuid"), name="prod_db", db_type=DatabaseType.postgres, @@ -1261,16 +1242,14 @@ def test_schema_discovery_from_config_request_accepts_discovery_config_id(): def test_schema_discovery_from_config_request_unwraps_discovery_config_model(): """Passing a full `DiscoveryConfig` object substitutes its `id` for the wire payload.""" - config = DiscoveryConfig( - project=PROJECT, name="my_cfg", config_type="database", id=DiscoveryConfigId(DISCOVERY_CONFIG_ID) - ) + config = DiscoveryConfig(name="my_cfg", config_type="database", id=DiscoveryConfigId(DISCOVERY_CONFIG_ID)) req = SchemaDiscoveryFromConfigRequest(connection="conn-1", discovery_config=config) assert req.model_dump(exclude_none=True, mode="json")["discovery_config"] == DISCOVERY_CONFIG_ID def test_schema_discovery_from_config_request_rejects_unsaved_discovery_config(): """A `DiscoveryConfig` without an `id` cannot be used yet — raises immediately.""" - config = DiscoveryConfig(project=PROJECT, name="my_cfg", config_type="database") + config = DiscoveryConfig(name="my_cfg", config_type="database") with pytest.raises(ValueError, match="id is None"): SchemaDiscoveryFromConfigRequest(connection="conn-1", discovery_config=config) @@ -1373,9 +1352,7 @@ def test_start_schema_discovery_run_from_config_sends_schemas(client): def test_start_file_data_discovery_run_from_config_sends_discovery_config(client): """`start_file_data_discovery_run_from_config` posts only the connection and discovery_config id.""" - config = DiscoveryConfig( - project=PROJECT, name="my_cfg", config_type="file", id=DiscoveryConfigId(DISCOVERY_CONFIG_ID) - ) + config = DiscoveryConfig(name="my_cfg", config_type="file", id=DiscoveryConfigId(DISCOVERY_CONFIG_ID)) req = FileDataDiscoveryFromConfigRequest(connection="conn-1", discovery_config=config) with requests_mock.Mocker() as m: m.post("http://test-server/api/run-file-data-discovery/v2/", json={"id": 99}, status_code=201) @@ -1641,7 +1618,7 @@ def test_generate_ruleset_with_rg_config_none_sends_null(client): def test_ruleset_generation_with_rg_config_request_unwraps_rg_config_model(): """Passing a full `RGConfig` object substitutes its `id` for the wire payload.""" - config = RGConfig(project=PROJECT, name="my_rgc", id=RGConfigId(RG_CONFIG_ID)) + config = RGConfig(name="my_rgc", id=RGConfigId(RG_CONFIG_ID)) req = RulesetGenerationWithRGConfigRequest( connection="conn-1", selected_columns={"public": {"users": ["email"]}}, @@ -1652,7 +1629,7 @@ def test_ruleset_generation_with_rg_config_request_unwraps_rg_config_model(): def test_ruleset_generation_with_rg_config_request_rejects_unsaved_rg_config(): """An `RGConfig` without an `id` cannot be used yet — raises immediately.""" - config = RGConfig(project=PROJECT, name="my_rgc") + config = RGConfig(name="my_rgc") with pytest.raises(ValueError, match="id is None"): RulesetGenerationWithRGConfigRequest( connection="conn-1", @@ -1782,7 +1759,7 @@ def test_start_async_ruleset_generation_with_rg_config_unwraps_rg_config_model(c """A full `RGConfig` object is unwrapped to its id in the request body.""" connection_id = ConnectionId("1") selected_columns = SelectedColumns(columns={"public": {"users": ["col1"]}}) - config = RGConfig(project=PROJECT, name="my_rgc", id=RGConfigId(RG_CONFIG_ID)) + config = RGConfig(name="my_rgc", id=RGConfigId(RG_CONFIG_ID)) with requests_mock.Mocker() as m: m.post(f"http://test-server/api/async-generate-ruleset/v2/{connection_id}/", status_code=201) diff --git a/tests/test_discovery_config_libraries.py b/tests/test_discovery_config_libraries.py index c73f76b..20a0d4d 100644 --- a/tests/test_discovery_config_libraries.py +++ b/tests/test_discovery_config_libraries.py @@ -13,7 +13,6 @@ ) from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.status import ValidationStatus -from tests.helpers import PROJECT, PROJECT_ID LIBRARY_ID_1 = "aaaaaaaa-1111-2222-3333-444444444444" LIBRARY_ID_2 = "bbbbbbbb-1111-2222-3333-444444444444" @@ -26,7 +25,7 @@ def sample_library_list_response() -> list[dict[str, Any]]: { "id": LIBRARY_ID_1, "name": "my_library", - "project": PROJECT, + "namespace": "org", "is_valid": "valid", "validation_error": None, "usage_count": 0, @@ -36,7 +35,7 @@ def sample_library_list_response() -> list[dict[str, Any]]: { "id": LIBRARY_ID_2, "name": "another_library", - "project": PROJECT, + "namespace": "", "is_valid": "invalid", "validation_error": "bad yaml", "usage_count": 2, @@ -52,7 +51,7 @@ def sample_library_detail_response() -> dict[str, Any]: return { "id": LIBRARY_ID_1, "name": "my_library", - "project": PROJECT, + "namespace": "org", "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "valid", "validation_error": None, @@ -66,14 +65,14 @@ def sample_library_detail_response() -> dict[str, Any]: def config_library() -> DiscoveryConfigLibrary: return DiscoveryConfigLibrary( name="test_library", - project=PROJECT, + namespace="test_ns", yaml="labels: []\nmetadata_rules: []\nidd_rules: []\n", ) def test_library_is_untyped() -> None: """A library carries no `config_type`; the same library serves both config types.""" - library = DiscoveryConfigLibrary(name="test_library", project=PROJECT, yaml="labels: []\n") + library = DiscoveryConfigLibrary(name="test_library", yaml="labels: []\n") assert "config_type" not in library.model_dump(by_alias=True) @@ -91,7 +90,7 @@ def test_list_discovery_config_libraries( assert len(libraries) == 2 assert libraries[0].id == DiscoveryConfigLibraryId(LIBRARY_ID_1) assert libraries[0].name == "my_library" - assert libraries[0].project == PROJECT + assert libraries[0].namespace == "org" assert libraries[0].yaml is None assert libraries[0].is_valid is ValidationStatus.valid assert libraries[0].usage_count == 0 @@ -125,7 +124,7 @@ def test_get_discovery_config_library(client: DataMasqueClient, sample_library_d assert library.id == DiscoveryConfigLibraryId(LIBRARY_ID_1) assert library.name == "my_library" - assert library.project == PROJECT + assert library.namespace == "org" assert library.yaml == "labels: []\nmetadata_rules: []\nidd_rules: []\n" assert library.is_valid is ValidationStatus.valid assert library.usage_count == 0 @@ -147,12 +146,67 @@ def test_get_discovery_config_library_by_name_found( json=sample_library_detail_response, status_code=200, ) - library = client.get_discovery_config_library_by_name("my_library", PROJECT_ID) + library = client.get_discovery_config_library_by_name("my_library", "org") assert library is not None assert library.name == "my_library" assert library.yaml == "labels: []\nmetadata_rules: []\nidd_rules: []\n" - assert m.request_history[0].qs == {"name_exact": ["my_library"], "project": [str(PROJECT_ID)]} + assert "name_exact=my_library" in m.request_history[0].url + + +def test_get_discovery_config_library_by_name_matches_namespace_client_side( + client: DataMasqueClient, sample_library_detail_response: dict[str, Any] +) -> None: + """The server's `namespace_exact` filter ignores empty strings, so namespace matching happens client-side.""" + same_name_two_namespaces = [ + { + "id": LIBRARY_ID_1, + "name": "my_library", + "namespace": "org", + "is_valid": "valid", + "created": "2025-01-01T12:00:00Z", + "modified": "2025-01-02T12:00:00Z", + }, + { + "id": LIBRARY_ID_2, + "name": "my_library", + "namespace": "", + "is_valid": "valid", + "created": "2025-01-01T12:00:00Z", + "modified": "2025-01-02T12:00:00Z", + }, + ] + + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/discovery/config-libraries/", + json=same_name_two_namespaces, + status_code=200, + ) + m.get( + f"http://test-server/api/discovery/config-libraries/{LIBRARY_ID_1}/", + json=sample_library_detail_response, + status_code=200, + ) + library = client.get_discovery_config_library_by_name("my_library", "org") + + assert library is not None + assert library.id == DiscoveryConfigLibraryId(LIBRARY_ID_1) + + +def test_get_discovery_config_library_by_name_default_namespace_does_not_match_other_namespaces( + client: DataMasqueClient, sample_library_list_response: list[dict[str, Any]] +) -> None: + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/discovery/config-libraries/", + json=[sample_library_list_response[0]], + status_code=200, + ) + library = client.get_discovery_config_library_by_name("my_library") + + assert library is None + assert m.call_count == 1 def test_get_discovery_config_library_by_name_not_found(client: DataMasqueClient) -> None: @@ -162,7 +216,7 @@ def test_get_discovery_config_library_by_name_not_found(client: DataMasqueClient json=[], status_code=200, ) - library = client.get_discovery_config_library_by_name("nonexistent", PROJECT_ID) + library = client.get_discovery_config_library_by_name("nonexistent") assert library is None @@ -172,7 +226,7 @@ def test_get_discovery_config_library_by_name_raises_when_server_omits_id(client list_response_without_id = [ { "name": "my_library", - "project": PROJECT, + "namespace": "org", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -186,14 +240,14 @@ def test_get_discovery_config_library_by_name_raises_when_server_omits_id(client status_code=200, ) with pytest.raises(DataMasqueApiError, match="without an `id`"): - client.get_discovery_config_library_by_name("my_library", PROJECT_ID) + client.get_discovery_config_library_by_name("my_library", "org") def test_create_discovery_config_library(client: DataMasqueClient, config_library: DiscoveryConfigLibrary) -> None: create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "valid", "validation_error": None, @@ -220,7 +274,7 @@ def test_create_discovery_config_library(client: DataMasqueClient, config_librar request_body = m.last_request.json() assert request_body == { "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", } @@ -231,7 +285,7 @@ def test_create_discovery_config_library_reports_validation_error( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "invalid", "validation_error": "metadata_rules is not a list", @@ -258,7 +312,7 @@ def test_update_discovery_config_library(client: DataMasqueClient, config_librar update_response = { "id": LIBRARY_ID_1, "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "valid", "validation_error": None, @@ -284,7 +338,7 @@ def test_update_discovery_config_library(client: DataMasqueClient, config_librar request_body = m.last_request.json() assert request_body == { "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", } @@ -312,7 +366,7 @@ def test_create_or_update_discovery_config_library_create( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "valid", "validation_error": None, @@ -335,9 +389,8 @@ def test_create_or_update_discovery_config_library_create( assert result.id == DiscoveryConfigLibraryId(LIBRARY_ID_1) assert m.request_history[0].method == "GET" - assert m.request_history[0].qs == {"name_exact": ["test_library"], "project": [PROJECT]} + assert "name_exact=test_library" in m.request_history[0].url assert m.request_history[1].method == "POST" - assert m.request_history[1].json()["project"] == PROJECT def test_create_or_update_discovery_config_library_update( @@ -347,7 +400,7 @@ def test_create_or_update_discovery_config_library_update( { "id": LIBRARY_ID_1, "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "is_valid": "valid", "created": "2025-06-01T10:00:00Z", "modified": "2025-06-01T10:00:00Z", @@ -356,7 +409,7 @@ def test_create_or_update_discovery_config_library_update( update_response = { "id": LIBRARY_ID_1, "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "is_valid": "valid", "validation_error": None, @@ -427,10 +480,10 @@ def test_delete_discovery_config_library_by_name( f"http://test-server/api/discovery/config-libraries/{LIBRARY_ID_1}/", status_code=204, ) - client.delete_discovery_config_library_by_name_if_exists("my_library", PROJECT_ID) + client.delete_discovery_config_library_by_name_if_exists("my_library", "org") assert m.request_history[0].method == "GET" - assert m.request_history[0].qs == {"name_exact": ["my_library"], "project": [str(PROJECT_ID)]} + assert "name_exact=my_library" in m.request_history[0].url assert m.request_history[1].method == "DELETE" @@ -443,45 +496,17 @@ def test_delete_discovery_config_library_by_name_not_found( json=sample_library_list_response, status_code=200, ) - client.delete_discovery_config_library_by_name_if_exists("nonexistent", PROJECT_ID) + client.delete_discovery_config_library_by_name_if_exists("nonexistent") assert m.call_count == 1 assert m.request_history[0].method == "GET" def test_create_discovery_config_library_with_empty_yaml_raises(client: DataMasqueClient) -> None: - library = DiscoveryConfigLibrary(name="test_library", project=PROJECT, yaml="") + library = DiscoveryConfigLibrary(name="test_library", yaml="") with requests_mock.Mocker() as m: with pytest.raises(ValueError, match="without YAML content"): client.create_discovery_config_library(library) assert not any(request.method == "POST" for request in m.request_history) - - -def test_namespace_in_a_response_is_dropped_and_not_sent_back( - client: DataMasqueClient, sample_library_detail_response: dict[str, Any] -) -> None: - """The server lets a sent `namespace` override `project`, so the client never echoes one back.""" - response = {**sample_library_detail_response, "namespace": "org"} - - with requests_mock.Mocker() as m: - m.get( - f"http://test-server/api/discovery/config-libraries/{LIBRARY_ID_1}/", - json=response, - status_code=200, - ) - m.put( - f"http://test-server/api/discovery/config-libraries/{LIBRARY_ID_1}/", - json=response, - status_code=200, - ) - library = client.get_discovery_config_library(DiscoveryConfigLibraryId(LIBRARY_ID_1)) - client.update_discovery_config_library(library) - - assert "namespace" not in library.model_dump() - assert m.last_request.json() == { - "name": "my_library", - "project": PROJECT, - "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", - } diff --git a/tests/test_discovery_configs.py b/tests/test_discovery_configs.py index a700251..d34c47d 100644 --- a/tests/test_discovery_configs.py +++ b/tests/test_discovery_configs.py @@ -8,7 +8,7 @@ from pydantic import JsonValue, ValidationError from datamasque.client import DataMasqueClient -from datamasque.client.exceptions import DataMasqueApiError +from datamasque.client.exceptions import DataMasqueApiError, DataMasqueException from datamasque.client.models.discovery_config import ( DiscoveryConfig, DiscoveryConfigId, @@ -16,7 +16,6 @@ unwrap_discovery_config_id, ) from datamasque.client.models.status import ValidationStatus -from tests.helpers import PROJECT, PROJECT_ID CONFIG_ID_1 = "aaaaaaaa-1111-2222-3333-444444444444" CONFIG_ID_2 = "bbbbbbbb-1111-2222-3333-444444444444" @@ -32,7 +31,6 @@ def sample_config_list_response() -> dict[str, Any]: { "id": CONFIG_ID_1, "name": "my_config", - "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -41,7 +39,6 @@ def sample_config_list_response() -> dict[str, Any]: { "id": CONFIG_ID_2, "name": "another_config", - "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-02-01T12:00:00Z", @@ -56,7 +53,6 @@ def sample_config_detail_response() -> dict[str, Any]: return { "id": CONFIG_ID_1, "name": "my_config", - "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "config_type": "database", "archived": False, @@ -69,7 +65,6 @@ def sample_config_detail_response() -> dict[str, Any]: def discovery_config() -> DiscoveryConfig: return DiscoveryConfig( name="test_config", - project=PROJECT, yaml="labels: []\nmetadata_rules: []\nidd_rules: []\n", config_type="database", ) @@ -100,7 +95,6 @@ def test_list_discovery_configs_pagination(client: DataMasqueClient) -> None: { "id": CONFIG_ID_1, "name": "c1", - "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -109,7 +103,6 @@ def test_list_discovery_configs_pagination(client: DataMasqueClient) -> None: { "id": CONFIG_ID_2, "name": "c2", - "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -125,7 +118,6 @@ def test_list_discovery_configs_pagination(client: DataMasqueClient) -> None: { "id": "cccccccc-1111-2222-3333-444444444444", "name": "c3", - "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -179,7 +171,6 @@ def test_get_discovery_config_by_name_found( { "id": CONFIG_ID_1, "name": "my_config", - "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -199,16 +190,12 @@ def test_get_discovery_config_by_name_found( json=sample_config_detail_response, status_code=200, ) - config = client.get_discovery_config_by_name("my_config", DiscoveryConfigType.database, PROJECT_ID) + config = client.get_discovery_config_by_name("my_config", DiscoveryConfigType.database) assert config is not None assert config.name == "my_config" - assert m.request_history[0].qs == { - "name_exact": ["my_config"], - "config_type": ["database"], - "project": [str(PROJECT_ID)], - "limit": ["1"], - } + assert "name_exact=my_config" in m.request_history[0].url + assert "config_type=database" in m.request_history[0].url def test_get_discovery_config_by_name_not_found(client: DataMasqueClient) -> None: @@ -220,7 +207,7 @@ def test_get_discovery_config_by_name_not_found(client: DataMasqueClient) -> Non json=empty_response, status_code=200, ) - config = client.get_discovery_config_by_name("nonexistent", DiscoveryConfigType.database, PROJECT_ID) + config = client.get_discovery_config_by_name("nonexistent", DiscoveryConfigType.database) assert config is None @@ -233,7 +220,6 @@ def test_get_discovery_config_by_name_raises_when_server_omits_id(client: DataMa "results": [ { "name": "my_config", - "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -249,14 +235,13 @@ def test_get_discovery_config_by_name_raises_when_server_omits_id(client: DataMa status_code=200, ) with pytest.raises(DataMasqueApiError, match="without an `id`"): - client.get_discovery_config_by_name("my_config", DiscoveryConfigType.database, PROJECT_ID) + client.get_discovery_config_by_name("my_config", DiscoveryConfigType.database) def test_create_discovery_config(client: DataMasqueClient, discovery_config: DiscoveryConfig) -> None: create_response = { "id": CONFIG_ID_1, "name": "test_config", - "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "config_type": "database", "is_valid": "in_progress", @@ -283,7 +268,6 @@ def test_create_discovery_config(client: DataMasqueClient, discovery_config: Dis assert request_body["name"] == "test_config" assert request_body["config_yaml"] == "labels: []\nmetadata_rules: []\nidd_rules: []\n" assert request_body["config_type"] == "database" - assert request_body["project"] == PROJECT # Server-managed read-only fields must not be echoed back in the request body. assert "id" not in request_body assert "is_valid" not in request_body @@ -297,7 +281,6 @@ def test_update_discovery_config(client: DataMasqueClient, discovery_config: Dis update_response = { "id": CONFIG_ID_1, "name": "test_config", - "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "config_type": "database", "archived": False, @@ -343,7 +326,6 @@ def test_create_or_update_discovery_config_create(client: DataMasqueClient, disc create_response = { "id": CONFIG_ID_1, "name": "test_config", - "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "config_type": "database", "archived": False, @@ -358,9 +340,7 @@ def test_create_or_update_discovery_config_create(client: DataMasqueClient, disc assert result.id == DiscoveryConfigId(CONFIG_ID_1) assert m.request_history[0].method == "GET" - assert m.request_history[0].qs["project"] == [PROJECT] assert m.request_history[1].method == "POST" - assert m.request_history[1].json()["project"] == PROJECT def test_create_or_update_discovery_config_update(client: DataMasqueClient, discovery_config: DiscoveryConfig) -> None: @@ -372,7 +352,6 @@ def test_create_or_update_discovery_config_update(client: DataMasqueClient, disc { "id": CONFIG_ID_1, "name": "test_config", - "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-06-01T10:00:00Z", @@ -383,7 +362,6 @@ def test_create_or_update_discovery_config_update(client: DataMasqueClient, disc update_response = { "id": CONFIG_ID_1, "name": "test_config", - "project": PROJECT, "config_yaml": "labels: []\nmetadata_rules: []\nidd_rules: []\n", "config_type": "database", "archived": False, @@ -421,28 +399,20 @@ def test_delete_discovery_config_by_id_not_found(client: DataMasqueClient) -> No def test_delete_discovery_config_by_name(client: DataMasqueClient, sample_config_list_response: dict[str, Any]) -> None: with requests_mock.Mocker() as m: - # The server filters by name, so only the matching config is listed. - only_match = {**sample_config_list_response, "count": 1, "results": sample_config_list_response["results"][:1]} - m.get("http://test-server/api/discovery/configs/", json=only_match, status_code=200) + m.get("http://test-server/api/discovery/configs/", json=sample_config_list_response, status_code=200) m.delete(f"http://test-server/api/discovery/configs/{CONFIG_ID_1}/", status_code=204) - client.delete_discovery_config_by_name_if_exists("my_config", DiscoveryConfigType.database, PROJECT_ID) + client.delete_discovery_config_by_name_if_exists("my_config", DiscoveryConfigType.database) assert m.request_history[0].method == "GET" - assert m.request_history[0].qs == { - "name_exact": ["my_config"], - "config_type": ["database"], - "project": [str(PROJECT_ID)], - "limit": ["1"], - } assert m.request_history[1].method == "DELETE" -def test_delete_discovery_config_by_name_not_found(client: DataMasqueClient) -> None: - empty_response = {"count": 0, "next": None, "previous": None, "results": []} - +def test_delete_discovery_config_by_name_not_found( + client: DataMasqueClient, sample_config_list_response: dict[str, Any] +) -> None: with requests_mock.Mocker() as m: - m.get("http://test-server/api/discovery/configs/", json=empty_response, status_code=200) - client.delete_discovery_config_by_name_if_exists("nonexistent", DiscoveryConfigType.database, PROJECT_ID) + m.get("http://test-server/api/discovery/configs/", json=sample_config_list_response, status_code=200) + client.delete_discovery_config_by_name_if_exists("nonexistent", DiscoveryConfigType.database) assert m.call_count == 1 assert m.request_history[0].method == "GET" @@ -456,7 +426,6 @@ def test_delete_discovery_config_by_name_raises_when_server_omits_id(client: Dat "results": [ { "name": "my_config", - "project": PROJECT, "config_type": "database", "archived": False, "created": "2025-01-01T12:00:00Z", @@ -471,8 +440,29 @@ def test_delete_discovery_config_by_name_raises_when_server_omits_id(client: Dat json=list_response_without_id, status_code=200, ) - with pytest.raises(DataMasqueApiError, match="without an `id`"): - client.delete_discovery_config_by_name_if_exists("my_config", DiscoveryConfigType.database, PROJECT_ID) + with pytest.raises(DataMasqueException, match="without an `id`"): + client.delete_discovery_config_by_name_if_exists("my_config", DiscoveryConfigType.database) + + +def test_delete_discovery_config_by_name_only_deletes_matching_type(client: DataMasqueClient) -> None: + shared_name_response = { + "count": 2, + "next": None, + "previous": None, + "results": [ + {"id": CONFIG_ID_1, "name": "shared", "config_type": "database", "archived": False}, + {"id": CONFIG_ID_2, "name": "shared", "config_type": "file", "archived": False}, + ], + } + + with requests_mock.Mocker() as m: + m.get("http://test-server/api/discovery/configs/", json=shared_name_response, status_code=200) + m.delete(f"http://test-server/api/discovery/configs/{CONFIG_ID_2}/", status_code=204) + client.delete_discovery_config_by_name_if_exists("shared", DiscoveryConfigType.file) + + assert m.request_history[-1].method == "DELETE" + assert CONFIG_ID_2 in m.request_history[-1].url + assert CONFIG_ID_1 not in m.request_history[-1].url def test_get_default_discovery_config_yaml(client: DataMasqueClient) -> None: @@ -496,7 +486,6 @@ def test_discovery_config_parses_validation_fields() -> None: { "id": CONFIG_ID_1, "name": "my_config", - "project": PROJECT, "config_type": "database", "config_yaml": "labels: []", "is_valid": "invalid", @@ -516,7 +505,6 @@ def test_discovery_config_validation_fields_optional() -> None: { "id": CONFIG_ID_1, "name": "my_config", - "project": PROJECT, "config_type": "database", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -533,12 +521,12 @@ def test_unwrap_discovery_config_id_passes_through_strings() -> None: def test_unwrap_discovery_config_id_extracts_id_from_model() -> None: - config = DiscoveryConfig(name="x", project=PROJECT, config_type="database", id=DiscoveryConfigId(CONFIG_ID_1)) + config = DiscoveryConfig(name="x", config_type="database", id=DiscoveryConfigId(CONFIG_ID_1)) assert unwrap_discovery_config_id(config) == CONFIG_ID_1 def test_unwrap_discovery_config_id_raises_without_id() -> None: - config = DiscoveryConfig(name="x", project=PROJECT, config_type="database") + config = DiscoveryConfig(name="x", config_type="database") with pytest.raises(ValueError, match="id is None"): unwrap_discovery_config_id(config) @@ -547,7 +535,6 @@ def _build_config_response(is_valid: str, **extra: JsonValue) -> dict[str, JsonV return { "id": CONFIG_ID_1, "name": "test_config", - "project": PROJECT, "config_type": "database", "is_valid": is_valid, "validation_error": None, @@ -608,7 +595,7 @@ def test_discovery_config_rejects_malformed_error_entry() -> None: def test_create_discovery_config_with_empty_yaml_raises(client: DataMasqueClient) -> None: - config = DiscoveryConfig(name="test_config", project=PROJECT, yaml="", config_type="database") + config = DiscoveryConfig(name="test_config", yaml="", config_type="database") with requests_mock.Mocker() as m: with pytest.raises(ValueError, match="without YAML content"): @@ -618,7 +605,7 @@ def test_create_discovery_config_with_empty_yaml_raises(client: DataMasqueClient def test_update_discovery_config_with_empty_yaml_raises(client: DataMasqueClient) -> None: - config = DiscoveryConfig(id=CONFIG_ID_1, name="test_config", project=PROJECT, yaml="", config_type="database") + config = DiscoveryConfig(id=CONFIG_ID_1, name="test_config", yaml="", config_type="database") with requests_mock.Mocker() as m: with pytest.raises(ValueError, match="without YAML content"): diff --git a/tests/test_files.py b/tests/test_files.py index 37a2913..3b8f99b 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -10,8 +10,8 @@ import requests_mock from datamasque.client import DataMasqueClient -from datamasque.client.models.files import DataMasqueFile, OracleWalletFile, SeedFile, SnowflakeKeyFile, SslZipFile -from tests.helpers import OTHER_PROJECT, PROJECT, PROJECT_ID, fake, parse_multipart_form +from datamasque.client.models.files import OracleWalletFile, SeedFile, SnowflakeKeyFile, SslZipFile +from tests.helpers import fake, parse_multipart_form @pytest.mark.parametrize( @@ -49,7 +49,7 @@ def test_upload_file(client, file_type, source_factory): "modified_date": mock_return_modified_date.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), }, ) - file = client.upload_file(file_type, name_of_file, source, PROJECT_ID) + file = client.upload_file(file_type, name_of_file, source) if isinstance(source, (str, Path)): m_open.assert_called_once_with(source, "rb") @@ -141,7 +141,7 @@ def test_get_files_by_type_and_name(client, file_type): }, ], ) - result = client.get_file_of_type_by_name(file_type, "file2", PROJECT_ID) + result = client.get_file_of_type_by_name(file_type, "file2") assert isinstance(result, file_type) assert result.id == id_2 @@ -202,7 +202,7 @@ def test_upload_file_if_not_exists_skips_when_same_name_exists(client, file_type }, ], ) - result = client.upload_file_if_not_exists(file_type, "already_here.csv", PROJECT_ID) + result = client.upload_file_if_not_exists(file_type, "already_here.csv") assert result is None # Only the list-by-type GET should have fired — no POST. @@ -231,7 +231,7 @@ def test_upload_file_if_not_exists_uploads_when_missing(client, file_type): "modified_date": "2024-01-01T00:00:00.000000Z", }, ) - result = client.upload_file_if_not_exists(file_type, Path("new_file.csv"), PROJECT_ID) + result = client.upload_file_if_not_exists(file_type, Path("new_file.csv")) assert isinstance(result, file_type) assert result.id == new_id @@ -264,7 +264,7 @@ def test_upload_file_retries_on_401(config): }, ], ) - client.upload_file(SeedFile, "seed.csv", file_content, PROJECT_ID) + client.upload_file(SeedFile, "seed.csv", file_content) assert m_request.call_count == 2 first_form = parse_multipart_form(m_request.request_history[0]) @@ -286,7 +286,7 @@ def test_upload_ssl_zip_file_sends_mysql_database_type(client): "modified_date": "2024-01-01T00:00:00.000000Z", }, ) - client.upload_file(SslZipFile, "certs.zip", b"zip bytes", PROJECT_ID) + client.upload_file(SslZipFile, "certs.zip", b"zip bytes") form = parse_multipart_form(m_request.request_history[0]) assert form["database_type"] == "mysql" @@ -306,77 +306,7 @@ def test_upload_non_ssl_zip_file_omits_database_type(client, file_type): "modified_date": "2024-01-01T00:00:00.000000Z", }, ) - client.upload_file(file_type, "file.bin", b"content", PROJECT_ID) + client.upload_file(file_type, "file.bin", b"content") form = parse_multipart_form(m_request.request_history[0]) assert "database_type" not in form - - -@pytest.mark.parametrize("file_type", [SeedFile, OracleWalletFile, SslZipFile, SnowflakeKeyFile]) -def test_upload_file_sends_project(client: DataMasqueClient, file_type: type[DataMasqueFile]) -> None: - with requests_mock.Mocker() as m_request: - m_request.post( - f"http://test-server/{file_type.get_url()}", - status_code=201, - json={ - "id": str(uuid.uuid4()), - "name": "file.bin", - "project": PROJECT, - "created_date": "2024-01-01T00:00:00.000000Z", - "modified_date": "2024-01-01T00:00:00.000000Z", - }, - ) - file = client.upload_file(file_type, "file.bin", b"content", PROJECT_ID) - - form = parse_multipart_form(m_request.request_history[0]) - assert form["project"] == str(PROJECT_ID) - assert file.project == PROJECT - - -@pytest.mark.parametrize("file_type", [SeedFile, OracleWalletFile, SslZipFile, SnowflakeKeyFile]) -def test_list_files_of_type_filters_by_project(client: DataMasqueClient, file_type: type[DataMasqueFile]) -> None: - with requests_mock.Mocker() as m_request: - m_request.get(f"http://test-server/{file_type.get_url()}?project={PROJECT_ID}", json=[], complete_qs=True) - assert client.list_files_of_type(file_type, PROJECT_ID) == [] - - -@pytest.mark.parametrize("file_type", [SeedFile, OracleWalletFile, SslZipFile, SnowflakeKeyFile]) -def test_get_file_of_type_by_name_is_scoped_to_the_project( - client: DataMasqueClient, file_type: type[DataMasqueFile] -) -> None: - """A file with the same name in another project is not matched, because the listing is filtered by project.""" - other_project_file = { - "id": str(uuid.uuid4()), - "name": "seed.csv", - "project": "2-other", - "created_date": "2024-01-01T00:00:00.000000Z", - } - with requests_mock.Mocker() as m_request: - m_request.get(f"http://test-server/{file_type.get_url()}", json=[other_project_file]) - m_request.get(f"http://test-server/{file_type.get_url()}?project={PROJECT_ID}", json=[], complete_qs=True) - result = client.get_file_of_type_by_name(file_type, "seed.csv", PROJECT_ID) - - assert result is None - assert m_request.last_request.qs == {"project": [str(PROJECT_ID)]} - - -def test_upload_file_sends_the_id_of_a_project(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m_request: - m_request.post( - f"http://test-server/{SeedFile.get_url()}", - status_code=201, - json={ - "id": str(uuid.uuid4()), - "name": "seed.csv", - "project": "2-banking", - "created_date": "2024-01-01T00:00:00.000000Z", - }, - ) - client.upload_file( - SeedFile, - "seed.csv", - b"content", - OTHER_PROJECT, - ) - - assert parse_multipart_form(m_request.request_history[0])["project"] == "2" diff --git a/tests/test_git.py b/tests/test_git.py deleted file mode 100644 index 98a99d8..0000000 --- a/tests/test_git.py +++ /dev/null @@ -1,332 +0,0 @@ -"""Tests for Git push, pull and file listing with the DataMasque client.""" - -from collections.abc import Callable -from datetime import datetime, timezone - -import pytest -import requests_mock -from pydantic import ValidationError - -from datamasque.client import ( - DataMasqueApiError, - DataMasqueClient, - DiscoveryConfig, - DiscoveryConfigLibrary, - DiscoveryConfigType, - GitFile, - Ruleset, - RulesetLibrary, - RulesetType, -) -from datamasque.client.models.dm_instance import DataMasqueInstanceConfig - -BASE_URL = "http://test-server" -COMMIT_SHA = "0123456789abcdef0123456789abcdef01234567" -COMMIT = {"commit": COMMIT_SHA, "date": "2026-10-01T10:00:00+00:00", "message": "Add customers"} - -LIST_METHODS: list[tuple[Callable[[DataMasqueClient], list[GitFile]], str, str]] = [ - (lambda client: client.list_ruleset_git_files(), "/api/ruleset-git/files/", "rulesets/Banking/customers.yml"), - (lambda client: client.list_ruleset_library_git_files(), "/api/library-git/files/", "Banking/emails.yml"), - ( - lambda client: client.list_discovery_config_git_files(), - "/api/discovery-config-git/files/", - "Banking/customers.yml", - ), - ( - lambda client: client.list_discovery_config_library_git_files(), - "/api/discovery-config-library-git/files/", - "Banking/emails.yml", - ), -] - - -@pytest.mark.parametrize(("list_files", "endpoint", "project_file"), LIST_METHODS) -def test_list_git_files_returns_each_file_with_its_commits( - client: DataMasqueClient, - list_files: Callable[[DataMasqueClient], list[GitFile]], - endpoint: str, - project_file: str, -) -> None: - with requests_mock.Mocker(case_sensitive=True) as m: - m.get(f"{BASE_URL}{endpoint}", json={project_file: [COMMIT], "legacy.yml": []}) - - files = list_files(client) - - assert m.last_request.qs == {} - assert [file.path for file in files] == [project_file, "legacy.yml"] - assert files[0].commits[0].commit_sha == COMMIT_SHA - assert files[0].commits[0].date == datetime(2026, 10, 1, 10, tzinfo=timezone.utc) - assert files[0].commits[0].message == "Add customers" - assert files[1].commits == [] - - -def test_list_git_files_sends_the_branch(client: DataMasqueClient) -> None: - with requests_mock.Mocker(case_sensitive=True) as m: - m.get(f"{BASE_URL}/api/ruleset-git/files/", json={}) - - assert client.list_ruleset_git_files(branch="Feature/X") == [] - - assert m.last_request.qs == {"branch": ["Feature/X"]} - - -@pytest.mark.parametrize( - ("path", "folder", "name", "extension"), - [ - ("rulesets/Banking/Customers.yml", "rulesets/Banking", "Customers", ".yml"), - ("Customers.yaml", "", "Customers", ".yaml"), - ("rulesets/Banking/v1.customers.yml", "rulesets/Banking", "v1.customers", ".yml"), - ], -) -def test_pull_ruleset_from_git_splits_the_listed_path( - client: DataMasqueClient, path: str, folder: str, name: str, extension: str -) -> None: - with requests_mock.Mocker(case_sensitive=True) as m: - m.get(f"{BASE_URL}/api/ruleset-git/", json={"config_yaml": "version: '1.0'"}) - - content = client.pull_ruleset_from_git(path, COMMIT_SHA, branch="main") - - assert content == "version: '1.0'" - assert m.last_request.qs == { - "ruleset_name": [name], - "extension": [extension], - "folder": [folder], - "commit_id": [COMMIT_SHA], - "branch": ["main"], - } - - -@pytest.mark.parametrize( - ("pull", "endpoint", "name_field", "folder_param"), - [ - (DataMasqueClient.pull_ruleset_library_from_git, "/api/library-git/", "library_name", "namespace"), - (DataMasqueClient.pull_discovery_config_from_git, "/api/discovery-config-git/", "config_name", "folder"), - ( - DataMasqueClient.pull_discovery_config_library_from_git, - "/api/discovery-config-library-git/", - "library_name", - "namespace", - ), - ], -) -def test_pull_from_git_sends_each_type_s_fields( - client: DataMasqueClient, - pull: Callable[[DataMasqueClient, str, str], str], - endpoint: str, - name_field: str, - folder_param: str, -) -> None: - with requests_mock.Mocker(case_sensitive=True) as m: - m.get(f"{BASE_URL}{endpoint}", json={"config_yaml": "libraries: []"}) - - assert pull(client, "Bank/AU/Emails.yml", COMMIT_SHA) == "libraries: []" - - assert m.last_request.qs == { - name_field: ["Emails"], - "extension": [".yml"], - folder_param: ["Bank/AU"], - "commit_id": [COMMIT_SHA], - } - - -def test_push_ruleset_to_git_sends_the_ruleset(client: DataMasqueClient) -> None: - ruleset = Ruleset(name="customers", project="7-banking", yaml="version: '1.0'", ruleset_type=RulesetType.file) - ruleset.id = "a-ruleset-id" - with requests_mock.Mocker(case_sensitive=True) as m: - m.post(f"{BASE_URL}/api/ruleset-git/", status_code=204) - - client.push_ruleset_to_git(ruleset, "Update customers", branch="main", extension=".yaml") - - assert m.last_request.json() == { - "ruleset_name": "customers", - "mask_type": "file", - "id": "a-ruleset-id", - "project": "7-banking", - "ruleset_content": "version: '1.0'", - "commit_message": "Update customers", - "extension": ".yaml", - "branch": "main", - } - - -def test_push_and_push_path_leave_a_saved_resource_s_unset_project_to_the_server() -> None: - client = DataMasqueClient( - DataMasqueInstanceConfig(base_url=BASE_URL, username="u", password="p", default_project="1-default") - ) - library = RulesetLibrary(name="emails", yaml="libraries: []") - library.id = "a-library-id" - with requests_mock.Mocker(case_sensitive=True) as m: - m.post(f"{BASE_URL}/api/library-git/", status_code=204) - m.get(f"{BASE_URL}/api/library-git/push-path/", json={"path": "masking/libraries/Banking/emails"}) - - client.push_ruleset_library_to_git(library, "Add emails") - assert client.get_ruleset_library_git_push_path(library) == "masking/libraries/Banking/emails" - - push, push_path = m.request_history - assert push_path.qs == {"library_name": ["emails"], "id": ["a-library-id"]} - assert push.json() == { - "library_name": "emails", - "id": "a-library-id", - "library_content": "libraries: []", - "commit_message": "Add emails", - "extension": ".yml", - } - - -def test_push_and_push_path_send_the_default_project_for_an_unsaved_resource_with_no_project() -> None: - client = DataMasqueClient( - DataMasqueInstanceConfig(base_url=BASE_URL, username="u", password="p", default_project="1-default") - ) - library = DiscoveryConfigLibrary(name="emails", yaml="x: 1") - with requests_mock.Mocker(case_sensitive=True) as m: - m.post(f"{BASE_URL}/api/discovery-config-library-git/", status_code=204) - m.get( - f"{BASE_URL}/api/discovery-config-library-git/push-path/", - json={"path": "masking/discovery-config-libraries/Default/emails"}, - ) - - client.push_discovery_config_library_to_git(library, "Add emails") - client.get_discovery_config_library_git_push_path(library) - - push, push_path = m.request_history - assert push_path.qs == {"library_name": ["emails"], "project": ["1-default"]} - assert push.json() == { - "library_name": "emails", - "project": "1-default", - "library_content": "x: 1", - "commit_message": "Add emails", - "extension": ".yml", - } - - -def test_push_and_push_path_send_no_project_when_neither_the_resource_nor_the_client_sets_one( - client: DataMasqueClient, -) -> None: - ruleset = Ruleset(name="customers", yaml="version: '1.0'") - with requests_mock.Mocker(case_sensitive=True) as m: - m.post(f"{BASE_URL}/api/ruleset-git/", status_code=204) - m.get(f"{BASE_URL}/api/ruleset-git/push-path/", json={"path": "masking/rulesets/Banking/customers"}) - - client.push_ruleset_to_git(ruleset, "Add customers") - client.get_ruleset_git_push_path(ruleset) - - push, push_path = m.request_history - assert "project" not in push.json() - assert "project" not in push_path.qs - - -def test_push_discovery_config_to_git_sends_its_type(client: DataMasqueClient) -> None: - config = DiscoveryConfig(name="customers", project="7-banking", yaml="x: 1", config_type=DiscoveryConfigType.file) - with requests_mock.Mocker(case_sensitive=True) as m: - m.post(f"{BASE_URL}/api/discovery-config-git/", status_code=204) - - client.push_discovery_config_to_git(config, "Add customers") - - assert m.last_request.json() == { - "config_name": "customers", - "config_type": "file", - "project": "7-banking", - "config_content": "x: 1", - "commit_message": "Add customers", - "extension": ".yml", - } - - -@pytest.mark.parametrize( - "push", - [ - lambda client: client.push_ruleset_to_git(Ruleset(name="customers", project="7-banking"), "Add customers"), - lambda client: client.push_ruleset_library_to_git(RulesetLibrary(name="emails", project="7-banking"), "Add"), - lambda client: client.push_discovery_config_to_git( - DiscoveryConfig(name="customers", project="7-banking", config_type=DiscoveryConfigType.database), "Add" - ), - lambda client: client.push_discovery_config_library_to_git( - DiscoveryConfigLibrary(name="emails", project="7-banking", yaml=""), "Add" - ), - ], -) -def test_push_without_yaml_raises(client: DataMasqueClient, push: Callable[[DataMasqueClient], None]) -> None: - with requests_mock.Mocker(case_sensitive=True) as m: - with pytest.raises(ValueError, match="yaml is empty or unset"): - push(client) - - assert m.request_history == [] - - -def test_push_raises_on_a_path_conflict(client: DataMasqueClient) -> None: - with requests_mock.Mocker(case_sensitive=True) as m: - m.post(f"{BASE_URL}/api/ruleset-git/", status_code=409, json={"detail": "Cannot push"}) - - with pytest.raises(DataMasqueApiError) as error: - client.push_ruleset_to_git( - Ruleset(name="customers", project="7-banking", yaml="version: '1.0'"), "Add customers" - ) - - assert error.value.response.status_code == 409 - - -@pytest.mark.parametrize( - ("get_push_path", "endpoint", "expected_params"), - [ - ( - lambda client: client.get_ruleset_git_push_path(Ruleset(name="customers", project="7-banking")), - "/api/ruleset-git/push-path/", - {"ruleset_name": ["customers"], "mask_type": ["database"], "project": ["7-banking"]}, - ), - ( - lambda client: client.get_ruleset_library_git_push_path(RulesetLibrary(name="emails", project="7-banking")), - "/api/library-git/push-path/", - {"library_name": ["emails"], "project": ["7-banking"]}, - ), - ( - lambda client: client.get_discovery_config_git_push_path( - DiscoveryConfig(name="customers", project="7-banking", config_type=DiscoveryConfigType.database) - ), - "/api/discovery-config-git/push-path/", - {"config_name": ["customers"], "config_type": ["database"], "project": ["7-banking"]}, - ), - ( - lambda client: client.get_discovery_config_library_git_push_path( - DiscoveryConfigLibrary(name="emails", project="7-banking") - ), - "/api/discovery-config-library-git/push-path/", - {"library_name": ["emails"], "project": ["7-banking"]}, - ), - ], -) -def test_get_git_push_path_returns_the_server_s_path( - client: DataMasqueClient, - get_push_path: Callable[[DataMasqueClient], str], - endpoint: str, - expected_params: dict[str, list[str]], -) -> None: - with requests_mock.Mocker(case_sensitive=True) as m: - m.get(f"{BASE_URL}{endpoint}", json={"path": "masking/rulesets/Banking/customers"}) - - assert get_push_path(client) == "masking/rulesets/Banking/customers" - - assert m.last_request.qs == expected_params - - -@pytest.mark.parametrize( - ("call", "endpoint", "response"), - [ - ( - lambda client: client.pull_ruleset_from_git("rulesets/Banking/customers.yml", COMMIT_SHA), - "/api/ruleset-git/", - {"config_yaml": None}, - ), - ( - lambda client: client.get_ruleset_git_push_path(Ruleset(name="customers", project="7-banking")), - "/api/ruleset-git/push-path/", - {}, - ), - ], -) -def test_a_malformed_git_response_raises( - client: DataMasqueClient, call: Callable[[DataMasqueClient], str], endpoint: str, response: dict[str, object] -) -> None: - with requests_mock.Mocker(case_sensitive=True) as m: - m.get(f"{BASE_URL}{endpoint}", json=response) - - with pytest.raises(ValidationError): - call(client) diff --git a/tests/test_model_project.py b/tests/test_model_project.py index 17251d0..112b18a 100644 --- a/tests/test_model_project.py +++ b/tests/test_model_project.py @@ -13,7 +13,6 @@ from datamasque.client.models.ruleset_library import RulesetLibrary from datamasque.client.models.runs import RunInfo, UnfinishedRun from datamasque.client.models.table_reference import TableReference -from tests.helpers import PROJECT ResourceModel = Union[ ConnectionConfig, Ruleset, RulesetLibrary, DiscoveryConfig, DiscoveryConfigLibrary, RGConfig, TableReference @@ -53,29 +52,17 @@ @pytest.mark.parametrize(("parse", "payload"), _RESOURCE_CASES) -def test_resource_parses_project_and_sends_it_when_serializing(parse: ParseResource, payload: Payload) -> None: - model = parse({**payload, "project": PROJECT}) +def test_resource_parses_project_and_excludes_it_when_serializing(parse: ParseResource, payload: Payload) -> None: + model = parse({**payload, "project": "1-default"}) - assert model.project == PROJECT + assert model.project == "1-default" assert "project" not in (model.model_extra or {}) - assert model.model_dump(by_alias=True, mode="json")["project"] == PROJECT + assert "project" not in model.model_dump(by_alias=True, mode="json") @pytest.mark.parametrize(("parse", "payload"), _RESOURCE_CASES) -def test_resource_project_is_optional_and_not_sent_when_unset(parse: ParseResource, payload: Payload) -> None: - model = parse(payload) - - assert model.project is None - assert "project" not in model.model_dump(by_alias=True, exclude_none=True, mode="json") - - -@pytest.mark.parametrize("library_class", [RulesetLibrary, DiscoveryConfigLibrary]) -def test_library_drops_namespace(library_class: type[Union[RulesetLibrary, DiscoveryConfigLibrary]]) -> None: - """The server still returns the deprecated `namespace`, which would override `project` if it were sent back.""" - library = library_class.model_validate({"id": "l-1", "name": "l", "project": PROJECT, "namespace": "org"}) - - assert "namespace" not in (library.model_extra or {}) - assert "namespace" not in library.model_dump(by_alias=True, mode="json") +def test_resource_project_defaults_to_none_when_absent(parse: ParseResource, payload: Payload) -> None: + assert parse(payload).project is None _RUN_PAYLOAD: Payload = { @@ -90,7 +77,7 @@ def test_library_drops_namespace(library_class: type[Union[RulesetLibrary, Disco @pytest.mark.parametrize("run_class", [RunInfo, UnfinishedRun]) def test_run_parses_project(run_class: type[Union[RunInfo, UnfinishedRun]]) -> None: - run = run_class.model_validate({**_RUN_PAYLOAD, "project": PROJECT}) + run = run_class.model_validate({**_RUN_PAYLOAD, "project": "1-default"}) - assert run.project == PROJECT + assert run.project == "1-default" assert "project" not in (run.model_extra or {}) diff --git a/tests/test_projects.py b/tests/test_projects.py deleted file mode 100644 index 11c3a96..0000000 --- a/tests/test_projects.py +++ /dev/null @@ -1,538 +0,0 @@ -"""Tests for project support in the DataMasque client.""" - -from datetime import datetime -from typing import Callable, Optional - -import pytest -import requests_mock -from pydantic import BaseModel, ValidationError - -from datamasque.client import ( - DataMasqueApiError, - DataMasqueClient, - Project, - ProjectCreateRequest, - ProjectDeleteConflict, - ProjectDeleteConflictError, - ProjectDeleteConflictReason, - ProjectId, - ProjectIdentifier, - ProjectUpdateRequest, -) -from datamasque.client.exceptions import DataMasqueException -from datamasque.client.models.connection import ConnectionId, S3ConnectionConfig -from datamasque.client.models.discovery_config import DiscoveryConfig -from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary -from datamasque.client.models.project import unwrap_project_id -from datamasque.client.models.rg_config import RGConfig -from datamasque.client.models.ruleset import Ruleset, RulesetType -from datamasque.client.models.ruleset_library import RulesetLibrary -from datamasque.client.models.table_reference import TableReference -from tests.helpers import PROJECT, PROJECT_ID, active_projects_page - -CREATED_ID = "aaaaaaaa-1111-2222-3333-444444444444" - -BANKING_PROJECT = { - "id": 7, - "reference": "7-banking", - "name": "Banking", - "description": "Customer banking data", - "created": "2026-10-01T10:00:00Z", - "updated": "2026-10-01T10:00:00Z", -} - -LEGACY_PROJECT = { - "id": 8, - "reference": "8-legacy", - "name": "Legacy", - "description": "", - "created": "2026-09-01T10:00:00Z", - "updated": "2026-10-01T10:00:00Z", -} - -FILES_EXIST_MESSAGE = 'Cannot rename the project, because files already exist at "seeds/retail".' - -RENAMED_PROJECT = {**BANKING_PROJECT, "reference": "7-retail", "name": "Retail"} - -UPDATE_PROJECT_CASES = [ - pytest.param(ProjectId(7), ProjectUpdateRequest(name="Retail"), {"name": "Retail"}, id="rename-by-id"), - pytest.param("7-banking", ProjectUpdateRequest(name="Retail"), {"name": "Retail"}, id="rename-by-reference"), - pytest.param( - Project.model_validate(BANKING_PROJECT), - ProjectUpdateRequest(description=""), - {"description": ""}, - id="project-object-clears-description", - ), - pytest.param( - ProjectId(7), - ProjectUpdateRequest(name="Banking", description="Customer banking data"), - {"name": "Banking", "description": "Customer banking data"}, - id="name-and-description", - ), -] - - -CONFLICTS = [ - { - "reason": "name_in_use", - "detail": 'The destination project already contains a ruleset named "3407".', - "type": "rulesets", - "id": "0b6c2f0e-4a0d-4d8e-9a57-2f0f3c1d9e11", - "name": "3407", - }, - { - "reason": "file_exists", - "detail": 'The destination project already has a file at "projects/8-retail/seeds/names.csv".', - "type": "seeds", - "id": "5e3b8a40-7c2d-4f61-b1a9-0d4e6f2c8b77", - "name": "names.csv", - "path": "projects/8-retail/seeds/names.csv", - }, - { - "reason": "unexpected_file", - "detail": 'The project directory contains "projects/7-banking/notes.txt", which belongs to no resource.', - "path": "projects/7-banking/notes.txt", - }, -] - - -def test_create_request_rejects_unknown_fields() -> None: - with pytest.raises(ValidationError, match="archived"): - ProjectCreateRequest(name="Banking", archived=True) # type: ignore[call-arg] - - -def test_create_project(client: DataMasqueClient) -> None: - request = ProjectCreateRequest(name="Banking", description="Customer banking data") - - with requests_mock.Mocker() as m: - m.post("http://test-server/api/projects/", json=BANKING_PROJECT, status_code=201) - project = client.create_project(request) - - assert m.last_request.json() == {"name": "Banking", "description": "Customer banking data"} - assert project.id == ProjectId(7) - assert project.reference == "7-banking" - assert project.name == "Banking" - assert project.description == "Customer banking data" - assert project.created == datetime.fromisoformat("2026-10-01T10:00:00+00:00") - assert project.updated == datetime.fromisoformat("2026-10-01T10:00:00+00:00") - - -def test_create_project_returns_the_name_and_description_as_stored(client: DataMasqueClient) -> None: - """Whitespace in the name and description is sent as given, and the server's stored values are returned.""" - request = ProjectCreateRequest(name=" Banking ", description=" Customer banking data ") - - with requests_mock.Mocker() as m: - m.post("http://test-server/api/projects/", json=BANKING_PROJECT, status_code=201) - project = client.create_project(request) - - assert m.last_request.json() == {"name": " Banking ", "description": " Customer banking data "} - assert project.name == "Banking" - assert project.description == "Customer banking data" - - -def test_create_project_rejected_name_raises(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - m.post( - "http://test-server/api/projects/", - json={"name": ['Project name must not contain "/".']}, - status_code=400, - ) - with pytest.raises(DataMasqueApiError) as exc_info: - client.create_project(ProjectCreateRequest(name="bank/au")) - - assert exc_info.value.response.status_code == 400 - - -def test_get_project_delete_conflicts_parses_each_reason(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - m.get("http://test-server/api/projects/7/delete-check/", json={"conflicts": CONFLICTS}) - conflicts = client.get_project_delete_conflicts(ProjectId(7), ProjectId(8)) - - assert m.last_request.qs == {"destination": ["8"]} - assert conflicts == [ - ProjectDeleteConflict( - reason=ProjectDeleteConflictReason.name_in_use, - detail='The destination project already contains a ruleset named "3407".', - type="rulesets", - id="0b6c2f0e-4a0d-4d8e-9a57-2f0f3c1d9e11", - name="3407", - ), - ProjectDeleteConflict( - reason=ProjectDeleteConflictReason.file_exists, - detail='The destination project already has a file at "projects/8-retail/seeds/names.csv".', - type="seeds", - id="5e3b8a40-7c2d-4f61-b1a9-0d4e6f2c8b77", - name="names.csv", - path="projects/8-retail/seeds/names.csv", - ), - ProjectDeleteConflict( - reason=ProjectDeleteConflictReason.unexpected_file, - detail='The project directory contains "projects/7-banking/notes.txt", which belongs to no resource.', - path="projects/7-banking/notes.txt", - ), - ] - assert conflicts[0].path is None - assert conflicts[2].type is None - assert conflicts[2].id is None - assert conflicts[2].name is None - - -def test_get_project_delete_conflicts_returns_empty_list_when_project_can_be_deleted( - client: DataMasqueClient, -) -> None: - with requests_mock.Mocker() as m: - m.get("http://test-server/api/projects/7/delete-check/", json={"conflicts": []}) - conflicts = client.get_project_delete_conflicts(ProjectId(7), ProjectId(8)) - - assert conflicts == [] - - -def test_project_delete_calls_accept_projects_as_well_as_ids(client: DataMasqueClient) -> None: - project = Project.model_validate(BANKING_PROJECT) - destination = Project.model_validate(BANKING_PROJECT | {"id": 8, "reference": "8-retail", "name": "Retail"}) - - with requests_mock.Mocker() as m: - m.get("http://test-server/api/projects/7/delete-check/", json={"conflicts": []}) - m.delete("http://test-server/api/projects/7/", status_code=204) - client.get_project_delete_conflicts(project, destination) - client.delete_project(project, destination) - - assert [request.qs for request in m.request_history] == [{"destination": ["8"]}, {"destination": ["8"]}] - - -def test_delete_project(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - m.delete("http://test-server/api/projects/7/", status_code=204) - client.delete_project(ProjectId(7), ProjectId(8)) - - assert m.last_request.method == "DELETE" - assert m.last_request.qs == {"destination": ["8"]} - - -def test_delete_project_conflict_raises_with_conflicts(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - m.delete( - "http://test-server/api/projects/7/", - json={"detail": 'Project "Banking" cannot be deleted.', "conflicts": CONFLICTS}, - status_code=409, - ) - with pytest.raises(ProjectDeleteConflictError) as exc_info: - client.delete_project(ProjectId(7), ProjectId(8)) - - assert exc_info.value.response.status_code == 409 - assert str(exc_info.value) == 'Project "Banking" cannot be deleted.' - assert [conflict.reason for conflict in exc_info.value.conflicts] == [ - ProjectDeleteConflictReason.name_in_use, - ProjectDeleteConflictReason.file_exists, - ProjectDeleteConflictReason.unexpected_file, - ] - assert exc_info.value.conflicts[0].name == "3407" - - -@pytest.mark.parametrize("status_code", [400, 404]) -def test_delete_project_raises_a_plain_api_error_for_a_status_other_than_409( - client: DataMasqueClient, status_code: int -) -> None: - with requests_mock.Mocker() as m: - m.delete("http://test-server/api/projects/7/", json={"detail": "Not found."}, status_code=status_code) - with pytest.raises(DataMasqueApiError) as exc_info: - client.delete_project(ProjectId(7), ProjectId(8)) - - assert not isinstance(exc_info.value, ProjectDeleteConflictError) - assert exc_info.value.response.status_code == status_code - - -def test_list_projects_follows_pages(client: DataMasqueClient) -> None: - """With more active projects than fit on a page of 100, the client follows `next` to the last page.""" - projects = [ - { - **BANKING_PROJECT, - "id": project_id, - "reference": f"{project_id}-project-{project_id}", - "name": f"Project {project_id}", - } - for project_id in range(1, 102) - ] - with requests_mock.Mocker() as m: - m.get( - "http://test-server/api/projects/?limit=100&offset=0", - json={ - "count": 101, - "next": "http://test-server/api/projects/?limit=100&offset=100", - "results": projects[:100], - }, - complete_qs=True, - ) - m.get( - "http://test-server/api/projects/?limit=100&offset=100", - json={"count": 101, "next": None, "results": projects[100:]}, - complete_qs=True, - ) - result = client.list_projects() - - assert [project.id for project in result] == list(range(1, 102)) - - -def test_list_projects_include_archived(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - m.get( - "http://test-server/api/projects/?include_archived=true&limit=100&offset=0", - json={"count": 2, "next": None, "results": [BANKING_PROJECT, LEGACY_PROJECT]}, - complete_qs=True, - ) - projects = client.list_projects(include_archived=True) - - assert [project.id for project in projects] == [ProjectId(7), ProjectId(8)] - - -@pytest.mark.parametrize("project", [ProjectId(7), Project.model_validate(BANKING_PROJECT), "7-banking"]) -def test_get_project(client: DataMasqueClient, project: ProjectIdentifier) -> None: - with requests_mock.Mocker() as m: - m.get("http://test-server/api/projects/7/", json=BANKING_PROJECT, complete_qs=True) - result = client.get_project(project) - - assert result.id == ProjectId(7) - assert result.name == "Banking" - - -def test_get_missing_project_raises(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - m.get("http://test-server/api/projects/99/", json={"detail": "Not found."}, status_code=404) - with pytest.raises(DataMasqueApiError) as exc_info: - client.get_project(ProjectId(99)) - - assert exc_info.value.response.status_code == 404 - - -EMPTY_PAGE = {"count": 0, "next": None, "previous": None, "results": []} - -CREATE_OR_UPDATE_WITHOUT_A_PROJECT = [ - pytest.param( - lambda: S3ConnectionConfig(name="s3", bucket="b", base_directory=""), - "create_or_update_connection", - "/api/connections/", - [], - id="connection", - ), - pytest.param( - lambda: RGConfig(name="rg", yaml="labels: []"), - "create_or_update_rg_config", - "/api/ruleset-generation-configs/", - EMPTY_PAGE, - id="rg_config", - ), - pytest.param( - lambda: RulesetLibrary(name="lib", yaml="version: '1.0'"), - "create_or_update_ruleset_library", - "/api/ruleset-libraries/", - EMPTY_PAGE, - id="ruleset_library", - ), - pytest.param( - lambda: DiscoveryConfig(name="dc", yaml="labels: []", config_type="database"), - "create_or_update_discovery_config", - "/api/discovery/configs/", - EMPTY_PAGE, - id="discovery_config", - ), - pytest.param( - lambda: DiscoveryConfigLibrary(name="dcl", yaml="labels: []"), - "create_or_update_discovery_config_library", - "/api/discovery/config-libraries/", - [], - id="discovery_config_library", - ), - pytest.param( - lambda: TableReference(name="tr", connection=ConnectionId("1"), source="customers.csv"), - "create_or_update_table_reference", - "/api/table-references/", - [], - id="table_reference", - ), -] -""" -Each `create_or_update_*` that looks up the existing resource itself: -a factory for a resource with no `project`, the method, the path it looks up and creates on, -and that path's empty listing. -""" - -RULESET_WITHOUT_A_PROJECT = pytest.param( - lambda: Ruleset(name="rs", yaml="version: '1.0'\ntasks: []", ruleset_type=RulesetType.database), - "create_or_update_ruleset", - "/api/rulesets/", - None, - id="ruleset", -) -"""`create_or_update_ruleset`, which leaves finding the existing ruleset to the server's upsert.""" - - -@pytest.mark.parametrize("make_resource, method, path, empty_listing", CREATE_OR_UPDATE_WITHOUT_A_PROJECT) -def test_create_or_update_without_a_project_uses_the_only_active_project( - client: DataMasqueClient, - make_resource: Callable[[], BaseModel], - method: str, - path: str, - empty_listing: object, -) -> None: - """The lookup is in the only active project, and the create leaves the choice of project to the server.""" - resource = make_resource() - created = resource.model_dump(mode="json", by_alias=True) | {"id": CREATED_ID, "project": PROJECT} - - with requests_mock.Mocker() as m: - m.get("http://test-server/api/projects/", json=active_projects_page(1), status_code=200) - m.get(f"http://test-server{path}", json=empty_listing, status_code=200) - m.post(f"http://test-server{path}", json=created, status_code=201) - getattr(client, method)(resource) - - assert [(request.method, request.path) for request in m.request_history] == [ - ("GET", "/api/projects/"), - ("GET", path), - ("POST", path), - ] - assert m.request_history[0].qs == {"limit": ["1"]} - assert m.request_history[1].qs["project"] == [str(PROJECT_ID)] - assert "project" not in m.last_request.json() - assert resource.model_dump()["project"] == PROJECT - - -def test_create_or_update_ruleset_without_a_project_uses_the_only_active_project(client: DataMasqueClient) -> None: - """The upsert leaves the choice of project to the server, after checking that exactly one project is active.""" - ruleset = Ruleset(name="rs", yaml="version: '1.0'\ntasks: []", ruleset_type=RulesetType.database) - - with requests_mock.Mocker() as m: - m.get("http://test-server/api/projects/", json=active_projects_page(1), status_code=200) - m.post( - "http://test-server/api/rulesets/?upsert=true", - json={"id": CREATED_ID, "name": "rs", "project": PROJECT}, - status_code=201, - ) - client.create_or_update_ruleset(ruleset) - - assert [(request.method, request.path) for request in m.request_history] == [ - ("GET", "/api/projects/"), - ("POST", "/api/rulesets/"), - ] - assert "project" not in m.last_request.json() - assert ruleset.project == PROJECT - - -@pytest.mark.parametrize("active_projects", [0, 2]) -@pytest.mark.parametrize( - "make_resource, method, path, empty_listing", [*CREATE_OR_UPDATE_WITHOUT_A_PROJECT, RULESET_WITHOUT_A_PROJECT] -) -def test_create_or_update_without_a_project_needs_exactly_one_active_project( - client: DataMasqueClient, - make_resource: Callable[[], BaseModel], - method: str, - path: str, - empty_listing: object, - active_projects: int, -) -> None: - """ - With no project, the server only accepts a create when exactly one project is active. - - So the client raises before looking up or writing anything, - rather than updating a same-named resource in a project the server would not have picked. - """ - with requests_mock.Mocker() as m: - m.get("http://test-server/api/projects/", json=active_projects_page(active_projects), status_code=200) - with pytest.raises(DataMasqueException, match=f"has {active_projects} active projects"): - getattr(client, method)(make_resource()) - - assert [request.path for request in m.request_history] == ["/api/projects/"] - - -@pytest.mark.parametrize( - "project", - [ProjectId(7), Project.model_validate(BANKING_PROJECT), "7-banking", "7"], - ids=["id", "project", "reference", "id as a string"], -) -def test_unwrap_project_id_reads_the_id_from_each_form(project: ProjectIdentifier) -> None: - assert unwrap_project_id(project) == ProjectId(7) - - -@pytest.mark.parametrize("project", ["myproject", "", "-7"]) -def test_unwrap_project_id_rejects_a_string_that_does_not_start_with_an_id(project: str) -> None: - with pytest.raises(ValueError, match=f'"{project}" is not a project ID or reference'): - unwrap_project_id(project) - - -def test_update_request_rejects_unknown_fields() -> None: - with pytest.raises(ValidationError) as exc_info: - # mypy rejects the unknown field, which the test passes on purpose. - ProjectUpdateRequest(name="Banking", archived=True) # type: ignore[call-arg] - - assert [(error["loc"], error["msg"]) for error in exc_info.value.errors()] == [ - (("archived",), "Extra inputs are not permitted") - ] - - -@pytest.mark.parametrize(("project", "request_body", "expected_body"), UPDATE_PROJECT_CASES) -def test_update_project_sends_only_the_fields_set( - client: DataMasqueClient, - project: ProjectIdentifier, - request_body: ProjectUpdateRequest, - expected_body: dict[str, object], -) -> None: - with requests_mock.Mocker() as m: - m.patch("http://test-server/api/projects/7/", json=BANKING_PROJECT) - client.update_project(project, request_body) - - assert m.last_request.json() == expected_body - - -def test_update_project_returns_the_renamed_project(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - m.patch("http://test-server/api/projects/7/", json=RENAMED_PROJECT) - project = client.update_project(ProjectId(7), ProjectUpdateRequest(name="Retail")) - - assert project.name == "Retail" - assert project.reference == "7-retail" - - -def test_update_project_rejected_name_raises(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - m.patch( - "http://test-server/api/projects/7/", - json={"name": [FILES_EXIST_MESSAGE]}, - status_code=400, - ) - with pytest.raises(DataMasqueApiError) as exc_info: - client.update_project(ProjectId(7), ProjectUpdateRequest(name="Retail")) - - assert exc_info.value.response.status_code == 400 - assert exc_info.value.response.json() == {"name": [FILES_EXIST_MESSAGE]} - - -@pytest.mark.parametrize( - ("status_code", "detail"), - [ - (403, "You do not have permission to perform this action."), - (404, "Not found."), - ], -) -def test_update_project_refused_raises(client: DataMasqueClient, status_code: int, detail: str) -> None: - with requests_mock.Mocker() as m: - m.patch("http://test-server/api/projects/7/", json={"detail": detail}, status_code=status_code) - with pytest.raises(DataMasqueApiError) as exc_info: - client.update_project(ProjectId(7), ProjectUpdateRequest(name="Retail")) - - assert exc_info.value.response.status_code == status_code - - -@pytest.mark.parametrize( - ("body", "expected"), - [ - pytest.param({**BANKING_PROJECT, "has_git_synced_resources": True}, True, id="synced"), - pytest.param({**BANKING_PROJECT, "has_git_synced_resources": False}, False, id="not-synced"), - pytest.param(BANKING_PROJECT, None, id="flag-absent"), - ], -) -def test_get_project_reports_git_synced_resources( - client: DataMasqueClient, body: dict[str, object], expected: Optional[bool] -) -> None: - with requests_mock.Mocker() as m: - m.get("http://test-server/api/projects/7/", json=body) - project = client.get_project(ProjectId(7)) - - assert project.has_git_synced_resources is expected diff --git a/tests/test_resources.py b/tests/test_resources.py deleted file mode 100644 index f755e6a..0000000 --- a/tests/test_resources.py +++ /dev/null @@ -1,265 +0,0 @@ -"""Tests for listing and moving resources with the DataMasque client.""" - -from datetime import datetime -from typing import Any, Optional - -import pytest -import requests_mock - -from datamasque.client import ( - ConnectionResource, - DataMasqueApiError, - DataMasqueClient, - MaskType, - Project, - ProjectId, - Resource, - ResourceMoveConflictReason, - ResourceOrdering, - ResourceTypeName, - RulesetResource, -) - -RULESET = { - "id": "0b6c2f0e-4a0d-4d8e-9a57-2f0f3c1d9e11", - "name": "mask_customers", - "project": "7-banking", - "created": "2026-10-01T10:00:00Z", - "modified": "2026-10-02T10:00:00Z", - "mask_type": "database", -} -CONNECTION = { - "id": "5e3b8a40-7c2d-4f61-b1a9-0d4e6f2c8b77", - "name": "customers_db", - "project": "7-banking", - "created": None, - "modified": None, - "mask_type": "database", -} -BUILT_IN_SEED = { - "id": "9a1d3c55-2b7e-4c0a-8f16-3e5d7b9c1a24", - "name": "first_names.csv", - "project": None, - "created": "2026-01-01T00:00:00Z", - "modified": "2026-01-01T00:00:00Z", -} - - -def page(results: dict[str, list[dict]], next_url: Optional[str] = None) -> dict: - return { - "count": sum(len(rows) for rows in results.values()), - "next": next_url, - "previous": None, - "results": results, - } - - -def test_list_resources_groups_them_by_type_name(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - m.get( - "http://test-server/api/resources/", - json=page({"rulesets": [RULESET], "connections": [CONNECTION], "seeds": [BUILT_IN_SEED]}), - ) - resources = client.list_resources() - - assert sorted(resources) == ["connections", "rulesets", "seeds"] - (ruleset,) = resources["rulesets"] - assert isinstance(ruleset, RulesetResource) - assert ruleset.id == RULESET["id"] - assert ruleset.name == "mask_customers" - assert ruleset.project == "7-banking" - assert ruleset.created == datetime.fromisoformat("2026-10-01T10:00:00+00:00") - assert ruleset.mask_type is MaskType.database - (connection,) = resources["connections"] - assert isinstance(connection, ConnectionResource) - assert connection.created is None - (seed,) = resources["seeds"] - assert type(seed) is Resource - assert seed.project is None - assert seed.position is None - - -def test_list_resources_sends_its_filters(client: DataMasqueClient) -> None: - retail = Project.model_validate( - { - "id": 8, - "reference": "8-retail", - "name": "Retail", - "created": "2026-10-01T10:00:00Z", - "updated": "2026-10-01T10:00:00Z", - } - ) - with requests_mock.Mocker() as m: - m.get("http://test-server/api/resources/", json=page({"rulesets": [], "connections": []})) - client.list_resources( - projects=[ProjectId(7), retail], - types=[ResourceTypeName.rulesets, ResourceTypeName.connections], - search="customers", - ordering=ResourceOrdering.modified, - ) - - assert m.last_request.qs == { - "project": ["7", "8"], - "type": ["rulesets", "connections"], - "search": ["customers"], - "ordering": ["modified"], - "limit": ["500"], - "offset": ["0"], - } - - -@pytest.mark.parametrize( - ("kwargs", "expected"), - [ - ({}, "type"), - ({"descending": True}, "-type"), - ({"ordering": ResourceOrdering.project}, "project"), - ({"ordering": ResourceOrdering.project, "descending": True}, "-project"), - ], -) -def test_list_resources_sends_its_ordering(client: DataMasqueClient, kwargs: dict[str, Any], expected: str) -> None: - with requests_mock.Mocker() as m: - m.get("http://test-server/api/resources/", json=page({})) - client.list_resources(**kwargs) - - assert m.last_request.qs["ordering"] == [expected] - - -def test_list_resources_keeps_each_resources_position(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - m.get( - "http://test-server/api/resources/", - json=page({"connections": [CONNECTION | {"position": 1}], "rulesets": [RULESET | {"position": 0}]}), - ) - resources = client.list_resources(ordering=ResourceOrdering.modified, descending=True) - - in_order = sorted((resource for group in resources.values() for resource in group), key=lambda r: r.position) - assert [resource.id for resource in in_order] == [RULESET["id"], CONNECTION["id"]] - - -@pytest.mark.parametrize( - ("projects", "types", "expected"), - [ - ([], None, {type_name.value: [] for type_name in ResourceTypeName}), - ([], [ResourceTypeName.rulesets], {"rulesets": []}), - ([ProjectId(7)], [], {}), - ], -) -def test_list_resources_with_an_empty_filter_returns_no_resources_without_a_request( - client: DataMasqueClient, - projects: list[ProjectId], - types: Optional[list[ResourceTypeName]], - expected: dict[str, list[Resource]], -) -> None: - with requests_mock.Mocker() as m: - resources = client.list_resources(projects=projects, types=types) - - assert resources == expected - assert m.call_count == 0 - - -def test_list_resources_follows_every_page_and_merges_the_groups(client: DataMasqueClient) -> None: - next_url = "http://test-server/api/resources/?limit=2&offset=2" - with requests_mock.Mocker() as m: - m.get( - "http://test-server/api/resources/", - [ - {"json": page({"rulesets": [RULESET], "connections": [CONNECTION]}, next_url=next_url)}, - {"json": page({"rulesets": [RULESET | {"id": "c3d1e5f7-0000-4000-8000-000000000001"}]})}, - ], - ) - resources = client.list_resources(page_size=2) - - assert [ruleset.id for ruleset in resources["rulesets"]] == [RULESET["id"], "c3d1e5f7-0000-4000-8000-000000000001"] - assert [connection.id for connection in resources["connections"]] == [CONNECTION["id"]] - assert m.call_count == 2 - assert m.request_history[1].qs == {"limit": ["2"], "offset": ["2"]} - - -def test_list_resources_raises_for_a_caller_who_cannot_list_them(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - m.get("http://test-server/api/resources/", status_code=403, json={"detail": "Forbidden"}) - - with pytest.raises(DataMasqueApiError) as raised: - client.list_resources() - - assert raised.value.response.status_code == 403 - - -# The move methods accept a listed `Resource` or a plain ID. -MOVING = { - ResourceTypeName.rulesets: [RulesetResource.model_validate(RULESET)], - ResourceTypeName.connections: [CONNECTION["id"]], -} -MOVE_REQUEST = { - "destination": "8", - "resources": [{"type": "rulesets", "id": RULESET["id"]}, {"type": "connections", "id": CONNECTION["id"]}], -} -NAME_CLASH = { - "reason": "name_in_use", - "detail": 'The destination project already contains a connection named "customers_db".', - "type": "connections", - "id": CONNECTION["id"], - "name": "customers_db", -} - - -def test_get_resource_move_conflicts_sends_the_resources_and_destination(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - m.post("http://test-server/api/resources/move-check/", json={"conflicts": [NAME_CLASH]}) - conflicts = client.get_resource_move_conflicts(MOVING, ProjectId(8)) - - assert m.last_request.json() == MOVE_REQUEST - assert len(conflicts) == 1 - assert conflicts[0].reason is ResourceMoveConflictReason.name_in_use - assert conflicts[0].type is ResourceTypeName.connections - assert conflicts[0].name == "customers_db" - - -def test_move_resources_returns_the_count_moved_and_the_conflicts(client: DataMasqueClient) -> None: - with requests_mock.Mocker() as m: - m.post("http://test-server/api/resources/move/", json={"moved": 1, "conflicts": [NAME_CLASH]}) - result = client.move_resources(MOVING, ProjectId(8)) - - assert m.last_request.json() == MOVE_REQUEST - assert result.moved == 1 - assert [conflict.id for conflict in result.conflicts] == [CONNECTION["id"]] - - -def test_move_resources_moves_what_list_resources_returned_to_a_project(client: DataMasqueClient) -> None: - retail = Project.model_validate( - { - "id": 8, - "reference": "8-retail", - "name": "Retail", - "created": "2026-10-01T10:00:00Z", - "updated": "2026-10-01T10:00:00Z", - } - ) - with requests_mock.Mocker() as m: - m.get("http://test-server/api/resources/", json=page({"rulesets": [RULESET], "connections": [CONNECTION]})) - m.post("http://test-server/api/resources/move/", json={"moved": 2, "conflicts": []}) - client.move_resources(client.list_resources(), retail) - - assert m.last_request.json() == MOVE_REQUEST - - -def test_move_resources_takes_a_listed_resources_project_as_the_destination(client: DataMasqueClient) -> None: - listed = RulesetResource.model_validate(RULESET) - with requests_mock.Mocker() as m: - m.post("http://test-server/api/resources/move/", json={"moved": 2, "conflicts": []}) - client.move_resources(MOVING, listed.project) - - assert m.last_request.json()["destination"] == "7" - - -@pytest.mark.parametrize("resources", [{}, {ResourceTypeName.rulesets: []}]) -def test_moving_no_resources_raises_without_a_request(client: DataMasqueClient, resources: dict) -> None: - with requests_mock.Mocker() as m: - with pytest.raises(ValueError, match="at least one resource"): - client.move_resources(resources, ProjectId(8)) - - with pytest.raises(ValueError, match="at least one resource"): - client.get_resource_move_conflicts(resources, ProjectId(8)) - - assert m.call_count == 0 diff --git a/tests/test_rg_configs.py b/tests/test_rg_configs.py index 82381b0..d393b84 100644 --- a/tests/test_rg_configs.py +++ b/tests/test_rg_configs.py @@ -14,7 +14,6 @@ unwrap_rg_config_id, ) from datamasque.client.models.status import ValidationErrorType, ValidationStatus -from tests.helpers import PROJECT, PROJECT_ID CONFIG_ID_1 = "aaaaaaaa-1111-2222-3333-444444444444" CONFIG_ID_2 = "bbbbbbbb-1111-2222-3333-444444444444" @@ -32,7 +31,6 @@ def sample_config_list_response() -> dict[str, Any]: { "id": CONFIG_ID_1, "name": "my_config", - "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -40,7 +38,6 @@ def sample_config_list_response() -> dict[str, Any]: { "id": CONFIG_ID_2, "name": "another_config", - "project": PROJECT, "archived": False, "created": "2025-02-01T12:00:00Z", "modified": "2025-02-02T12:00:00Z", @@ -54,7 +51,6 @@ def sample_config_detail_response() -> dict[str, Any]: return { "id": CONFIG_ID_1, "name": "my_config", - "project": PROJECT, "config_yaml": CONFIG_YAML, "archived": False, "created": "2025-01-01T12:00:00Z", @@ -64,7 +60,7 @@ def sample_config_detail_response() -> dict[str, Any]: @pytest.fixture def rg_config() -> RGConfig: - return RGConfig(name="test_config", project=PROJECT, yaml=CONFIG_YAML) + return RGConfig(name="test_config", yaml=CONFIG_YAML) def test_list_rg_configs(client: DataMasqueClient, sample_config_list_response: dict[str, Any]) -> None: @@ -92,7 +88,6 @@ def test_list_rg_configs_pagination(client: DataMasqueClient) -> None: { "id": CONFIG_ID_1, "name": "c1", - "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -100,7 +95,6 @@ def test_list_rg_configs_pagination(client: DataMasqueClient) -> None: { "id": CONFIG_ID_2, "name": "c2", - "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -115,7 +109,6 @@ def test_list_rg_configs_pagination(client: DataMasqueClient) -> None: { "id": "cccccccc-1111-2222-3333-444444444444", "name": "c3", - "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -166,7 +159,6 @@ def test_get_rg_config_by_name_found(client: DataMasqueClient, sample_config_det { "id": CONFIG_ID_1, "name": "my_config", - "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -185,11 +177,11 @@ def test_get_rg_config_by_name_found(client: DataMasqueClient, sample_config_det json=sample_config_detail_response, status_code=200, ) - config = client.get_rg_config_by_name("my_config", PROJECT_ID) + config = client.get_rg_config_by_name("my_config") assert config is not None assert config.name == "my_config" - assert m.request_history[0].qs == {"name_exact": ["my_config"], "project": [str(PROJECT_ID)], "limit": ["1"]} + assert "name_exact=my_config" in m.request_history[0].url def test_get_rg_config_by_name_not_found(client: DataMasqueClient) -> None: @@ -201,7 +193,7 @@ def test_get_rg_config_by_name_not_found(client: DataMasqueClient) -> None: json=empty_response, status_code=200, ) - config = client.get_rg_config_by_name("nonexistent", PROJECT_ID) + config = client.get_rg_config_by_name("nonexistent") assert config is None @@ -214,7 +206,6 @@ def test_get_rg_config_by_name_raises_when_server_omits_id(client: DataMasqueCli "results": [ { "name": "my_config", - "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -229,14 +220,13 @@ def test_get_rg_config_by_name_raises_when_server_omits_id(client: DataMasqueCli status_code=200, ) with pytest.raises(DataMasqueApiError, match="without an `id`"): - client.get_rg_config_by_name("my_config", PROJECT_ID) + client.get_rg_config_by_name("my_config") def test_create_rg_config(client: DataMasqueClient, rg_config: RGConfig) -> None: create_response = { "id": CONFIG_ID_1, "name": "test_config", - "project": PROJECT, "config_yaml": CONFIG_YAML, "is_valid": "valid", "archived": False, @@ -260,7 +250,6 @@ def test_create_rg_config(client: DataMasqueClient, rg_config: RGConfig) -> None request_body = m.last_request.json() assert request_body["name"] == "test_config" - assert request_body["project"] == PROJECT assert request_body["config_yaml"] == CONFIG_YAML # Server-managed read-only fields must not be echoed back in the request body. assert "id" not in request_body @@ -275,7 +264,6 @@ def test_create_and_update_rg_config_populate_validation_errors(client: DataMasq invalid_response = { "id": CONFIG_ID_1, "name": "test_config", - "project": PROJECT, "config_yaml": CONFIG_YAML, "is_valid": "invalid", "validation_errors": [ @@ -324,7 +312,6 @@ def test_update_rg_config(client: DataMasqueClient, rg_config: RGConfig) -> None update_response = { "id": CONFIG_ID_1, "name": "test_config", - "project": PROJECT, "config_yaml": CONFIG_YAML, "archived": False, "created": "2025-06-01T10:00:00Z", @@ -357,7 +344,7 @@ def test_update_rg_config_no_id_raises(client: DataMasqueClient, rg_config: RGCo def test_update_rg_config_no_yaml_raises(client: DataMasqueClient) -> None: """A config from a list response carries no YAML, so a PUT would silently blank the server's copy.""" - listed = RGConfig(name="test_config", project=PROJECT) + listed = RGConfig(name="test_config") listed.id = RGConfigId(CONFIG_ID_1) with pytest.raises(ValueError, match="yaml is None"): @@ -369,7 +356,6 @@ def test_create_or_update_rg_config_create(client: DataMasqueClient, rg_config: create_response = { "id": CONFIG_ID_1, "name": "test_config", - "project": PROJECT, "config_yaml": CONFIG_YAML, "archived": False, "created": "2025-06-01T10:00:00Z", @@ -383,9 +369,7 @@ def test_create_or_update_rg_config_create(client: DataMasqueClient, rg_config: assert result.id == RGConfigId(CONFIG_ID_1) assert m.request_history[0].method == "GET" - assert m.request_history[0].qs["project"] == [PROJECT] assert m.request_history[1].method == "POST" - assert m.request_history[1].json()["project"] == PROJECT def test_create_or_update_rg_config_update(client: DataMasqueClient, rg_config: RGConfig) -> None: @@ -397,7 +381,6 @@ def test_create_or_update_rg_config_update(client: DataMasqueClient, rg_config: { "id": CONFIG_ID_1, "name": "test_config", - "project": PROJECT, "archived": False, "created": "2025-06-01T10:00:00Z", "modified": "2025-06-01T10:00:00Z", @@ -407,7 +390,6 @@ def test_create_or_update_rg_config_update(client: DataMasqueClient, rg_config: update_response = { "id": CONFIG_ID_1, "name": "test_config", - "project": PROJECT, "config_yaml": CONFIG_YAML, "archived": False, "created": "2025-06-01T10:00:00Z", @@ -451,7 +433,6 @@ def test_delete_rg_config_by_name(client: DataMasqueClient) -> None: { "id": CONFIG_ID_1, "name": "my_config", - "project": PROJECT, "archived": False, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -462,10 +443,10 @@ def test_delete_rg_config_by_name(client: DataMasqueClient) -> None: with requests_mock.Mocker() as m: m.get("http://test-server/api/ruleset-generation-configs/", json=list_response, status_code=200) m.delete(f"http://test-server/api/ruleset-generation-configs/{CONFIG_ID_1}/", status_code=204) - client.delete_rg_config_by_name_if_exists("my_config", PROJECT_ID) + client.delete_rg_config_by_name_if_exists("my_config") assert [r.method for r in m.request_history] == ["GET", "DELETE"] - assert m.request_history[0].qs == {"name_exact": ["my_config"], "project": [str(PROJECT_ID)], "limit": ["1"]} + assert "name_exact=my_config" in m.request_history[0].url def test_delete_rg_config_by_name_not_found(client: DataMasqueClient) -> None: @@ -473,7 +454,7 @@ def test_delete_rg_config_by_name_not_found(client: DataMasqueClient) -> None: with requests_mock.Mocker() as m: m.get("http://test-server/api/ruleset-generation-configs/", json=empty_response, status_code=200) - client.delete_rg_config_by_name_if_exists("nonexistent", PROJECT_ID) + client.delete_rg_config_by_name_if_exists("nonexistent") assert m.call_count == 1 assert m.request_history[0].method == "GET" @@ -498,7 +479,6 @@ def test_rg_config_parses_validation_fields() -> None: { "id": CONFIG_ID_1, "name": "my_config", - "project": PROJECT, "config_yaml": "labels: []", "is_valid": "invalid", "validation_errors": [{"message": "bad shape on line 3", "line_number": 3, "column_number": 1}], @@ -517,7 +497,6 @@ def test_rg_config_validation_fields_optional() -> None: { "id": CONFIG_ID_1, "name": "my_config", - "project": PROJECT, "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", } @@ -533,11 +512,11 @@ def test_unwrap_rg_config_id_passes_through_strings() -> None: def test_unwrap_rg_config_id_extracts_id_from_model() -> None: - config = RGConfig(name="x", project=PROJECT, id=RGConfigId(CONFIG_ID_1)) + config = RGConfig(name="x", id=RGConfigId(CONFIG_ID_1)) assert unwrap_rg_config_id(config) == CONFIG_ID_1 def test_unwrap_rg_config_id_raises_without_id() -> None: - config = RGConfig(name="x", project=PROJECT) + config = RGConfig(name="x") with pytest.raises(ValueError, match="id is None"): unwrap_rg_config_id(config) diff --git a/tests/test_ruleset_library.py b/tests/test_ruleset_library.py index 29b558a..b3fec81 100644 --- a/tests/test_ruleset_library.py +++ b/tests/test_ruleset_library.py @@ -15,7 +15,6 @@ ValidationStatus, ) from datamasque.client.models.status import ValidationErrorDetails -from tests.helpers import PROJECT, PROJECT_ID LIBRARY_ID_1 = "aaaaaaaa-1111-2222-3333-444444444444" LIBRARY_ID_2 = "bbbbbbbb-1111-2222-3333-444444444444" @@ -32,7 +31,7 @@ def sample_library_list_response(): { "id": LIBRARY_ID_1, "name": "my_library", - "project": PROJECT, + "namespace": "org", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -40,7 +39,7 @@ def sample_library_list_response(): { "id": LIBRARY_ID_2, "name": "another_library", - "project": PROJECT, + "namespace": "", "is_valid": "invalid", "created": "2025-02-01T12:00:00Z", "modified": "2025-02-02T12:00:00Z", @@ -55,7 +54,7 @@ def sample_library_detail_response(): return { "id": LIBRARY_ID_1, "name": "my_library", - "project": PROJECT, + "namespace": "org", "config_yaml": "version: '1.0'\nfunctions:\n - name: my_func", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", @@ -67,7 +66,7 @@ def sample_library_detail_response(): def ruleset_library(): return RulesetLibrary( name="test_library", - project=PROJECT, + namespace="test_ns", yaml="version: '1.0'\nfunctions: []", ) @@ -84,7 +83,7 @@ def test_list_ruleset_libraries(client: DataMasqueClient, sample_library_list_re assert len(libraries) == 2 assert libraries[0].id == RulesetLibraryId(LIBRARY_ID_1) assert libraries[0].name == "my_library" - assert libraries[0].project == PROJECT + assert libraries[0].namespace == "org" assert libraries[0].yaml is None assert libraries[0].is_valid is ValidationStatus.valid assert libraries[1].id == RulesetLibraryId(LIBRARY_ID_2) @@ -101,7 +100,7 @@ def test_list_ruleset_libraries_pagination(client: DataMasqueClient) -> None: { "id": LIBRARY_ID_1, "name": "lib1", - "project": PROJECT, + "namespace": "", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -109,7 +108,7 @@ def test_list_ruleset_libraries_pagination(client: DataMasqueClient) -> None: { "id": LIBRARY_ID_2, "name": "lib2", - "project": PROJECT, + "namespace": "", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -124,7 +123,7 @@ def test_list_ruleset_libraries_pagination(client: DataMasqueClient) -> None: { "id": "cccccccc-1111-2222-3333-444444444444", "name": "lib3", - "project": PROJECT, + "namespace": "", "is_valid": "unknown", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-01T12:00:00Z", @@ -170,7 +169,7 @@ def test_get_ruleset_library(client: DataMasqueClient, sample_library_detail_res assert library.id == RulesetLibraryId(LIBRARY_ID_1) assert library.name == "my_library" - assert library.project == PROJECT + assert library.namespace == "org" assert library.yaml == "version: '1.0'\nfunctions:\n - name: my_func" assert library.is_valid is ValidationStatus.valid @@ -186,7 +185,7 @@ def test_get_ruleset_library_by_name_found( { "id": LIBRARY_ID_1, "name": "my_library", - "project": PROJECT, + "namespace": "org", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -205,12 +204,13 @@ def test_get_ruleset_library_by_name_found( json=sample_library_detail_response, status_code=200, ) - library = client.get_ruleset_library_by_name("my_library", PROJECT_ID) + library = client.get_ruleset_library_by_name("my_library", "org") assert library is not None assert library.name == "my_library" assert library.yaml == "version: '1.0'\nfunctions:\n - name: my_func" - assert m.request_history[0].qs == {"name_exact": ["my_library"], "project": [str(PROJECT_ID)], "limit": ["1"]} + assert "name_exact=my_library" in m.request_history[0].url + assert "namespace_exact=org" in m.request_history[0].url def test_get_ruleset_library_by_name_raises_when_server_omits_id(client: DataMasqueClient) -> None: @@ -222,7 +222,7 @@ def test_get_ruleset_library_by_name_raises_when_server_omits_id(client: DataMas "results": [ { "name": "my_library", - "project": PROJECT, + "namespace": "org", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", "modified": "2025-01-02T12:00:00Z", @@ -237,7 +237,7 @@ def test_get_ruleset_library_by_name_raises_when_server_omits_id(client: DataMas status_code=200, ) with pytest.raises(DataMasqueApiError, match="without an `id`"): - client.get_ruleset_library_by_name("my_library", PROJECT_ID) + client.get_ruleset_library_by_name("my_library", "org") def test_get_ruleset_library_by_name_not_found(client: DataMasqueClient) -> None: @@ -249,7 +249,7 @@ def test_get_ruleset_library_by_name_not_found(client: DataMasqueClient) -> None json=empty_response, status_code=200, ) - library = client.get_ruleset_library_by_name("nonexistent", PROJECT_ID) + library = client.get_ruleset_library_by_name("nonexistent") assert library is None @@ -258,7 +258,7 @@ def test_create_ruleset_library(client: DataMasqueClient, ruleset_library: Rules create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "unknown", "created": "2025-06-01T10:00:00Z", @@ -281,35 +281,17 @@ def test_create_ruleset_library(client: DataMasqueClient, ruleset_library: Rules request_body = m.last_request.json() assert request_body["name"] == "test_library" - assert request_body["project"] == PROJECT + assert request_body["namespace"] == "test_ns" assert request_body["config_yaml"] == "version: '1.0'\nfunctions: []" -def test_update_ruleset_library_does_not_send_namespace_back( - client: DataMasqueClient, sample_library_detail_response: dict[str, Any] -) -> None: - """A `namespace` the server returns is not sent back, because the server lets it override `project`.""" - with requests_mock.Mocker() as m: - m.get( - f"http://test-server/api/ruleset-libraries/{LIBRARY_ID_1}/", - json={**sample_library_detail_response, "namespace": "org"}, - ) - m.put(f"http://test-server/api/ruleset-libraries/{LIBRARY_ID_1}/", json=sample_library_detail_response) - library = client.get_ruleset_library(RulesetLibraryId(LIBRARY_ID_1)) - client.update_ruleset_library(library) - - request_body = m.last_request.json() - assert "namespace" not in request_body - assert request_body["project"] == PROJECT - - def test_update_ruleset_library(client: DataMasqueClient, ruleset_library: RulesetLibrary) -> None: ruleset_library.id = RulesetLibraryId(LIBRARY_ID_1) update_response = { "id": LIBRARY_ID_1, "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "unknown", "created": "2025-06-01T10:00:00Z", @@ -347,7 +329,7 @@ def test_create_or_update_ruleset_library_create( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "unknown", "created": "2025-06-01T10:00:00Z", @@ -386,17 +368,26 @@ def test_create_or_update_ruleset_library_update( { "id": LIBRARY_ID_1, "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "is_valid": "valid", "created": "2025-06-01T10:00:00Z", "modified": "2025-06-01T10:00:00Z", }, ], } + detail_response = { + "id": LIBRARY_ID_1, + "name": "test_library", + "namespace": "test_ns", + "config_yaml": "version: '1.0'\nfunctions: []", + "is_valid": "valid", + "created": "2025-06-01T10:00:00Z", + "modified": "2025-06-01T10:00:00Z", + } update_response = { "id": LIBRARY_ID_1, "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "unknown", "created": "2025-06-01T10:00:00Z", @@ -409,6 +400,11 @@ def test_create_or_update_ruleset_library_update( json=list_response, status_code=200, ) + m.get( + f"http://test-server/api/ruleset-libraries/{LIBRARY_ID_1}/", + json=detail_response, + status_code=200, + ) m.put( f"http://test-server/api/ruleset-libraries/{LIBRARY_ID_1}/", json=update_response, @@ -417,8 +413,10 @@ def test_create_or_update_ruleset_library_update( result = client.create_or_update_ruleset_library(ruleset_library) assert result.id == RulesetLibraryId(LIBRARY_ID_1) - assert [request.method for request in m.request_history] == ["GET", "PUT"] - assert m.request_history[0].qs == {"name_exact": ["test_library"], "project": [PROJECT], "limit": ["1"]} + # Should have called GET (list), GET (detail), then PUT (update) + assert m.request_history[0].method == "GET" + assert m.request_history[1].method == "GET" + assert m.request_history[2].method == "PUT" def test_delete_ruleset_library_by_id(client: DataMasqueClient) -> None: @@ -455,35 +453,31 @@ def test_delete_ruleset_library_by_id_force(client: DataMasqueClient) -> None: def test_delete_ruleset_library_by_name(client: DataMasqueClient, sample_library_list_response: dict[str, Any]) -> None: with requests_mock.Mocker() as m: - # The server filters by name, so only the matching library is listed. - only_match = { - **sample_library_list_response, - "count": 1, - "results": sample_library_list_response["results"][:1], - } m.get( "http://test-server/api/ruleset-libraries/", - json=only_match, + json=sample_library_list_response, status_code=200, ) m.delete( f"http://test-server/api/ruleset-libraries/{LIBRARY_ID_1}/", status_code=204, ) - client.delete_ruleset_library_by_name_if_exists("my_library", PROJECT_ID) + client.delete_ruleset_library_by_name_if_exists("my_library", "org") - assert m.request_history[0].qs["project"] == [str(PROJECT_ID)] - assert [request.method for request in m.request_history] == ["GET", "DELETE"] + assert m.request_history[0].method == "GET" + assert m.request_history[1].method == "DELETE" -def test_delete_ruleset_library_by_name_not_found(client: DataMasqueClient) -> None: +def test_delete_ruleset_library_by_name_not_found( + client: DataMasqueClient, sample_library_list_response: dict[str, Any] +) -> None: with requests_mock.Mocker() as m: m.get( "http://test-server/api/ruleset-libraries/", - json={"count": 0, "next": None, "previous": None, "results": []}, + json=sample_library_list_response, status_code=200, ) - client.delete_ruleset_library_by_name_if_exists("nonexistent", PROJECT_ID) + client.delete_ruleset_library_by_name_if_exists("nonexistent") # Only the list call should have been made, no DELETE assert m.call_count == 1 @@ -494,7 +488,7 @@ def test_validate_ruleset_library(client: DataMasqueClient) -> None: validate_response = { "id": LIBRARY_ID_1, "name": "my_library", - "project": PROJECT, + "namespace": "org", "config_yaml": "version: '1.0'", "is_valid": "unknown", "created": "2025-01-01T12:00:00Z", @@ -523,7 +517,6 @@ def test_list_rulesets_using_library(client: DataMasqueClient) -> None: { "id": "eeeeeeee-1111-2222-3333-444444444444", "name": "ruleset_a", - "project": PROJECT, "mask_type": "database", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", @@ -532,7 +525,6 @@ def test_list_rulesets_using_library(client: DataMasqueClient) -> None: { "id": "ffffffff-1111-2222-3333-444444444444", "name": "ruleset_b", - "project": PROJECT, "mask_type": "file", "is_valid": "unknown", "created": "2025-02-01T12:00:00Z", @@ -583,7 +575,6 @@ def test_list_rulesets_using_library_pagination(client: DataMasqueClient) -> Non { "id": "aaa", "name": "r1", - "project": PROJECT, "mask_type": "database", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", @@ -592,7 +583,6 @@ def test_list_rulesets_using_library_pagination(client: DataMasqueClient) -> Non { "id": "bbb", "name": "r2", - "project": PROJECT, "mask_type": "database", "is_valid": "valid", "created": "2025-01-01T12:00:00Z", @@ -608,7 +598,6 @@ def test_list_rulesets_using_library_pagination(client: DataMasqueClient) -> Non { "id": "ccc", "name": "r3", - "project": PROJECT, "mask_type": "file", "is_valid": "unknown", "created": "2025-01-01T12:00:00Z", @@ -637,7 +626,7 @@ def test_create_ruleset_library_populates_validation_errors( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "invalid", "validation_errors": [{"message": "Invalid function definition"}], @@ -665,7 +654,7 @@ def test_update_ruleset_library_populates_validation_errors( update_response = { "id": LIBRARY_ID_1, "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "invalid", "validation_errors": [{"message": "Invalid function definition"}], @@ -695,7 +684,7 @@ def test_create_ruleset_library_does_not_send_read_only_fields( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "invalid", "validation_errors": [{"message": "Invalid function definition"}], @@ -726,7 +715,7 @@ def test_create_ruleset_library_collapses_git_snapshot( create_response = { "id": LIBRARY_ID_1, "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "valid", "git_branch": "main", @@ -763,7 +752,7 @@ def test_create_and_update_ruleset_library_populate_unlicensed_feature_warnings( response = { "id": LIBRARY_ID_1, "name": "test_library", - "project": PROJECT, + "namespace": "test_ns", "config_yaml": "version: '1.0'\nfunctions: []", "is_valid": "valid", "unlicensed_feature_warnings": returned, diff --git a/tests/test_rulesets.py b/tests/test_rulesets.py index b0a989d..f304439 100644 --- a/tests/test_rulesets.py +++ b/tests/test_rulesets.py @@ -8,7 +8,6 @@ from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.ruleset import RulesetType from datamasque.client.models.status import ValidationErrorType, ValidationStatus -from tests.helpers import PROJECT, PROJECT_ID def test_list_rulesets(client, existing_rulesets_json): @@ -35,7 +34,7 @@ def test_create_or_update_ruleset_create(client, ruleset): # Test creating a new ruleset with upsert m.post( "http://test-server/api/rulesets/?upsert=true", - json={"id": "2", "name": "test_ruleset", "project": PROJECT, "is_valid": "in_progress"}, + json={"id": "2", "name": "test_ruleset", "is_valid": "in_progress"}, status_code=201, ) @@ -47,7 +46,6 @@ def test_create_or_update_ruleset_create(client, ruleset): sent = m.last_request.json() assert sent["config_yaml"] == "version: '1.0'\ntasks: []" assert sent["mask_type"] == "database" - assert sent["project"] == PROJECT def test_create_or_update_ruleset_create_fail(client, ruleset): @@ -66,7 +64,7 @@ def test_create_or_update_ruleset_update(client, ruleset): # Test updating an existing ruleset with upsert (returns 200 status for update) m.post( "http://test-server/api/rulesets/?upsert=true", - json={"id": "1", "name": "test_ruleset", "project": PROJECT, "is_valid": "valid"}, + json={"id": "1", "name": "test_ruleset", "is_valid": "valid"}, status_code=200, ) @@ -96,31 +94,41 @@ def test_delete_ruleset_by_id(client): def test_delete_ruleset_by_name(client, existing_rulesets_json): - """The server filters by name, type and project, so a same-named ruleset elsewhere is never returned.""" with requests_mock.Mocker() as m: m.get( - f"http://test-server/api/v2/rulesets/?name_exact=file_masking_ruleset&mask_type=file&project={PROJECT_ID}", - json=[existing_rulesets_json[1]], + "http://test-server/api/v2/rulesets/", + json=existing_rulesets_json, status_code=200, - complete_qs=True, ) m.delete("http://test-server/api/rulesets/2/", status_code=204) - client.delete_ruleset_by_name_if_exists("file_masking_ruleset", RulesetType.file, PROJECT_ID) + client.delete_ruleset_by_name_if_exists("file_masking_ruleset", RulesetType.file) assert m.call_count == 2 assert m.request_history[0].method == "GET" assert m.request_history[1].method == "DELETE" -def test_delete_ruleset_that_does_not_exist(client): +def test_delete_ruleset_by_name_skips_other_type(client, existing_rulesets_json): + with requests_mock.Mocker() as m: + m.get( + "http://test-server/api/v2/rulesets/", + json=existing_rulesets_json, + status_code=200, + ) + client.delete_ruleset_by_name_if_exists("file_masking_ruleset", RulesetType.database) + + assert m.call_count == 1 + assert m.request_history[0].method == "GET" + + +def test_delete_ruleset_that_does_not_exist(client, existing_rulesets_json): with requests_mock.Mocker() as m: m.get( - f"http://test-server/api/v2/rulesets/?name_exact=not_a_ruleset&mask_type=database&project={PROJECT_ID}", - json=[], + "http://test-server/api/v2/rulesets/", + json=existing_rulesets_json, status_code=200, - complete_qs=True, ) - client.delete_ruleset_by_name_if_exists("not_a_ruleset", RulesetType.database, PROJECT_ID) + client.delete_ruleset_by_name_if_exists("not_a_ruleset", RulesetType.database) assert m.call_count == 1 assert m.request_history[0].method == "GET" @@ -134,7 +142,6 @@ def test_create_or_update_ruleset_populates_validation_errors(client, ruleset): json={ "id": "2", "name": "test_ruleset", - "project": PROJECT, "is_valid": "invalid", "validation_errors": [ { @@ -162,7 +169,7 @@ def test_create_or_update_ruleset_validation_errors_empty_when_valid(client, rul with requests_mock.Mocker() as m: m.post( "http://test-server/api/rulesets/?upsert=true", - json={"id": "2", "name": "test_ruleset", "project": PROJECT, "is_valid": "valid"}, + json={"id": "2", "name": "test_ruleset", "is_valid": "valid"}, status_code=201, ) result = client.create_or_update_ruleset(ruleset) @@ -188,13 +195,7 @@ def test_create_or_update_ruleset_populates_unlicensed_feature_warnings(client, with requests_mock.Mocker() as m: m.post( "http://test-server/api/rulesets/?upsert=true", - json={ - "id": "2", - "name": "test_ruleset", - "project": PROJECT, - "is_valid": "valid", - "unlicensed_feature_warnings": returned, - }, + json={"id": "2", "name": "test_ruleset", "is_valid": "valid", "unlicensed_feature_warnings": returned}, status_code=201, ) result = client.create_or_update_ruleset(ruleset) @@ -207,7 +208,7 @@ def test_create_or_update_ruleset_unlicensed_feature_warnings_default_to_none(cl with requests_mock.Mocker() as m: m.post( "http://test-server/api/rulesets/?upsert=true", - json={"id": "2", "name": "test_ruleset", "project": PROJECT, "is_valid": "in_progress"}, + json={"id": "2", "name": "test_ruleset", "is_valid": "in_progress"}, status_code=201, ) result = client.create_or_update_ruleset(ruleset) @@ -224,7 +225,6 @@ def test_list_rulesets_unlicensed_feature_warnings(client): { "id": "1", "name": "unstructured_ruleset", - "project": PROJECT, "mask_type": "database", "is_valid": "valid", "unlicensed_feature_warnings": [ @@ -234,7 +234,6 @@ def test_list_rulesets_unlicensed_feature_warnings(client): { "id": "2", "name": "licensed_ruleset", - "project": PROJECT, "mask_type": "database", "is_valid": "valid", "unlicensed_feature_warnings": [], @@ -242,7 +241,6 @@ def test_list_rulesets_unlicensed_feature_warnings(client): { "id": "3", "name": "unvalidated_ruleset", - "project": PROJECT, "mask_type": "database", "is_valid": "in_progress", "unlicensed_feature_warnings": None, @@ -267,7 +265,6 @@ def test_create_or_update_ruleset_does_not_send_read_only_fields(client, ruleset json={ "id": "2", "name": "test_ruleset", - "project": PROJECT, "is_valid": "invalid", "validation_errors": [{"message": "bad", "validation_error_type": "ruleset"}], "unlicensed_feature_warnings": [ @@ -286,7 +283,6 @@ def test_create_or_update_ruleset_does_not_send_read_only_fields(client, ruleset assert read_only_field not in body # Input fields are still present. assert body["config_yaml"] == "version: '1.0'\ntasks: []" - assert body["project"] == PROJECT def test_create_or_update_ruleset_collapses_git_snapshot(client, ruleset): @@ -297,7 +293,6 @@ def test_create_or_update_ruleset_collapses_git_snapshot(client, ruleset): json={ "id": "2", "name": "test_ruleset", - "project": PROJECT, "is_valid": "valid", "git_branch": "main", "git_commit_sha": "abc123", @@ -323,7 +318,6 @@ def test_create_or_update_ruleset_git_none_when_not_synced(client, ruleset): json={ "id": "2", "name": "test_ruleset", - "project": PROJECT, "is_valid": "valid", "git_branch": None, "git_commit_sha": None, @@ -346,7 +340,6 @@ def test_list_rulesets_collapses_git_snapshot(client): { "id": "1", "name": "synced_ruleset", - "project": PROJECT, "mask_type": "database", "config_yaml": "version: '1.0'", "is_valid": "valid", @@ -358,7 +351,6 @@ def test_list_rulesets_collapses_git_snapshot(client): { "id": "2", "name": "local_ruleset", - "project": PROJECT, "mask_type": "database", "config_yaml": "version: '1.0'", "is_valid": "valid", diff --git a/tests/test_runs.py b/tests/test_runs.py index 85f1aff..295409a 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -3,10 +3,8 @@ import pytest import requests_mock -from datamasque.client import DataMasqueClient from datamasque.client.exceptions import ( DataMasqueApiError, - DataMasqueException, FailedToStartError, InvalidLibraryError, InvalidRulesetError, @@ -23,7 +21,7 @@ UnfinishedRun, ) from datamasque.client.models.status import MaskingRunStatus -from tests.helpers import PROJECT, fake +from tests.helpers import fake def test_get_run_log(client): @@ -94,101 +92,78 @@ def test_unfinished_run_str_without_destination(): assert str(run) == '"source_db": Run ID 42 in status `queued`, ruleset "my_ruleset"' -QUEUED_SOURCE_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a01") -QUEUED_DESTINATION_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a02") -RUNNING_SOURCE_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a03") -RUNNING_DESTINATION_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a04") -VALIDATING_SOURCE_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a05") -VALIDATING_DESTINATION_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a06") -CANCELLING_SOURCE_ID = ConnectionId("0b6f6c1e-2d3a-4c55-9a1e-5f0b8d7e1a07") - -UNFINISHED_RUNS_URL = ( - "http://test-server/api/runs/?connection_ruleset_name=&ruleset_name=&run_status={}&limit=1&offset=0" -) - - -def unfinished_run_json( - run_id: int, status: str, source: tuple[str, str], destination: tuple[str, str] -) -> dict[str, object]: - """ - Builds an unfinished-run list entry from `(id, name)` pairs. - - An empty ID is sent as `None`, and an empty destination name means none. - """ - return { - "id": run_id, - "source_connection": source[0] or None, - "source_connection_name": source[1], - "destination_connection": destination[0] or None, - "destination_connection_name": destination[1], - "ruleset_name": f"ruleset_{run_id}", - "project": PROJECT, - "status": status, - } - - -def test_get_unfinished_runs(client: DataMasqueClient) -> None: - runs = { - "queued": unfinished_run_json( - 1, "queued", (QUEUED_SOURCE_ID, "queued_src"), (QUEUED_DESTINATION_ID, "queued_dst") - ), - "running": unfinished_run_json( - 2, "running", (RUNNING_SOURCE_ID, "running_src"), (RUNNING_DESTINATION_ID, "running_dst") - ), - "validating": unfinished_run_json( - 3, "validating", (VALIDATING_SOURCE_ID, "validating_src"), (VALIDATING_DESTINATION_ID, "validating_dst") - ), - "cancelling": unfinished_run_json(4, "cancelling", (CANCELLING_SOURCE_ID, "cancelling_src"), ("", "")), - } +def test_get_unfinished_runs(client): with requests_mock.Mocker() as m: - for status, run in runs.items(): - m.get(UNFINISHED_RUNS_URL.format(status), json={"results": [run]}, status_code=200) - - ur = client.get_unfinished_runs() - - # 3 statuses have both source and destination keys, cancelling has only source (empty destination) - assert {connection_id: run.id for connection_id, run in ur.items()} == { - QUEUED_SOURCE_ID: 1, - QUEUED_DESTINATION_ID: 1, - RUNNING_SOURCE_ID: 2, - RUNNING_DESTINATION_ID: 2, - VALIDATING_SOURCE_ID: 3, - VALIDATING_DESTINATION_ID: 3, - CANCELLING_SOURCE_ID: 4, - } - - -def test_get_unfinished_runs_keeps_same_named_connections_apart(client: DataMasqueClient) -> None: - """Two connections with the same name in different projects each keep their own run.""" - runs = { - "queued": unfinished_run_json(1, "queued", (QUEUED_SOURCE_ID, "orders_db"), ("", "")), - "running": unfinished_run_json(2, "running", (RUNNING_SOURCE_ID, "orders_db"), ("", "")), - } - with requests_mock.Mocker() as m: - for status in ("queued", "running", "validating", "cancelling"): - results = [runs[status]] if status in runs else [] - m.get(UNFINISHED_RUNS_URL.format(status), json={"results": results}, status_code=200) + m.get( + "http://test-server/api/runs/?connection_ruleset_name=&ruleset_name=&run_status=queued&limit=1&offset=0", + json={ + "results": [ + { + "source_connection_name": "queued_src", + "destination_connection_name": "queued_dst", + "id": 1, + "ruleset_name": "ruleset_1", + "project": "1-default", + "status": "queued", + } + ] + }, + status_code=200, + ) + m.get( + "http://test-server/api/runs/?connection_ruleset_name=&ruleset_name=&run_status=running&limit=1&offset=0", + json={ + "results": [ + { + "source_connection_name": "running_src", + "destination_connection_name": "running_dst", + "id": 2, + "ruleset_name": "ruleset_2", + "project": "1-default", + "status": "running", + } + ] + }, + status_code=200, + ) + m.get( + "http://test-server/api/runs/?connection_ruleset_name=&ruleset_name=&run_status=validating&limit=1&offset=0", + json={ + "results": [ + { + "source_connection_name": "validating_src", + "destination_connection_name": "validating_dst", + "id": 3, + "ruleset_name": "ruleset_3", + "project": "1-default", + "status": "validating", + } + ] + }, + status_code=200, + ) + m.get( + "http://test-server/api/runs/?connection_ruleset_name=&ruleset_name=&run_status=cancelling&limit=1&offset=0", + json={ + "results": [ + { + "source_connection_name": "cancelling_src", + "destination_connection_name": "", + "id": 4, + "ruleset_name": "ruleset_4", + "project": "1-default", + "status": "cancelling", + } + ] + }, + status_code=200, + ) ur = client.get_unfinished_runs() - - assert {connection_id: run.id for connection_id, run in ur.items()} == {QUEUED_SOURCE_ID: 1, RUNNING_SOURCE_ID: 2} - - -@pytest.mark.parametrize( - "source, destination, missing", - [ - (("", "orders_db"), ("", ""), "source"), - ((QUEUED_SOURCE_ID, "orders_db"), ("", "orders_copy"), "destination"), - ], -) -def test_get_unfinished_runs_without_connection_id_raises( - client: DataMasqueClient, source: tuple[str, str], destination: tuple[str, str], missing: str -) -> None: - run = unfinished_run_json(1, "queued", source, destination) - with requests_mock.Mocker() as m: - m.get(UNFINISHED_RUNS_URL.format("queued"), json={"results": [run]}, status_code=200) - with pytest.raises(DataMasqueException, match=f"without a {missing} connection ID"): - client.get_unfinished_runs() + # 3 statuses have both source and destination keys, cancelling has only source (empty destination) + assert len(ur) == 7 + for run in ur.values(): + assert isinstance(run, UnfinishedRun) def test_start_masking_run(client): @@ -385,7 +360,6 @@ def test_masking_run_request_requires_ruleset(): def test_masking_run_request_accepts_connection_config_and_ruleset_objects(): """Callers may pass full `ConnectionConfig` / `Ruleset` objects; their IDs are extracted at construction.""" connection = DatabaseConnectionConfig( - project=PROJECT, id=ConnectionId("conn-uuid"), name="prod_db", db_type=DatabaseType.postgres, @@ -395,7 +369,6 @@ def test_masking_run_request_accepts_connection_config_and_ruleset_objects(): user="masker", ) dest = DatabaseConnectionConfig( - project=PROJECT, id=ConnectionId("dest-uuid"), name="staging_db", db_type=DatabaseType.postgres, @@ -405,7 +378,6 @@ def test_masking_run_request_accepts_connection_config_and_ruleset_objects(): user="masker", ) ruleset = Ruleset( - project=PROJECT, id=RulesetId("rs-uuid"), name="my_ruleset", yaml="version: '1.0'", @@ -423,7 +395,6 @@ def test_masking_run_request_accepts_connection_config_and_ruleset_objects(): def test_masking_run_request_rejects_unpersisted_connection(): """A `ConnectionConfig` without an `id` means the caller hasn't created it yet — raise at construction.""" connection = DatabaseConnectionConfig( - project=PROJECT, name="not_yet_created", db_type=DatabaseType.postgres, host="localhost", @@ -437,7 +408,7 @@ def test_masking_run_request_rejects_unpersisted_connection(): def test_masking_run_request_rejects_unpersisted_ruleset(): """Same check on the ruleset side.""" - ruleset = Ruleset(project=PROJECT, name="fresh_ruleset", yaml="version: '1.0'", ruleset_type=RulesetType.database) + ruleset = Ruleset(name="fresh_ruleset", yaml="version: '1.0'", ruleset_type=RulesetType.database) with pytest.raises(ValueError, match="has not been created yet"): MaskingRunRequest(connection="conn-1", ruleset=ruleset) diff --git a/tests/test_table_references.py b/tests/test_table_references.py index a2af76d..1a7968f 100644 --- a/tests/test_table_references.py +++ b/tests/test_table_references.py @@ -15,7 +15,7 @@ TableReferenceOptions, ) from datamasque.client.exceptions import DataMasqueException -from tests.helpers import PROJECT, PROJECT_ID, file_connection_config +from tests.helpers import file_connection_config TABLE_REFERENCE_ID_1 = "aaaaaaaa-1111-2222-3333-444444444444" TABLE_REFERENCE_ID_2 = "bbbbbbbb-1111-2222-3333-444444444444" @@ -38,7 +38,6 @@ def sample_table_reference_list_response() -> list[dict[str, object]]: { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", - "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -48,7 +47,6 @@ def sample_table_reference_list_response() -> list[dict[str, object]]: { "id": TABLE_REFERENCE_ID_2, "name": "account_identities", - "project": PROJECT, "connection": CONNECTION_ID_2, "source": "OPS.ACCOUNTS", "options": { @@ -68,7 +66,6 @@ def sample_table_reference_list_response() -> list[dict[str, object]]: def table_reference() -> TableReference: return TableReference( name="customer_identities", - project=PROJECT, connection=ConnectionId(CONNECTION_ID_1), source="identities/customers.csv", ) @@ -76,7 +73,7 @@ def table_reference() -> TableReference: def test_name_connection_and_source_are_required_on_the_model() -> None: with pytest.raises(ValidationError, match="source"): - TableReference(name="customer_identities", project=PROJECT, connection=ConnectionId(CONNECTION_ID_1)) + TableReference(name="customer_identities", connection=ConnectionId(CONNECTION_ID_1)) def test_options_default_to_the_servers_defaults() -> None: @@ -93,9 +90,7 @@ def test_connection_accepts_a_connection_config() -> None: connection = file_connection_config() connection.id = ConnectionId(CONNECTION_ID_1) - reference = TableReference( - name="customer_identities", project=PROJECT, connection=connection, source="customers.csv" - ) + reference = TableReference(name="customer_identities", connection=connection, source="customers.csv") assert reference.connection == ConnectionId(CONNECTION_ID_1) assert reference.connection_id == ConnectionId(CONNECTION_ID_1) @@ -103,9 +98,7 @@ def test_connection_accepts_a_connection_config() -> None: def test_connection_config_without_an_id_is_rejected() -> None: with pytest.raises(ValidationError, match="id is None"): - TableReference( - name="customer_identities", project=PROJECT, connection=file_connection_config(), source="customers.csv" - ) + TableReference(name="customer_identities", connection=file_connection_config(), source="customers.csv") def test_list_table_references( @@ -137,21 +130,6 @@ def test_list_table_references( assert references[1].options.null_string == "NULL" -def test_list_table_references_in_a_project( - client: DataMasqueClient, sample_table_reference_list_response: list[dict[str, object]] -) -> None: - with requests_mock.Mocker() as m: - m.get( - "http://test-server/api/table-references/", - json=sample_table_reference_list_response, - status_code=200, - ) - references = client.list_table_references(PROJECT_ID) - - assert m.last_request.qs == {"project": [str(PROJECT_ID)]} - assert [reference.project for reference in references] == [PROJECT, PROJECT] - - def test_list_table_references_empty(client: DataMasqueClient) -> None: with requests_mock.Mocker() as m: m.get( @@ -190,12 +168,11 @@ def test_get_table_reference_by_name_found( json=sample_table_reference_list_response, status_code=200, ) - reference = client.get_table_reference_by_name("account_identities", PROJECT_ID) + reference = client.get_table_reference_by_name("account_identities") assert reference is not None assert reference.id == TableReferenceId(TABLE_REFERENCE_ID_2) assert m.call_count == 1 - assert m.last_request.qs == {"project": [str(PROJECT_ID)]} def test_get_table_reference_by_name_not_found( @@ -207,7 +184,7 @@ def test_get_table_reference_by_name_not_found( json=sample_table_reference_list_response, status_code=200, ) - reference = client.get_table_reference_by_name("nonexistent", PROJECT_ID) + reference = client.get_table_reference_by_name("nonexistent") assert reference is None @@ -216,7 +193,6 @@ def test_create_table_reference(client: DataMasqueClient, table_reference: Table create_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", - "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -242,7 +218,6 @@ def test_create_table_reference(client: DataMasqueClient, table_reference: Table request_body = m.last_request.json() assert request_body["name"] == "customer_identities" - assert request_body["project"] == PROJECT assert request_body["connection"] == CONNECTION_ID_1 assert request_body["source"] == "identities/customers.csv" assert "options" not in request_body @@ -250,32 +225,9 @@ def test_create_table_reference(client: DataMasqueClient, table_reference: Table assert not read_only_fields & request_body.keys() -def test_create_table_reference_without_a_project(client: DataMasqueClient) -> None: - """No `project` is sent, so the server uses its default project, which is read back.""" - reference = TableReference( - name="customer_identities", connection=ConnectionId(CONNECTION_ID_1), source="identities/customers.csv" - ) - create_response = { - "id": TABLE_REFERENCE_ID_1, - "name": "customer_identities", - "project": PROJECT, - "connection": CONNECTION_ID_1, - "source": "identities/customers.csv", - "options": DEFAULT_OPTIONS_JSON, - } - - with requests_mock.Mocker() as m: - m.post("http://test-server/api/table-references/", json=create_response, status_code=201) - client.create_table_reference(reference) - - assert "project" not in m.last_request.json() - assert reference.project == PROJECT - - def test_create_table_reference_sends_options(client: DataMasqueClient) -> None: reference = TableReference( name="customer_identities", - project=PROJECT, connection=ConnectionId(CONNECTION_ID_1), source="identities/customers.csv", options=TableReferenceOptions(format=TableReferenceFormat.parquet, delimiter=";", null_string="NULL"), @@ -283,7 +235,6 @@ def test_create_table_reference_sends_options(client: DataMasqueClient) -> None: create_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", - "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": { @@ -315,7 +266,6 @@ def test_create_table_reference_sends_null_string_default(client: DataMasqueClie """`null_string`'s default of `None` must be sent explicitly, not dropped from the payload.""" reference = TableReference( name="customer_identities", - project=PROJECT, connection=ConnectionId(CONNECTION_ID_1), source="identities/customers.csv", options=TableReferenceOptions(), @@ -323,7 +273,6 @@ def test_create_table_reference_sends_null_string_default(client: DataMasqueClie create_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", - "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -351,7 +300,6 @@ def test_update_table_reference(client: DataMasqueClient, table_reference: Table update_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", - "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers_v2.csv", "options": DEFAULT_OPTIONS_JSON, @@ -385,7 +333,6 @@ def test_update_table_reference_without_options_sends_none( update_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", - "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": {**DEFAULT_OPTIONS_JSON, "delimiter": ";"}, @@ -416,7 +363,6 @@ def test_create_or_update_table_reference_create(client: DataMasqueClient, table create_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", - "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -439,9 +385,7 @@ def test_create_or_update_table_reference_create(client: DataMasqueClient, table assert result.id == TableReferenceId(TABLE_REFERENCE_ID_1) assert m.request_history[0].method == "GET" - assert m.request_history[0].qs == {"project": [PROJECT]} assert m.request_history[1].method == "POST" - assert m.request_history[1].json()["project"] == PROJECT def test_create_or_update_table_reference_update( @@ -452,7 +396,6 @@ def test_create_or_update_table_reference_update( update_response = { "id": TABLE_REFERENCE_ID_1, "name": "customer_identities", - "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -485,7 +428,6 @@ def test_create_or_update_table_reference_raises_when_server_omits_id( list_response_without_id = [ { "name": "customer_identities", - "project": PROJECT, "connection": CONNECTION_ID_1, "source": "identities/customers.csv", "options": DEFAULT_OPTIONS_JSON, @@ -537,11 +479,10 @@ def test_delete_table_reference_by_name( f"http://test-server/api/table-references/{TABLE_REFERENCE_ID_2}/", status_code=204, ) - client.delete_table_reference_by_name_if_exists("account_identities", PROJECT_ID) + client.delete_table_reference_by_name_if_exists("account_identities") assert m.call_count == 2 assert m.request_history[0].method == "GET" - assert m.request_history[0].qs == {"project": [str(PROJECT_ID)]} assert m.request_history[1].method == "DELETE" assert TABLE_REFERENCE_ID_2 in m.request_history[1].url @@ -555,7 +496,7 @@ def test_delete_table_reference_by_name_not_found( json=sample_table_reference_list_response, status_code=200, ) - client.delete_table_reference_by_name_if_exists("nonexistent", PROJECT_ID) + client.delete_table_reference_by_name_if_exists("nonexistent") assert m.call_count == 1 assert m.request_history[0].method == "GET" diff --git a/uv.lock b/uv.lock index c712118..7164d65 100644 --- a/uv.lock +++ b/uv.lock @@ -419,7 +419,7 @@ toml = [ [[package]] name = "datamasque-python" -version = "1.7.0" +version = "1.5.0" source = { editable = "." } dependencies = [ { name = "pydantic" },