Skip to content

revert: DM-4993 defer projects to a later release #71

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
elliotsimpson-dm merged 1 commit into from
Nov 4, 2026
Merged

revert: DM-4993 defer projects to a later release #71

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 2 additions & 33 deletions HISTORY.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,39 +2,8 @@
History
=======

1.7.0 (2026-11-06)
------------------

* Added project models and management APIs.
* Added resource management APIs.
* Added ``default_project`` to ``DataMasqueInstanceConfig``.
* ``ConnectionConfig``, ``Ruleset``, ``RulesetLibrary``, ``DiscoveryConfig``, ``DiscoveryConfigLibrary``, ``RGConfig``
and ``TableReference`` send their ``project`` to the server when it is set.
It takes a project's ``<id>-<slug>`` reference, such as ``Project.reference``, or its ID as a string.
Left unset, a create goes into the client's ``default_project``, else the server's default project.
Each create sets the model's ``project`` to the project the server stored it in.
* **Breaking:** names are unique per project, so every by-name helper takes a required ``project``:
``get_*_by_name``, ``delete_*_by_name_if_exists`` and ``get_file_of_type_by_name``.
These ``project`` parameters take a ``Project`` or its ``ProjectId``.
``create_or_update_*`` updates an existing resource only when it has the same name in the same project.
If the model has no ``project``, it looks in the client's ``default_project``, else the server's only active project,
and raises ``DataMasqueException`` when there is no active project or more than one.
* **Breaking:** ``upload_file`` and ``upload_file_if_not_exists`` take a required ``project``.
* **Breaking:** ``namespace`` is removed from ``RulesetLibrary`` and ``DiscoveryConfigLibrary``,
and the client drops it from server responses so that it is never sent back.
The library by-name helpers take ``project`` in its place.
* **Breaking:** ``get_unfinished_runs`` returns its runs keyed by ``ConnectionId`` instead of connection name,
because two connections in different projects can share a name.
* ``list_connections``, ``list_table_references`` and ``list_files_of_type`` take an optional ``project`` filter.
* ``DataMasqueFile`` gains a read-only ``project`` field, which is ``None`` for built-in seeds.
* ``get_generated_rulesets`` sets each ruleset's ``project`` to the connection's project.
* Added Git APIs for rulesets, ruleset libraries, discovery configs and discovery config libraries.
* Added bundle export and import APIs.

Requires server version 3.26.21

1.6.0
-----
1.6.0 (unreleased)
------------------

* Added a read-only ``project`` field to ``ConnectionConfig``, ``Ruleset``, ``RulesetLibrary``, ``DiscoveryConfig``,
``DiscoveryConfigLibrary``, ``RGConfig`` and ``TableReference``:
Expand Down
62 changes: 1 addition & 61 deletions datamasque/client/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,21 +23,10 @@
InvalidDiscoveryConfigError,
InvalidLibraryError,
InvalidRulesetError,
ProjectDeleteConflictError,
RGConfigNotFoundError,
RunNotCancellableError,
)
from datamasque.client.git import GitExtension
from datamasque.client.ifm import DataMasqueIfmClient
from datamasque.client.models.bundle import (
BundleImportCounts,
BundleImportItemResults,
BundleImportResult,
BundleImportStatus,
LibraryImportResult,
RulesetImportResult,
SeedFileImportResult,
)
from datamasque.client.models.connection import (
AzureConnectionConfig,
CassandraConnectionConfig,
Expand Down Expand Up @@ -111,7 +100,7 @@
SnowflakeKeyFile,
SslZipFile,
)
from datamasque.client.models.git import GitFile, GitFileCommit, GitSnapshot
from datamasque.client.models.git import GitSnapshot
from datamasque.client.models.ifm import (
DataMasqueIfmInstanceConfig,
IfmLog,
Expand All @@ -126,27 +115,6 @@
RulesetPlanUpdateRequest,
)
from datamasque.client.models.license import LicenseInfo, SwitchableLicenseMetadata
from datamasque.client.models.project import (
Project,
ProjectCreateRequest,
ProjectDeleteConflict,
ProjectDeleteConflictReason,
ProjectId,
ProjectIdentifier,
ProjectUpdateRequest,
)
from datamasque.client.models.resource import (
ConnectionResource,
DiscoveryConfigResource,
Resource,
ResourceMoveConflict,
ResourceMoveConflictReason,
ResourceMoveResult,
ResourceOrdering,
ResourcesByType,
RulesetResource,
)
from datamasque.client.models.resource_type import ResourceTypeName
from datamasque.client.models.rg_config import RGConfig, RGConfigId
from datamasque.client.models.ruleset import Ruleset, RulesetId, RulesetType
from datamasque.client.models.ruleset_library import RulesetLibrary, RulesetLibraryId
Expand Down Expand Up @@ -214,17 +182,12 @@
"AzureConnectionConfig",
"BooleanPreview",
"BooleanStatistics",
"BundleImportCounts",
"BundleImportItemResults",
"BundleImportResult",
"BundleImportStatus",
"CassandraConnectionConfig",
"ColumnKind",
"ColumnPreview",
"CommonStatistics",
"ConnectionConfig",
"ConnectionId",
"ConnectionResource",
"ConstraintColumns",
"DataMasqueApiError",
"DataMasqueClient",
Expand All @@ -245,7 +208,6 @@
"DiscoveryConfigLibrary",
"DiscoveryConfigLibraryId",
"DiscoveryConfigNotFoundError",
"DiscoveryConfigResource",
"DiscoveryConfigType",
"DiscoveryMatch",
"CosmosDbConnectionConfig",
Expand All @@ -269,9 +231,6 @@
"FirstCharsStatistics",
"ForeignKeyRef",
"GcsConnectionConfig",
"GitExtension",
"GitFile",
"GitFileCommit",
"GitSnapshot",
"HashColumnsTableConfig",
"IfmAuthError",
Expand All @@ -289,7 +248,6 @@
"LengthEntry",
"LengthUnit",
"LengthsStatistics",
"LibraryImportResult",
"LicenseInfo",
"LicenseLock",
"Locator",
Expand All @@ -311,38 +269,21 @@
"PatternComposition",
"PatternEntry",
"PatternsStatistics",
"Project",
"ProjectCreateRequest",
"ProjectDeleteConflict",
"ProjectDeleteConflictError",
"ProjectDeleteConflictReason",
"ProjectId",
"ProjectIdentifier",
"ProjectUpdateRequest",
"RGConfig",
"RGConfigId",
"RGConfigNotFoundError",
"ReferencingForeignKey",
"Resource",
"ResourceMoveConflict",
"ResourceMoveConflictReason",
"ResourceMoveResult",
"ResourceOrdering",
"ResourceTypeName",
"ResourcesByType",
"Ruleset",
"RulesetGenerationRequest",
"RulesetGenerationWithRGConfigRequest",
"RulesetId",
"RulesetImportResult",
"RulesetLibrary",
"RulesetLibraryId",
"RulesetPlan",
"RulesetPlanCreateRequest",
"RulesetPlanOptions",
"RulesetPlanPartialUpdateRequest",
"RulesetPlanUpdateRequest",
"RulesetResource",
"RulesetType",
"RunConnectionRef",
"RunId",
Expand All @@ -359,7 +300,6 @@
"SchemaDiscoveryRequest",
"SchemaDiscoveryResult",
"SeedFile",
"SeedFileImportResult",
"SelectedColumns",
"SelectedData",
"SelectedFileData",
Expand Down
49 changes: 2 additions & 47 deletions datamasque/client/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,10 @@

from datamasque.client.exceptions import (
DataMasqueApiError,
DataMasqueException,
DataMasqueNotReadyError,
DataMasqueTransportError,
)
from datamasque.client.models.dm_instance import DataMasqueInstanceConfig
from datamasque.client.models.pagination import Page
from datamasque.client.models.project import Project, ProjectId
from datamasque.client.spcs import install_spcs_gateway_auth

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -133,15 +130,13 @@ class BaseClient:
password: Optional[str]
verify_ssl: bool
token_source: Optional[Callable[[], str]]
default_project: Optional[str]

def __init__(self, connection_config: DataMasqueInstanceConfig) -> None:
self.base_url = connection_config.base_url
self.username = connection_config.username
self.password = connection_config.password
self.verify_ssl = connection_config.verify_ssl
self.token_source = connection_config.token_source
self.default_project = connection_config.default_project
self._session = _build_session(self.verify_ssl)
self._timeout = (connection_config.connect_timeout_seconds, connection_config.read_timeout_seconds)
if connection_config.spcs_pat:
Expand Down Expand Up @@ -318,40 +313,6 @@ def _delete_if_exists(self, path: str, *, params: Optional[dict] = None) -> None

self._raise_for_status(response)

def _resolve_project(self, project: Optional[Union[ProjectId, str]]) -> Union[ProjectId, str]:
"""
Returns `project`, else the client's `default_project`, else the ID of the only active project.

The last is the project the server puts a create into when it is sent no `project`.
With no active project, or more than one, the server rejects such a create,
so this raises `DataMasqueException` instead.
"""

if project is not None:
return project
if self.default_project is not None:
return self.default_project

response = self.make_request("GET", "/api/projects/", params={"limit": 1})
page = Page[Project].model_validate(response.json())
if page.count != 1:
raise DataMasqueException(
f"No `project` is set and the server has {page.count} active projects. Set `project` to choose one."
)

return page.results[0].id

def _default_project_field(self, project: Optional[Union[ProjectId, str]]) -> dict[str, str]:
"""
Returns the `project` field a create sends when the resource names no `project`: the client's `default_project`.

Empty when the resource names one, which its own fields carry, or when there is no `default_project`.
"""

if project is None and self.default_project is not None:
return {"project": self.default_project}
return {}

def _iter_paginated(
self,
path: str,
Expand All @@ -360,14 +321,8 @@ def _iter_paginated(
params: Optional[dict] = None,
page_size: int = 100,
) -> Iterator[_T]:
"""Iterate every `T` across all pages of an admin-server list endpoint."""

for results in self._iter_pages(path, params=params, page_size=page_size):
yield from (model.model_validate(item) for item in results)

def _iter_pages(self, path: str, *, params: Optional[dict] = None, page_size: int = 100) -> Iterator[Any]:
"""
Iterate the `results` of each page of an admin-server list endpoint.
Iterate every `T` across all pages of an admin-server list endpoint.

Opts into pagination by sending `limit`/`offset` on the first request,
then follows the absolute `next` URL returned by the server.
Expand All @@ -383,7 +338,7 @@ def _iter_pages(self, path: str, *, params: Optional[dict] = None, page_size: in
while url:
response = self.make_request("GET", url, params=current_params)
data = response.json()
yield data["results"]
yield from (model.model_validate(item) for item in data["results"])
url = data.get("next")
# The `next` URL is absolute and already contains the pagination cursor;
# do not re-send our initial params alongside it.
Expand Down
Loading
Loading