DM-4993 reland projects

HISTORY.rst

@@ -2,8 +2,39 @@
 History
 =======
 
-1.6.0 (unreleased)
-------------------
+1.7.0 (2026-11-06)
+------------------
+
+* Added project models and management APIs.
+* Added resource management APIs.
+* Added ``default_project`` to ``DataMasqueInstanceConfig``.
+* ``ConnectionConfig``, ``Ruleset``, ``RulesetLibrary``, ``DiscoveryConfig``, ``DiscoveryConfigLibrary``, ``RGConfig``
+  and ``TableReference`` send their ``project`` to the server when it is set.
+  It takes a project's ``<id>-<slug>`` reference, such as ``Project.reference``, or its ID as a string.
+  Left unset, a create goes into the client's ``default_project``, else the server's default project.
+  Each create sets the model's ``project`` to the project the server stored it in.
+* **Breaking:** names are unique per project, so every by-name helper takes a required ``project``:
+  ``get_*_by_name``, ``delete_*_by_name_if_exists`` and ``get_file_of_type_by_name``.
+  These ``project`` parameters take a ``Project`` or its ``ProjectId``.
+  ``create_or_update_*`` updates an existing resource only when it has the same name in the same project.
+  If the model has no ``project``, it looks in the client's ``default_project``, else the server's only active project,
+  and raises ``DataMasqueException`` when there is no active project or more than one.
+* **Breaking:** ``upload_file`` and ``upload_file_if_not_exists`` take a required ``project``.
+* **Breaking:** ``namespace`` is removed from ``RulesetLibrary`` and ``DiscoveryConfigLibrary``,
+  and the client drops it from server responses so that it is never sent back.
+  The library by-name helpers take ``project`` in its place.
+* **Breaking:** ``get_unfinished_runs`` returns its runs keyed by ``ConnectionId`` instead of connection name,
+  because two connections in different projects can share a name.
+* ``list_connections``, ``list_table_references`` and ``list_files_of_type`` take an optional ``project`` filter.
+* ``DataMasqueFile`` gains a read-only ``project`` field, which is ``None`` for built-in seeds.
+* ``get_generated_rulesets`` sets each ruleset's ``project`` to the connection's project.
+* Added Git APIs for rulesets, ruleset libraries, discovery configs and discovery config libraries.
+* Added bundle export and import APIs.
+
+Requires server version 3.26.21
+
+1.6.0
+-----
 
 * Added a read-only ``project`` field to ``ConnectionConfig``, ``Ruleset``, ``RulesetLibrary``, ``DiscoveryConfig``,
   ``DiscoveryConfigLibrary``, ``RGConfig`` and ``TableReference``:

datamasque/client/__init__.py

@@ -23,10 +23,21 @@
     InvalidDiscoveryConfigError,
     InvalidLibraryError,
     InvalidRulesetError,
+    ProjectDeleteConflictError,
     RGConfigNotFoundError,
     RunNotCancellableError,
 )
+from datamasque.client.git import GitExtension
 from datamasque.client.ifm import DataMasqueIfmClient
+from datamasque.client.models.bundle import (
+    BundleImportCounts,
+    BundleImportItemResults,
+    BundleImportResult,
+    BundleImportStatus,
+    LibraryImportResult,
+    RulesetImportResult,
+    SeedFileImportResult,
+)
 from datamasque.client.models.connection import (
     AzureConnectionConfig,
     CassandraConnectionConfig,
@@ -100,7 +111,7 @@
     SnowflakeKeyFile,
     SslZipFile,
 )
-from datamasque.client.models.git import GitSnapshot
+from datamasque.client.models.git import GitFile, GitFileCommit, GitSnapshot
 from datamasque.client.models.ifm import (
     DataMasqueIfmInstanceConfig,
     IfmLog,
@@ -115,6 +126,27 @@
     RulesetPlanUpdateRequest,
 )
 from datamasque.client.models.license import LicenseInfo, SwitchableLicenseMetadata
+from datamasque.client.models.project import (
+    Project,
+    ProjectCreateRequest,
+    ProjectDeleteConflict,
+    ProjectDeleteConflictReason,
+    ProjectId,
+    ProjectIdentifier,
+    ProjectUpdateRequest,
+)
+from datamasque.client.models.resource import (
+    ConnectionResource,
+    DiscoveryConfigResource,
+    Resource,
+    ResourceMoveConflict,
+    ResourceMoveConflictReason,
+    ResourceMoveResult,
+    ResourceOrdering,
+    ResourcesByType,
+    RulesetResource,
+)
+from datamasque.client.models.resource_type import ResourceTypeName
 from datamasque.client.models.rg_config import RGConfig, RGConfigId
 from datamasque.client.models.ruleset import Ruleset, RulesetId, RulesetType
 from datamasque.client.models.ruleset_library import RulesetLibrary, RulesetLibraryId
@@ -182,12 +214,17 @@
     "AzureConnectionConfig",
     "BooleanPreview",
     "BooleanStatistics",
+    "BundleImportCounts",
+    "BundleImportItemResults",
+    "BundleImportResult",
+    "BundleImportStatus",
     "CassandraConnectionConfig",
     "ColumnKind",
     "ColumnPreview",
     "CommonStatistics",
     "ConnectionConfig",
     "ConnectionId",
+    "ConnectionResource",
     "ConstraintColumns",
     "DataMasqueApiError",
     "DataMasqueClient",
@@ -208,6 +245,7 @@
     "DiscoveryConfigLibrary",
     "DiscoveryConfigLibraryId",
     "DiscoveryConfigNotFoundError",
+    "DiscoveryConfigResource",
     "DiscoveryConfigType",
     "DiscoveryMatch",
     "CosmosDbConnectionConfig",
@@ -231,6 +269,9 @@
     "FirstCharsStatistics",
     "ForeignKeyRef",
     "GcsConnectionConfig",
+    "GitExtension",
+    "GitFile",
+    "GitFileCommit",
     "GitSnapshot",
     "HashColumnsTableConfig",
     "IfmAuthError",
@@ -248,6 +289,7 @@
     "LengthEntry",
     "LengthUnit",
     "LengthsStatistics",
+    "LibraryImportResult",
     "LicenseInfo",
     "LicenseLock",
     "Locator",
@@ -269,21 +311,38 @@
     "PatternComposition",
     "PatternEntry",
     "PatternsStatistics",
+    "Project",
+    "ProjectCreateRequest",
+    "ProjectDeleteConflict",
+    "ProjectDeleteConflictError",
+    "ProjectDeleteConflictReason",
+    "ProjectId",
+    "ProjectIdentifier",
+    "ProjectUpdateRequest",
     "RGConfig",
     "RGConfigId",
     "RGConfigNotFoundError",
     "ReferencingForeignKey",
+    "Resource",
+    "ResourceMoveConflict",
+    "ResourceMoveConflictReason",
+    "ResourceMoveResult",
+    "ResourceOrdering",
+    "ResourceTypeName",
+    "ResourcesByType",
     "Ruleset",
     "RulesetGenerationRequest",
     "RulesetGenerationWithRGConfigRequest",
     "RulesetId",
+    "RulesetImportResult",
     "RulesetLibrary",
     "RulesetLibraryId",
     "RulesetPlan",
     "RulesetPlanCreateRequest",
     "RulesetPlanOptions",
     "RulesetPlanPartialUpdateRequest",
     "RulesetPlanUpdateRequest",
+    "RulesetResource",
     "RulesetType",
     "RunConnectionRef",
     "RunId",
@@ -300,6 +359,7 @@
     "SchemaDiscoveryRequest",
     "SchemaDiscoveryResult",
     "SeedFile",
+    "SeedFileImportResult",
     "SelectedColumns",
     "SelectedData",
     "SelectedFileData",

datamasque/client/base.py

@@ -17,10 +17,13 @@
 
 from datamasque.client.exceptions import (
     DataMasqueApiError,
+    DataMasqueException,
     DataMasqueNotReadyError,
     DataMasqueTransportError,
 )
 from datamasque.client.models.dm_instance import DataMasqueInstanceConfig
+from datamasque.client.models.pagination import Page
+from datamasque.client.models.project import Project, ProjectId
 from datamasque.client.spcs import install_spcs_gateway_auth
 
 logger = logging.getLogger(__name__)
@@ -130,13 +133,15 @@ class BaseClient:
     password: Optional[str]
     verify_ssl: bool
     token_source: Optional[Callable[[], str]]
+    default_project: Optional[str]
 
     def __init__(self, connection_config: DataMasqueInstanceConfig) -> None:
         self.base_url = connection_config.base_url
         self.username = connection_config.username
         self.password = connection_config.password
         self.verify_ssl = connection_config.verify_ssl
         self.token_source = connection_config.token_source
+        self.default_project = connection_config.default_project
         self._session = _build_session(self.verify_ssl)
         self._timeout = (connection_config.connect_timeout_seconds, connection_config.read_timeout_seconds)
         if connection_config.spcs_pat:
@@ -313,6 +318,40 @@ def _delete_if_exists(self, path: str, *, params: Optional[dict] = None) -> None
 
         self._raise_for_status(response)
 
+    def _resolve_project(self, project: Optional[Union[ProjectId, str]]) -> Union[ProjectId, str]:
+        """
+        Returns `project`, else the client's `default_project`, else the ID of the only active project.
+
+        The last is the project the server puts a create into when it is sent no `project`.
+        With no active project, or more than one, the server rejects such a create,
+        so this raises `DataMasqueException` instead.
+        """
+
+        if project is not None:
+            return project
+        if self.default_project is not None:
+            return self.default_project
+
+        response = self.make_request("GET", "/api/projects/", params={"limit": 1})
+        page = Page[Project].model_validate(response.json())
+        if page.count != 1:
+            raise DataMasqueException(
+                f"No `project` is set and the server has {page.count} active projects. Set `project` to choose one."
+            )
+
+        return page.results[0].id
+
+    def _default_project_field(self, project: Optional[Union[ProjectId, str]]) -> dict[str, str]:
+        """
+        Returns the `project` field a create sends when the resource names no `project`: the client's `default_project`.
+
+        Empty when the resource names one, which its own fields carry, or when there is no `default_project`.
+        """
+
+        if project is None and self.default_project is not None:
+            return {"project": self.default_project}
+        return {}
+
     def _iter_paginated(
         self,
         path: str,
@@ -321,8 +360,14 @@ def _iter_paginated(
         params: Optional[dict] = None,
         page_size: int = 100,
     ) -> Iterator[_T]:
+        """Iterate every `T` across all pages of an admin-server list endpoint."""
+
+        for results in self._iter_pages(path, params=params, page_size=page_size):
+            yield from (model.model_validate(item) for item in results)
+
+    def _iter_pages(self, path: str, *, params: Optional[dict] = None, page_size: int = 100) -> Iterator[Any]:
         """
-        Iterate every `T` across all pages of an admin-server list endpoint.
+        Iterate the `results` of each page of an admin-server list endpoint.
 
         Opts into pagination by sending `limit`/`offset` on the first request,
         then follows the absolute `next` URL returned by the server.
@@ -338,7 +383,7 @@ def _iter_paginated(
         while url:
             response = self.make_request("GET", url, params=current_params)
             data = response.json()
-            yield from (model.model_validate(item) for item in data["results"])
+            yield data["results"]
             url = data.get("next")
             # The `next` URL is absolute and already contains the pagination cursor;
             # do not re-send our initial params alongside it.