feat: DM-4756 add unstructured discovery to the client

HISTORY.rst

@@ -14,6 +14,33 @@ History
 * ``RunInfo`` and ``UnfinishedRun`` gain a required ``project`` field:
   the run's project, taken from its ruleset's project when the run starts.
 * Added ``connect_timeout_seconds`` and ``read_timeout_seconds`` to the instance config models.
+* Added optional keyword-only ``config_type`` and ``discovery_kind`` arguments to ``get_default_discovery_config_yaml``,
+  typed by ``DiscoveryConfigType`` and the new ``DiscoveryKind`` enum.
+  ``discovery_kind=DiscoveryKind.unstructured`` with ``config_type=DiscoveryConfigType.file``
+  returns the template of an unstructured discovery config,
+  which counts the occurrences of its labels in the text of documents.
+* Added methods that read an unstructured discovery run's results:
+  ``get_unstructured_discovery_folder_page`` and ``iter_unstructured_discovery_folders``
+  for the folders found on the connection,
+  optionally narrowed by ``search``, ``labels`` and ``categories``
+  and sorted by ``UnstructuredDiscoveryFolderOrdering``,
+  ``get_unstructured_discovery_file_page`` and ``iter_unstructured_discovery_files`` for the files of one folder,
+  optionally narrowed by ``UnstructuredDiscoveryFileStatus``, ``search``, ``labels``, ``categories``
+  and ``UnstructuredDiscoveryFileOutcome``,
+  ``get_unstructured_discovery_summary`` for the run's connection, status, end time,
+  ``UnstructuredDiscoverySettings``, ``UnstructuredDiscoveryAiEngine``, totals and how many times it found each label,
+  and ``get_unstructured_discovery_report`` for its findings CSV,
+  or a zip of CSV parts as ``bytes`` for a report of more than 1,000,000 rows.
+  A skip reason is a code such as ``over_sample_limit``, with its sentence in ``skip_reason_text``.
+* Added ``get_last_unstructured_discovery_run_id``,
+  which returns the ID of a connection's newest finished unstructured discovery run that is not a dry run,
+  or ``None`` when it has none.
+* Added ``discovery_kind`` to ``RunInfo``: ``DiscoveryKind.structured`` or ``DiscoveryKind.unstructured``
+  for a file discovery run, ``None`` for a run that ran no discovery task.
+* Added ``ai_engine_model`` and ``ai_engine_version`` to ``RunInfo``:
+  the AI Engine model and engine version the run's ``ai_detect`` matchers use,
+  ``None`` when the run recorded none, such as a run that uses no AI Engine,
+  or on a server that does not report them.
 
 Requires server version 3.26.20
 

datamasque/client/__init__.py

@@ -89,7 +89,12 @@
     TableConstraints,
     ValueCountStatus,
 )
-from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType
+from datamasque.client.models.discovery_config import (
+    DiscoveryConfig,
+    DiscoveryConfigId,
+    DiscoveryConfigType,
+    DiscoveryKind,
+)
 from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary, DiscoveryConfigLibraryId
 from datamasque.client.models.dm_instance import DataMasqueInstanceConfig
 from datamasque.client.models.files import (
@@ -172,6 +177,20 @@
     TableReferenceId,
     TableReferenceOptions,
 )
+from datamasque.client.models.unstructured_discovery import (
+    UnstructuredDiscoveryAiEngine,
+    UnstructuredDiscoveryFile,
+    UnstructuredDiscoveryFileOutcome,
+    UnstructuredDiscoveryFileStatus,
+    UnstructuredDiscoveryFinding,
+    UnstructuredDiscoveryFolder,
+    UnstructuredDiscoveryFolderOrdering,
+    UnstructuredDiscoveryLabelTotal,
+    UnstructuredDiscoverySampleOrder,
+    UnstructuredDiscoverySettings,
+    UnstructuredDiscoverySummary,
+    UnstructuredDiscoveryUnreadCount,
+)
 from datamasque.client.models.user import User, UserId, UserRole
 
 __version__ = version("datamasque-python")
@@ -209,6 +228,7 @@
     "DiscoveryConfigLibraryId",
     "DiscoveryConfigNotFoundError",
     "DiscoveryConfigType",
+    "DiscoveryKind",
     "DiscoveryMatch",
     "CosmosDbConnectionConfig",
     "DocumentDbConnectionConfig",
@@ -323,6 +343,18 @@
     "TemporalStatistics",
     "TemporalSummaries",
     "UnfinishedRun",
+    "UnstructuredDiscoveryFile",
+    "UnstructuredDiscoveryFileOutcome",
+    "UnstructuredDiscoveryFileStatus",
+    "UnstructuredDiscoveryFinding",
+    "UnstructuredDiscoveryFolder",
+    "UnstructuredDiscoveryFolderOrdering",
+    "UnstructuredDiscoveryLabelTotal",
+    "UnstructuredDiscoveryAiEngine",
+    "UnstructuredDiscoverySampleOrder",
+    "UnstructuredDiscoverySettings",
+    "UnstructuredDiscoverySummary",
+    "UnstructuredDiscoveryUnreadCount",
     "UnsupportedPreview",
     "UnsupportedStatistics",
     "User",

datamasque/client/base.py

@@ -313,6 +313,14 @@ def _delete_if_exists(self, path: str, *, params: Optional[dict] = None) -> None
 
         self._raise_for_status(response)
 
+    @staticmethod
+    def _read_csv_or_zip(response: Response) -> Union[str, bytes]:
+        """Returns a report's CSV text, or as `bytes` the zip of CSV parts the server splits a large report into."""
+
+        if response.headers.get("Content-Type", "").startswith("application/zip"):
+            return response.content
+        return response.text
+
     def _iter_paginated(
         self,
         path: str,

datamasque/client/discovery_configs.py

@@ -3,7 +3,12 @@
 
 from datamasque.client.base import BaseClient
 from datamasque.client.exceptions import DataMasqueApiError, DataMasqueException
-from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType
+from datamasque.client.models.discovery_config import (
+    DiscoveryConfig,
+    DiscoveryConfigId,
+    DiscoveryConfigType,
+    DiscoveryKind,
+)
 from datamasque.client.models.pagination import Page
 
 logger = logging.getLogger(__name__)
@@ -163,8 +168,28 @@ def delete_discovery_config_by_name_if_exists(self, name: str, config_type: Disc
 
             self.delete_discovery_config_by_id_if_exists(config.id)
 
-    def get_default_discovery_config_yaml(self) -> str:
-        """Returns the server's built-in default discovery configuration as a YAML string."""
+    def get_default_discovery_config_yaml(
+        self,
+        *,
+        config_type: Optional[DiscoveryConfigType] = None,
+        discovery_kind: Optional[DiscoveryKind] = None,
+    ) -> str:
+        """
+        Returns the server's built-in default discovery configuration as a YAML string.
+
+        `config_type` selects the database or the file default;
+        the server returns the database default when it is omitted.
+        `discovery_kind` selects what a file config looks for;
+        the server returns the structured default when it is omitted.
+        Asking for `DiscoveryKind.unstructured` without `config_type=DiscoveryConfigType.file`
+        raises `DataMasqueApiError` with a 400 response.
+        """
+
+        params: dict[str, str] = {}
+        if config_type is not None:
+            params["config_type"] = config_type.value
+        if discovery_kind is not None:
+            params["discovery_kind"] = discovery_kind.value
 
-        response = self.make_request("GET", "/api/discovery/configs/defaults/")
+        response = self.make_request("GET", "/api/discovery/configs/defaults/", params=params or None)
         return response.content.decode("utf-8")

datamasque/client/dmclient.py

@@ -11,6 +11,7 @@
 from datamasque.client.runs import RunClient
 from datamasque.client.settings import SettingsClient
 from datamasque.client.table_references import TableReferenceClient
+from datamasque.client.unstructured_discovery import UnstructuredDiscoveryClient
 from datamasque.client.users import UserClient
 
 __all__ = ["DataMasqueClient", "FileOrContent", "UploadFile"]
@@ -24,6 +25,7 @@ class DataMasqueClient(
     FileClient,
     RunClient,
     DiscoveryClient,
+    UnstructuredDiscoveryClient,
     DiscoveryConfigClient,
     DiscoveryConfigLibraryClient,
     TableReferenceClient,

datamasque/client/models/discovery_config.py

@@ -16,6 +16,13 @@ class DiscoveryConfigType(enum.Enum):
     file = "file"
 
 
+class DiscoveryKind(enum.Enum):
+    """What a file discovery config looks for: sensitive locators, or its labels in the text of documents."""
+
+    structured = "structured"
+    unstructured = "unstructured"
+
+
 def unwrap_discovery_config_id(value: Any) -> Any:
     """
     Coerce a `DiscoveryConfig` to its `id`; pass other values through unchanged.

datamasque/client/models/runs.py

@@ -7,6 +7,7 @@
 from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 
 from datamasque.client.models.connection import ConnectionConfig, ConnectionId, unwrap_connection_id
+from datamasque.client.models.discovery_config import DiscoveryKind
 from datamasque.client.models.ruleset import Ruleset, RulesetId, unwrap_ruleset_id
 from datamasque.client.models.status import MaskingRunStatus
 
@@ -148,6 +149,25 @@ class RunInfo(BaseModel):
 
     Taken from the ruleset's project when the run starts, and never changes after that.
     """
+    ai_engine_model: Optional[str] = None
+    """
+    The AI Engine model the run's `ai_detect` matchers use:
+    for a Local AI Engine, the name of its model file or folder,
+    and for a Bedrock AI Engine, the Bedrock model set in its configuration.
+    `None` when the run recorded none, such as a run that uses no AI Engine,
+    or on a server that does not report it.
+    """
+    ai_engine_version: Optional[str] = None
+    """
+    The version the AI Engine reported when the run started.
+    `None` when the run recorded none, such as a run that uses no AI Engine,
+    or on a server that does not report it.
+    """
+    discovery_kind: Optional[DiscoveryKind] = None
+    """
+    What a file discovery run looked for: sensitive locators, or labels in the text of documents.
+    `None` for a run that ran no discovery task, or on a server that does not report it.
+    """
 
     @model_validator(mode="before")
     @classmethod