From a26519a23a4d00f14f44d50840a01de8b7b464ee Mon Sep 17 00:00:00 2001 From: Timotius Date: Wed, 4 Nov 2026 16:55:39 +1300 Subject: [PATCH 01/23] feat: DM-4756: take config_type and discovery_kind for the defaults YAML --- HISTORY.rst | 5 ++ datamasque/client/__init__.py | 8 ++- datamasque/client/discovery_configs.py | 33 ++++++++++-- datamasque/client/models/discovery_config.py | 7 +++ docs/client.models.rst | 8 +++ docs/client.rst | 8 +++ tests/test_discovery_configs.py | 56 ++++++++++++++++++++ 7 files changed, 120 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 191e613..6d4b981 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,11 @@ History * ``RunInfo`` and ``UnfinishedRun`` gain a required ``project`` field: the run's project, taken from its ruleset's project when the run starts. * Added ``connect_timeout_seconds`` and ``read_timeout_seconds`` to the instance config models. +* Added optional keyword-only ``config_type`` and ``discovery_kind`` arguments to ``get_default_discovery_config_yaml``, + typed by ``DiscoveryConfigType`` and the new ``DiscoveryKind`` enum. + ``discovery_kind=DiscoveryKind.unstructured`` with ``config_type=DiscoveryConfigType.file`` + returns the template of an unstructured discovery config, + which counts entities in the text of documents. Requires server version 3.26.20 diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index 2122c6d..5947d33 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -89,7 +89,12 @@ TableConstraints, ValueCountStatus, ) -from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType +from datamasque.client.models.discovery_config import ( + DiscoveryConfig, + DiscoveryConfigId, + DiscoveryConfigType, + DiscoveryKind, +) from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary, DiscoveryConfigLibraryId from datamasque.client.models.dm_instance import DataMasqueInstanceConfig from datamasque.client.models.files import ( @@ -209,6 +214,7 @@ "DiscoveryConfigLibraryId", "DiscoveryConfigNotFoundError", "DiscoveryConfigType", + "DiscoveryKind", "DiscoveryMatch", "CosmosDbConnectionConfig", "DocumentDbConnectionConfig", diff --git a/datamasque/client/discovery_configs.py b/datamasque/client/discovery_configs.py index d19f900..6196b8a 100644 --- a/datamasque/client/discovery_configs.py +++ b/datamasque/client/discovery_configs.py @@ -3,7 +3,12 @@ from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueApiError, DataMasqueException -from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType +from datamasque.client.models.discovery_config import ( + DiscoveryConfig, + DiscoveryConfigId, + DiscoveryConfigType, + DiscoveryKind, +) from datamasque.client.models.pagination import Page logger = logging.getLogger(__name__) @@ -163,8 +168,28 @@ def delete_discovery_config_by_name_if_exists(self, name: str, config_type: Disc self.delete_discovery_config_by_id_if_exists(config.id) - def get_default_discovery_config_yaml(self) -> str: - """Returns the server's built-in default discovery configuration as a YAML string.""" + def get_default_discovery_config_yaml( + self, + *, + config_type: Optional[DiscoveryConfigType] = None, + discovery_kind: Optional[DiscoveryKind] = None, + ) -> str: + """ + Returns the server's built-in default discovery configuration as a YAML string. + + `config_type` selects the database or the file default; + the server returns the database default when it is omitted. + `discovery_kind` selects what a file config looks for; + the server returns the structured default when it is omitted. + Asking for `DiscoveryKind.unstructured` without `config_type=DiscoveryConfigType.file` + raises `DataMasqueApiError` with a 400 response. + """ + + params: dict[str, str] = {} + if config_type is not None: + params["config_type"] = config_type.value + if discovery_kind is not None: + params["discovery_kind"] = discovery_kind.value - response = self.make_request("GET", "/api/discovery/configs/defaults/") + response = self.make_request("GET", "/api/discovery/configs/defaults/", params=params or None) return response.content.decode("utf-8") diff --git a/datamasque/client/models/discovery_config.py b/datamasque/client/models/discovery_config.py index 086257f..051dcf8 100644 --- a/datamasque/client/models/discovery_config.py +++ b/datamasque/client/models/discovery_config.py @@ -16,6 +16,13 @@ class DiscoveryConfigType(enum.Enum): file = "file" +class DiscoveryKind(enum.Enum): + """What a file discovery config looks for: sensitive locators, or entities in the text of documents.""" + + structured = "structured" + unstructured = "unstructured" + + def unwrap_discovery_config_id(value: Any) -> Any: """ Coerce a `DiscoveryConfig` to its `id`; pass other values through unchanged. diff --git a/docs/client.models.rst b/docs/client.models.rst index e98789b..31d4585 100644 --- a/docs/client.models.rst +++ b/docs/client.models.rst @@ -28,6 +28,14 @@ datamasque.client.models.discovery module :undoc-members: :show-inheritance: +datamasque.client.models.discovery_config module +------------------------------------------------ + +.. automodule:: datamasque.client.models.discovery_config + :members: + :undoc-members: + :show-inheritance: + datamasque.client.models.dm_instance module ------------------------------------------- diff --git a/docs/client.rst b/docs/client.rst index f013c64..8e7d73d 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -36,6 +36,14 @@ datamasque.client.discovery module :undoc-members: :show-inheritance: +datamasque.client.discovery_configs module +------------------------------------------ + +.. automodule:: datamasque.client.discovery_configs + :members: + :undoc-members: + :show-inheritance: + datamasque.client.dmclient module --------------------------------- diff --git a/tests/test_discovery_configs.py b/tests/test_discovery_configs.py index d34c47d..a5378d6 100644 --- a/tests/test_discovery_configs.py +++ b/tests/test_discovery_configs.py @@ -13,6 +13,7 @@ DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType, + DiscoveryKind, unwrap_discovery_config_id, ) from datamasque.client.models.status import ValidationStatus @@ -478,6 +479,61 @@ def test_get_default_discovery_config_yaml(client: DataMasqueClient) -> None: result = client.get_default_discovery_config_yaml() assert result == yaml_body + assert m.last_request.qs == {} + + +def test_get_default_discovery_config_yaml_asks_for_the_type_and_kind(client: DataMasqueClient) -> None: + yaml_body = "discovery_kind: unstructured\nlabels: []\n" + + with requests_mock.Mocker(case_sensitive=True) as m: + m.get( + "http://test-server/api/discovery/configs/defaults/", + text=yaml_body, + status_code=200, + headers={"Content-Type": "application/x-yaml"}, + ) + result = client.get_default_discovery_config_yaml( + config_type=DiscoveryConfigType.file, discovery_kind=DiscoveryKind.unstructured + ) + + assert result == yaml_body + assert m.last_request.qs == {"config_type": ["file"], "discovery_kind": ["unstructured"]} + + +@pytest.mark.parametrize( + "kwargs, expected_query", + [ + ({"config_type": DiscoveryConfigType.database}, {"config_type": ["database"]}), + ({"discovery_kind": DiscoveryKind.structured}, {"discovery_kind": ["structured"]}), + ], +) +def test_get_default_discovery_config_yaml_sends_only_the_parameters_given( + client: DataMasqueClient, kwargs: dict[str, Any], expected_query: dict[str, list[str]] +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get("http://test-server/api/discovery/configs/defaults/", text="labels: []\n", status_code=200) + client.get_default_discovery_config_yaml(**kwargs) + + assert m.last_request.qs == expected_query + + +def test_get_default_discovery_config_yaml_raises_when_the_server_refuses_the_kind(client: DataMasqueClient) -> None: + refusal = {"discovery_kind": "`discovery_kind` `unstructured` is available for a `file` config only."} + with requests_mock.Mocker(case_sensitive=True) as m: + m.get("http://test-server/api/discovery/configs/defaults/", json=refusal, status_code=400) + with pytest.raises(DataMasqueApiError) as exc_info: + client.get_default_discovery_config_yaml( + config_type=DiscoveryConfigType.database, discovery_kind=DiscoveryKind.unstructured + ) + + assert m.last_request.qs == {"config_type": ["database"], "discovery_kind": ["unstructured"]} + assert exc_info.value.response.status_code == 400 + assert exc_info.value.response.json() == refusal + + +def test_default_discovery_config_parameters_are_keyword_only(client: DataMasqueClient) -> None: + with pytest.raises(TypeError): + client.get_default_discovery_config_yaml(DiscoveryConfigType.file) def test_discovery_config_parses_validation_fields() -> None: From 5808da016363d271b474c058e56a2ce1a29fc2bf Mon Sep 17 00:00:00 2001 From: Timotius Date: Wed, 4 Nov 2026 19:26:54 +1300 Subject: [PATCH 02/23] feat: DM-4756: read a document run's folders, files, totals and CSV --- HISTORY.rst | 10 + datamasque/client/__init__.py | 18 ++ datamasque/client/dmclient.py | 2 + .../client/models/unstructured_discovery.py | 119 +++++++++ datamasque/client/unstructured_discovery.py | 124 ++++++++++ docs/client.models.rst | 8 + docs/client.rst | 8 + tests/test_unstructured_discovery.py | 230 ++++++++++++++++++ 8 files changed, 519 insertions(+) create mode 100644 datamasque/client/models/unstructured_discovery.py create mode 100644 datamasque/client/unstructured_discovery.py create mode 100644 tests/test_unstructured_discovery.py diff --git a/HISTORY.rst b/HISTORY.rst index 6d4b981..c50dc5c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,6 +19,16 @@ History ``discovery_kind=DiscoveryKind.unstructured`` with ``config_type=DiscoveryConfigType.file`` returns the template of an unstructured discovery config, which counts entities in the text of documents. +* Added methods that read an unstructured discovery run's results: + ``get_unstructured_discovery_folder_page`` and ``iter_unstructured_discovery_folders`` + for the folders found on the connection, + ``get_unstructured_discovery_file_page`` and ``iter_unstructured_discovery_files`` for the files of one folder, + optionally narrowed by ``UnstructuredDiscoveryFileStatus``, + ``get_unstructured_discovery_summary`` for its totals, + and ``get_unstructured_report`` for its findings CSV, + or a zip of CSV parts as ``bytes`` for a report of more than 1,000,000 rows. + Each page is an ``UnstructuredDiscoveryPage``, + which also carries the run's connection, end time and ``UnstructuredDiscoverySampling``. Requires server version 3.26.20 diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index 5947d33..603203a 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -177,6 +177,16 @@ TableReferenceId, TableReferenceOptions, ) +from datamasque.client.models.unstructured_discovery import ( + UnstructuredDiscoveryFile, + UnstructuredDiscoveryFileStatus, + UnstructuredDiscoveryFinding, + UnstructuredDiscoveryFolder, + UnstructuredDiscoveryPage, + UnstructuredDiscoverySampling, + UnstructuredDiscoverySummary, + UnstructuredDiscoveryUnreadCount, +) from datamasque.client.models.user import User, UserId, UserRole __version__ = version("datamasque-python") @@ -329,6 +339,14 @@ "TemporalStatistics", "TemporalSummaries", "UnfinishedRun", + "UnstructuredDiscoveryFile", + "UnstructuredDiscoveryFileStatus", + "UnstructuredDiscoveryFinding", + "UnstructuredDiscoveryFolder", + "UnstructuredDiscoveryPage", + "UnstructuredDiscoverySampling", + "UnstructuredDiscoverySummary", + "UnstructuredDiscoveryUnreadCount", "UnsupportedPreview", "UnsupportedStatistics", "User", diff --git a/datamasque/client/dmclient.py b/datamasque/client/dmclient.py index 7e8cc7f..4c38302 100644 --- a/datamasque/client/dmclient.py +++ b/datamasque/client/dmclient.py @@ -11,6 +11,7 @@ from datamasque.client.runs import RunClient from datamasque.client.settings import SettingsClient from datamasque.client.table_references import TableReferenceClient +from datamasque.client.unstructured_discovery import UnstructuredDiscoveryClient from datamasque.client.users import UserClient __all__ = ["DataMasqueClient", "FileOrContent", "UploadFile"] @@ -24,6 +25,7 @@ class DataMasqueClient( FileClient, RunClient, DiscoveryClient, + UnstructuredDiscoveryClient, DiscoveryConfigClient, DiscoveryConfigLibraryClient, TableReferenceClient, diff --git a/datamasque/client/models/unstructured_discovery.py b/datamasque/client/models/unstructured_discovery.py new file mode 100644 index 0000000..be11b83 --- /dev/null +++ b/datamasque/client/models/unstructured_discovery.py @@ -0,0 +1,119 @@ +"""Typed response shapes for an unstructured discovery run's results.""" + +import enum +from datetime import datetime +from typing import Generic, Optional, TypeVar + +from pydantic import BaseModel, ConfigDict + +from datamasque.client.models.pagination import Page +from datamasque.client.models.runs import RunConnectionRef, RunId + +T = TypeVar("T") + + +class UnstructuredDiscoveryFileStatus(enum.Enum): + """Whether an unstructured discovery run read a file's text.""" + + read = "read" + unread = "unread" + """The run tried to read the file and could not; `skip_reason` says why.""" + + +class UnstructuredDiscoverySampling(enum.Enum): + """How many files of each folder an unstructured discovery run read.""" + + sampled = "sampled" + """At most `files_per_directory` files of each folder.""" + all = "all" + """Every file of each folder that the run could read.""" + unknown = "unknown" + """The run's discovery config cannot be read.""" + + +class UnstructuredDiscoveryUnreadCount(BaseModel): + """How many of a folder's files the run did not try to read for one reason, at one extension.""" + + model_config = ConfigDict(extra="allow") + + # A string rather than an enum, because each server release can add reasons. + skip_reason: str + file_extension: str + """The extension without a leading dot, blank for files that have none.""" + count: int + + +class UnstructuredDiscoveryFolder(BaseModel): + """One folder an unstructured discovery run found on the connection, and what its files add up to.""" + + model_config = ConfigDict(extra="allow") + + directory: str + """The folder's path, blank at the connection root.""" + files_present: int + """How many files were found in the folder, read or not.""" + files_read: int + size_bytes: int + """The total size, in bytes, of the folder's files, read or not.""" + entity_counts: dict[str, int] + """Each label's occurrences, summed over the folder's files.""" + unread: list[UnstructuredDiscoveryUnreadCount] + """How many of the folder's files the run did not try to read, by reason and extension.""" + + +class UnstructuredDiscoveryFinding(BaseModel): + """How many times one label was found in one file.""" + + model_config = ConfigDict(extra="allow") + + label: str + """The label's `name` in the discovery config.""" + description: str + occurrences: int + categories: list[str] + + +class UnstructuredDiscoveryFile(BaseModel): + """One file an unstructured discovery run tried to read, and how many times each label was found in it.""" + + model_config = ConfigDict(extra="allow") + + path: str + file_extension: str + size_bytes: int + status: UnstructuredDiscoveryFileStatus + skip_reason: str + """Why the run could not read the file, blank if it did.""" + findings: list[UnstructuredDiscoveryFinding] + + +class UnstructuredDiscoveryPage(Page[T], Generic[T]): + """ + A page of an unstructured discovery run's folders, or of one folder's files. + + Every page also carries the run it belongs to and how the run sampled each folder. + `get_unstructured_discovery_summary` returns the run's totals. + """ + + run_id: RunId + connection: RunConnectionRef + finished_at: Optional[datetime] = None + """When the run ended, `None` until it has.""" + sampling: UnstructuredDiscoverySampling + files_per_directory: Optional[int] = None + """The sample size when `sampling` is `UnstructuredDiscoverySampling.sampled`, `None` otherwise.""" + + +class UnstructuredDiscoverySummary(BaseModel): + """An unstructured discovery run's totals.""" + + model_config = ConfigDict(extra="allow") + + run_id: RunId + files_found: int + """How many files were found on the connection, summed over the folders.""" + files_read: int + files_with_findings: int + """How many of the files the run read hold a finding.""" + bytes_opened: int + """The total size, in bytes, of the files the run read.""" diff --git a/datamasque/client/unstructured_discovery.py b/datamasque/client/unstructured_discovery.py new file mode 100644 index 0000000..e3bf79f --- /dev/null +++ b/datamasque/client/unstructured_discovery.py @@ -0,0 +1,124 @@ +from typing import Iterator, Optional, Union + +from datamasque.client.base import BaseClient +from datamasque.client.models.runs import RunId +from datamasque.client.models.unstructured_discovery import ( + UnstructuredDiscoveryFile, + UnstructuredDiscoveryFileStatus, + UnstructuredDiscoveryFolder, + UnstructuredDiscoveryPage, + UnstructuredDiscoverySummary, +) + +SERVER_MAX_PAGE_SIZE = 1000 + + +class UnstructuredDiscoveryClient(BaseClient): + """ + Unstructured discovery result API methods. Mixed into `DataMasqueClient`. + + Each method raises `DataMasqueApiError` with a 404 response + when the run is not an unstructured discovery run. + """ + + @staticmethod + def _build_unstructured_file_page_params( + directory: str, status: Optional[UnstructuredDiscoveryFileStatus] + ) -> dict[str, str]: + params = {"directory": directory} + if status is not None: + params["status"] = status.value + + return params + + def get_unstructured_discovery_folder_page( + self, run_id: RunId, *, limit: int = 50, offset: int = 0 + ) -> UnstructuredDiscoveryPage[UnstructuredDiscoveryFolder]: + """Returns one page of the folders an unstructured discovery run found on the connection, ordered by path.""" + + response = self.make_request( + "GET", + f"/api/runs/{run_id}/unstructured-discovery/", + params={"limit": limit, "offset": offset}, + ) + return UnstructuredDiscoveryPage[UnstructuredDiscoveryFolder].model_validate(response.json()) + + # `iter_unstructured_discovery_folders` and `iter_unstructured_discovery_files` have no `list_` counterparts, + # because a run's folder and file counts are unbounded. + def iter_unstructured_discovery_folders(self, run_id: RunId) -> Iterator[UnstructuredDiscoveryFolder]: + """Lazily iterate every folder an unstructured discovery run found on the connection, ordered by path.""" + + return self._iter_paginated( + f"/api/runs/{run_id}/unstructured-discovery/", + model=UnstructuredDiscoveryFolder, + page_size=SERVER_MAX_PAGE_SIZE, + ) + + def get_unstructured_discovery_file_page( + self, + run_id: RunId, + directory: str, + *, + status: Optional[UnstructuredDiscoveryFileStatus] = None, + limit: int = 50, + offset: int = 0, + ) -> UnstructuredDiscoveryPage[UnstructuredDiscoveryFile]: + """ + Returns one page of the files an unstructured discovery run tried to read in one folder, ordered by name. + + `directory` is a folder's `directory` exactly as the folder pages name it; + an empty string names the connection root. + `status` keeps only the files with that status. + Files the run did not try to read have no row; + the folder's `unread` counts them. + """ + + params = self._build_unstructured_file_page_params(directory, status) | {"limit": limit, "offset": offset} + response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-discovery/", params=params) + return UnstructuredDiscoveryPage[UnstructuredDiscoveryFile].model_validate(response.json()) + + def iter_unstructured_discovery_files( + self, + run_id: RunId, + directory: str, + *, + status: Optional[UnstructuredDiscoveryFileStatus] = None, + ) -> Iterator[UnstructuredDiscoveryFile]: + """ + Lazily iterate every file an unstructured discovery run tried to read in one folder, ordered by name. + + `directory` and `status` are as for `get_unstructured_discovery_file_page`. + """ + + return self._iter_paginated( + f"/api/runs/{run_id}/unstructured-discovery/", + model=UnstructuredDiscoveryFile, + params=self._build_unstructured_file_page_params(directory, status), + page_size=SERVER_MAX_PAGE_SIZE, + ) + + def get_unstructured_discovery_summary(self, run_id: RunId) -> UnstructuredDiscoverySummary: + """ + Returns an unstructured discovery run's totals. + + They are counted from the run's stored results on each request, + so they change while results are still being stored. + """ + + response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-discovery/summary/") + return UnstructuredDiscoverySummary.model_validate(response.json()) + + def get_unstructured_report(self, run_id: RunId) -> Union[str, bytes]: + """ + Returns an unstructured discovery run's findings, one row for each label found in each file. + + Returns CSV text (`str`) that starts with a byte-order mark, + or a zip of numbered CSV parts as `bytes` when the server splits a report of more than 1,000,000 rows. + The first row of each CSV names the columns. + A label's categories share one cell, joined with `;`. + """ + + response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-report/") + if response.headers.get("Content-Type", "").startswith("application/zip"): + return response.content + return response.text diff --git a/docs/client.models.rst b/docs/client.models.rst index 31d4585..37dcb1f 100644 --- a/docs/client.models.rst +++ b/docs/client.models.rst @@ -108,6 +108,14 @@ datamasque.client.models.status module :undoc-members: :show-inheritance: +datamasque.client.models.unstructured_discovery module +------------------------------------------------------ + +.. automodule:: datamasque.client.models.unstructured_discovery + :members: + :undoc-members: + :show-inheritance: + datamasque.client.models.user module ------------------------------------ diff --git a/docs/client.rst b/docs/client.rst index 8e7d73d..b3dc81d 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -132,6 +132,14 @@ datamasque.client.settings module :undoc-members: :show-inheritance: +datamasque.client.unstructured_discovery module +----------------------------------------------- + +.. automodule:: datamasque.client.unstructured_discovery + :members: + :undoc-members: + :show-inheritance: + datamasque.client.users module ------------------------------ diff --git a/tests/test_unstructured_discovery.py b/tests/test_unstructured_discovery.py new file mode 100644 index 0000000..43cd960 --- /dev/null +++ b/tests/test_unstructured_discovery.py @@ -0,0 +1,230 @@ +"""Tests for reading an unstructured discovery run's results.""" + +from collections.abc import Callable +from datetime import datetime, timezone +from typing import Any, Optional + +import pytest +import requests_mock + +from datamasque.client import DataMasqueClient +from datamasque.client.exceptions import DataMasqueApiError +from datamasque.client.models.runs import RunId +from datamasque.client.models.unstructured_discovery import ( + UnstructuredDiscoveryFileStatus, + UnstructuredDiscoverySampling, +) + +RESULTS_URL = "http://test-server/api/runs/41/unstructured-discovery/" +REPORT_URL = "http://test-server/api/runs/41/unstructured-report/" +CONNECTION_ID = "aaaaaaaa-1111-2222-3333-444444444444" + +ClientCall = Callable[[DataMasqueClient], object] + + +def _envelope(results: list[dict], **overrides: Any) -> dict: + return { + "run_id": 41, + "connection": {"id": CONNECTION_ID, "name": "invoices"}, + "finished_at": "2026-09-04T02:11:07Z", + "sampling": "sampled", + "files_per_directory": 3, + "count": len(results), + "next": None, + "previous": None, + "results": results, + **overrides, + } + + +def _folder(directory: str) -> dict: + return { + "directory": directory, + "files_present": 340, + "files_read": 20, + "size_bytes": 14399, + "entity_counts": {"CREDIT_CARD": 3, "EMAIL": 62}, + "unread": [{"skip_reason": "Not selected by the per-directory sample", "file_extension": "pdf", "count": 318}], + } + + +def _file(path: str, *, status: str = "read", skip_reason: str = "", findings: Optional[list[dict]] = None) -> dict: + return { + "path": path, + "file_extension": "pdf", + "size_bytes": 204817, + "status": status, + "skip_reason": skip_reason, + "findings": findings if findings is not None else [], + } + + +def test_get_unstructured_discovery_folder_page_parses_the_run_and_its_folders(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get(RESULTS_URL, json=_envelope([_folder("finance/invoices")])) + page = client.get_unstructured_discovery_folder_page(RunId(41), limit=10, offset=20) + + assert m.last_request.qs == {"limit": ["10"], "offset": ["20"]} + assert page.run_id == 41 + assert page.connection.id == CONNECTION_ID + assert page.connection.name == "invoices" + assert page.finished_at == datetime(2026, 9, 4, 2, 11, 7, tzinfo=timezone.utc) + assert page.sampling is UnstructuredDiscoverySampling.sampled + assert page.files_per_directory == 3 + assert page.count == 1 + + folder = page.results[0] + assert folder.directory == "finance/invoices" + assert folder.files_present == 340 + assert folder.files_read == 20 + assert folder.size_bytes == 14399 + assert folder.entity_counts == {"CREDIT_CARD": 3, "EMAIL": 62} + assert folder.unread[0].skip_reason == "Not selected by the per-directory sample" + assert folder.unread[0].file_extension == "pdf" + assert folder.unread[0].count == 318 + + +def test_get_unstructured_discovery_folder_page_asks_for_the_first_page_by_default(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get(RESULTS_URL, json=_envelope([], sampling="all", files_per_directory=None, finished_at=None)) + page = client.get_unstructured_discovery_folder_page(RunId(41)) + + assert m.last_request.qs == {"limit": ["50"], "offset": ["0"]} + assert page.results == [] + assert page.sampling is UnstructuredDiscoverySampling.all + assert page.files_per_directory is None + assert page.finished_at is None + + +def test_get_unstructured_discovery_file_page_parses_the_files_and_their_findings(client: DataMasqueClient) -> None: + finding = {"label": "EMAIL", "description": "Email address", "occurrences": 4, "categories": ["PII"]} + rows = [ + _file("Finance/Invoices/0142.pdf", findings=[finding]), + _file( + "Finance/Invoices/0143.pdf", status="unread", skip_reason="An error occurred while running data discovery" + ), + ] + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(RESULTS_URL, json=_envelope(rows)) + page = client.get_unstructured_discovery_file_page( + RunId(41), "Finance/Invoices", status=UnstructuredDiscoveryFileStatus.read, limit=1000, offset=5 + ) + + assert m.last_request.qs == { + "directory": ["Finance/Invoices"], + "status": ["read"], + "limit": ["1000"], + "offset": ["5"], + } + read_file, unread_file = page.results + assert read_file.path == "Finance/Invoices/0142.pdf" + assert read_file.file_extension == "pdf" + assert read_file.size_bytes == 204817 + assert read_file.status is UnstructuredDiscoveryFileStatus.read + assert read_file.skip_reason == "" + assert read_file.findings[0].label == "EMAIL" + assert read_file.findings[0].description == "Email address" + assert read_file.findings[0].occurrences == 4 + assert read_file.findings[0].categories == ["PII"] + assert unread_file.status is UnstructuredDiscoveryFileStatus.unread + assert unread_file.skip_reason == "An error occurred while running data discovery" + assert unread_file.findings == [] + + +def test_get_unstructured_discovery_file_page_sends_a_blank_directory_for_the_root(client: DataMasqueClient) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(RESULTS_URL, json=_envelope([_file("notes.pdf")])) + page = client.get_unstructured_discovery_file_page(RunId(41), "") + + assert m.last_request.qs == {"directory": [""], "limit": ["50"], "offset": ["0"]} + assert page.results[0].path == "notes.pdf" + + +def test_iter_unstructured_discovery_folders_follows_the_next_page(client: DataMasqueClient) -> None: + next_url = f"{RESULTS_URL}?limit=1000&offset=1000" + with requests_mock.Mocker() as m: + m.get(RESULTS_URL, [{"json": _envelope([_folder("a")], next=next_url)}, {"json": _envelope([_folder("b")])}]) + folders = list(client.iter_unstructured_discovery_folders(RunId(41))) + + assert [folder.directory for folder in folders] == ["a", "b"] + assert m.request_history[0].qs == {"limit": ["1000"], "offset": ["0"]} + assert m.request_history[1].url == next_url + + +@pytest.mark.parametrize( + "status, expected_status_query", + [(None, {}), (UnstructuredDiscoveryFileStatus.unread, {"status": ["unread"]})], +) +def test_iter_unstructured_discovery_files_asks_for_one_folder( + client: DataMasqueClient, + status: Optional[UnstructuredDiscoveryFileStatus], + expected_status_query: dict[str, list[str]], +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(RESULTS_URL, json=_envelope([_file("claims/1.pdf"), _file("claims/2.pdf")])) + files = list(client.iter_unstructured_discovery_files(RunId(41), "claims", status=status)) + + assert [file.path for file in files] == ["claims/1.pdf", "claims/2.pdf"] + assert m.last_request.qs == { + "directory": ["claims"], + "limit": ["1000"], + "offset": ["0"], + **expected_status_query, + } + + +def test_get_unstructured_discovery_summary(client: DataMasqueClient) -> None: + totals = {"run_id": 41, "files_found": 340, "files_read": 20, "files_with_findings": 14, "bytes_opened": 8123344} + with requests_mock.Mocker() as m: + m.get(f"{RESULTS_URL}summary/", json=totals) + summary = client.get_unstructured_discovery_summary(RunId(41)) + + assert summary.run_id == 41 + assert summary.files_found == 340 + assert summary.files_read == 20 + assert summary.files_with_findings == 14 + assert summary.bytes_opened == 8123344 + assert not summary.model_extra + + +def test_get_unstructured_report_returns_the_csv_text(client: DataMasqueClient) -> None: + csv_text = ( + "\ufeffpath,directory,label,description,occurrences,categories\r\n" + "réunions/0142.pdf,réunions,EMAIL,Email address,4,PII;Contact\r\n" + ) + with requests_mock.Mocker() as m: + m.get( + REPORT_URL, + content=csv_text.encode("utf-8"), + headers={"Content-Type": "text/csv; charset=utf-8"}, + ) + + assert client.get_unstructured_report(RunId(41)) == csv_text + + +def test_get_unstructured_report_returns_zip_bytes_when_split(client: DataMasqueClient) -> None: + zip_bytes = b"PK\x03\x04 split report zip bytes" + with requests_mock.Mocker() as m: + m.get(REPORT_URL, content=zip_bytes, headers={"Content-Type": "application/zip"}) + + assert client.get_unstructured_report(RunId(41)) == zip_bytes + + +@pytest.mark.parametrize( + "url, call", + [ + (RESULTS_URL, lambda client: client.get_unstructured_discovery_folder_page(RunId(41))), + (RESULTS_URL, lambda client: client.get_unstructured_discovery_file_page(RunId(41), "")), + (f"{RESULTS_URL}summary/", lambda client: client.get_unstructured_discovery_summary(RunId(41))), + (REPORT_URL, lambda client: client.get_unstructured_report(RunId(41))), + ], +) +def test_reading_a_run_that_is_not_unstructured_discovery_raises( + client: DataMasqueClient, url: str, call: ClientCall +) -> None: + with requests_mock.Mocker() as m: + m.get(url, json={"detail": "No Run matches the given query."}, status_code=404) + with pytest.raises(DataMasqueApiError) as exc_info: + call(client) + + assert exc_info.value.response.status_code == 404 From 326b92f3e0b654dd245174b14cd38731ad705ed2 Mon Sep 17 00:00:00 2001 From: Timotius Date: Wed, 4 Nov 2026 19:27:41 +1300 Subject: [PATCH 03/23] feat: DM-4756: find a connection's last unstructured discovery run --- HISTORY.rst | 3 ++ datamasque/client/unstructured_discovery.py | 14 ++++++++- tests/test_unstructured_discovery.py | 35 +++++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index c50dc5c..128e448 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -29,6 +29,9 @@ History or a zip of CSV parts as ``bytes`` for a report of more than 1,000,000 rows. Each page is an ``UnstructuredDiscoveryPage``, which also carries the run's connection, end time and ``UnstructuredDiscoverySampling``. +* Added ``get_last_unstructured_discovery_run_id``, + which returns the ID of a connection's newest finished unstructured discovery run, + or ``None`` when it has none. Requires server version 3.26.20 diff --git a/datamasque/client/unstructured_discovery.py b/datamasque/client/unstructured_discovery.py index e3bf79f..c6694e6 100644 --- a/datamasque/client/unstructured_discovery.py +++ b/datamasque/client/unstructured_discovery.py @@ -1,6 +1,7 @@ from typing import Iterator, Optional, Union from datamasque.client.base import BaseClient +from datamasque.client.models.connection import ConnectionId from datamasque.client.models.runs import RunId from datamasque.client.models.unstructured_discovery import ( UnstructuredDiscoveryFile, @@ -17,7 +18,7 @@ class UnstructuredDiscoveryClient(BaseClient): """ Unstructured discovery result API methods. Mixed into `DataMasqueClient`. - Each method raises `DataMasqueApiError` with a 404 response + Each method that takes a run ID raises `DataMasqueApiError` with a 404 response when the run is not an unstructured discovery run. """ @@ -31,6 +32,17 @@ def _build_unstructured_file_page_params( return params + def get_last_unstructured_discovery_run_id(self, connection_id: ConnectionId) -> Optional[RunId]: + """ + Returns the ID of a connection's newest finished unstructured discovery run, with or without warnings. + + Returns `None` when no unstructured discovery run on the connection has finished. + """ + + response = self.make_request("GET", f"/api/connections/{connection_id}/") + run_id = response.json()["last_unstructured_discovery_run_id"] + return RunId(run_id) if run_id is not None else None + def get_unstructured_discovery_folder_page( self, run_id: RunId, *, limit: int = 50, offset: int = 0 ) -> UnstructuredDiscoveryPage[UnstructuredDiscoveryFolder]: diff --git a/tests/test_unstructured_discovery.py b/tests/test_unstructured_discovery.py index 43cd960..244a186 100644 --- a/tests/test_unstructured_discovery.py +++ b/tests/test_unstructured_discovery.py @@ -9,6 +9,7 @@ from datamasque.client import DataMasqueClient from datamasque.client.exceptions import DataMasqueApiError +from datamasque.client.models.connection import ConnectionId from datamasque.client.models.runs import RunId from datamasque.client.models.unstructured_discovery import ( UnstructuredDiscoveryFileStatus, @@ -59,6 +60,40 @@ def _file(path: str, *, status: str = "read", skip_reason: str = "", findings: O } +@pytest.mark.parametrize( + "last_run_field, expected", + [ + ({"last_unstructured_discovery_run_id": 19}, RunId(19)), + ({"last_unstructured_discovery_run_id": None}, None), + ], +) +def test_get_last_unstructured_discovery_run_id_reads_the_connection( + client: DataMasqueClient, existing_connection_json: dict, last_run_field: dict, expected: Optional[RunId] +) -> None: + with requests_mock.Mocker() as m: + m.get( + f"http://test-server/api/connections/{CONNECTION_ID}/", + json=existing_connection_json | {"id": CONNECTION_ID} | last_run_field, + ) + + assert client.get_last_unstructured_discovery_run_id(ConnectionId(CONNECTION_ID)) == expected + + +def test_get_last_unstructured_discovery_run_id_raises_on_a_server_without_the_field( + client: DataMasqueClient, existing_connection_json: dict +) -> None: + with requests_mock.Mocker() as m: + m.get( + f"http://test-server/api/connections/{CONNECTION_ID}/", + json=existing_connection_json | {"id": CONNECTION_ID}, + ) + + with pytest.raises(KeyError) as exc_info: + client.get_last_unstructured_discovery_run_id(ConnectionId(CONNECTION_ID)) + + assert exc_info.value.args == ("last_unstructured_discovery_run_id",) + + def test_get_unstructured_discovery_folder_page_parses_the_run_and_its_folders(client: DataMasqueClient) -> None: with requests_mock.Mocker() as m: m.get(RESULTS_URL, json=_envelope([_folder("finance/invoices")])) From 2400c120a9b07c3645a679fa1ad15a683f52165b Mon Sep 17 00:00:00 2001 From: Timotius Date: Sat, 7 Nov 2026 18:59:15 +1300 Subject: [PATCH 04/23] feat: DM-4756: read a run's AI Engine model and version --- HISTORY.rst | 3 +++ datamasque/client/models/runs.py | 10 ++++++++++ tests/test_runs.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 128e448..27ab28b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -32,6 +32,9 @@ History * Added ``get_last_unstructured_discovery_run_id``, which returns the ID of a connection's newest finished unstructured discovery run, or ``None`` when it has none. +* Added ``ai_engine_model`` and ``ai_engine_version`` to ``RunInfo``: + the AI Engine model and engine version the run's ``ai_detect`` matchers use, + blank for a run that uses no AI Engine, and ``None`` on a server that does not report them. Requires server version 3.26.20 diff --git a/datamasque/client/models/runs.py b/datamasque/client/models/runs.py index 09bae55..631cf19 100644 --- a/datamasque/client/models/runs.py +++ b/datamasque/client/models/runs.py @@ -148,6 +148,16 @@ class RunInfo(BaseModel): Taken from the ruleset's project when the run starts, and never changes after that. """ + ai_engine_model: Optional[str] = None + """ + The AI Engine model the run's `ai_detect` matchers use, as the engine reports it. + Blank for a run that uses no AI Engine, and `None` on a server that does not report it. + """ + ai_engine_version: Optional[str] = None + """ + The version of the AI Engine the run's `ai_detect` matchers use, as the engine reports it. + Blank for a run that uses no AI Engine, and `None` on a server that does not report it. + """ @model_validator(mode="before") @classmethod diff --git a/tests/test_runs.py b/tests/test_runs.py index 295409a..ed11f68 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -1,5 +1,7 @@ """Tests for `RunClient` (start, status, log, run-report endpoints).""" +from typing import Optional + import pytest import requests_mock @@ -340,6 +342,35 @@ def test_get_run_info(client): assert result.name == "r1" +@pytest.mark.parametrize( + "ai_engine_fields, expected_model, expected_version", + [ + ({"ai_engine_model": "base", "ai_engine_version": "1.26.11"}, "base", "1.26.11"), + ({"ai_engine_model": "", "ai_engine_version": ""}, "", ""), + ({}, None, None), + ], + ids=["ai_detect_run", "run_without_ai_detect", "server_without_the_fields"], +) +def test_run_info_reads_the_ai_engine_model_and_version( + ai_engine_fields: dict, expected_model: Optional[str], expected_version: Optional[str] +) -> None: + run = RunInfo.model_validate( + { + "id": 1, + "status": "finished", + "mask_type": "file_data_discovery", + "source_connection_name": "documents", + "ruleset_name": "", + "project": "1-default", + **ai_engine_fields, + } + ) + + assert run.ai_engine_model == expected_model + assert run.ai_engine_version == expected_version + assert not run.model_extra + + def test_masking_run_request_model_dump_minimal(): """A minimal request dumps with an empty `options` object (the server rejects missing `options`).""" req = MaskingRunRequest(connection="conn-1", ruleset="rs-1") From a249b99633a8c89df97767fe3789c2ad9816c8a2 Mon Sep 17 00:00:00 2001 From: Timotius Date: Sat, 7 Nov 2026 19:00:01 +1300 Subject: [PATCH 05/23] feat: DM-4756: read per-label totals and character counts --- HISTORY.rst | 2 +- datamasque/client/__init__.py | 2 + .../client/models/unstructured_discovery.py | 25 +++++++++ datamasque/client/unstructured_discovery.py | 6 +- tests/test_unstructured_discovery.py | 56 +++++++++++++++++-- 5 files changed, 83 insertions(+), 8 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 27ab28b..4cf7fe8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -24,7 +24,7 @@ History for the folders found on the connection, ``get_unstructured_discovery_file_page`` and ``iter_unstructured_discovery_files`` for the files of one folder, optionally narrowed by ``UnstructuredDiscoveryFileStatus``, - ``get_unstructured_discovery_summary`` for its totals, + ``get_unstructured_discovery_summary`` for its totals and how many times it found each label, and ``get_unstructured_report`` for its findings CSV, or a zip of CSV parts as ``bytes`` for a report of more than 1,000,000 rows. Each page is an ``UnstructuredDiscoveryPage``, diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index 603203a..e15b7f3 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -182,6 +182,7 @@ UnstructuredDiscoveryFileStatus, UnstructuredDiscoveryFinding, UnstructuredDiscoveryFolder, + UnstructuredDiscoveryLabelTotal, UnstructuredDiscoveryPage, UnstructuredDiscoverySampling, UnstructuredDiscoverySummary, @@ -343,6 +344,7 @@ "UnstructuredDiscoveryFileStatus", "UnstructuredDiscoveryFinding", "UnstructuredDiscoveryFolder", + "UnstructuredDiscoveryLabelTotal", "UnstructuredDiscoveryPage", "UnstructuredDiscoverySampling", "UnstructuredDiscoverySummary", diff --git a/datamasque/client/models/unstructured_discovery.py b/datamasque/client/models/unstructured_discovery.py index be11b83..a533f94 100644 --- a/datamasque/client/models/unstructured_discovery.py +++ b/datamasque/client/models/unstructured_discovery.py @@ -81,6 +81,8 @@ class UnstructuredDiscoveryFile(BaseModel): path: str file_extension: str size_bytes: int + character_count: Optional[int] = None + """How many characters of the file's text the run searched, `None` where the run reported none.""" status: UnstructuredDiscoveryFileStatus skip_reason: str """Why the run could not read the file, blank if it did.""" @@ -104,6 +106,21 @@ class UnstructuredDiscoveryPage(Page[T], Generic[T]): """The sample size when `sampling` is `UnstructuredDiscoverySampling.sampled`, `None` otherwise.""" +class UnstructuredDiscoveryLabelTotal(BaseModel): + """How many times one label was found over an unstructured discovery run's files.""" + + model_config = ConfigDict(extra="allow") + + label: str + """The label's `name` in the discovery config.""" + description: str + categories: list[str] + occurrences: int + """How many times the label was found, summed over the files.""" + files: int + """How many files hold the label.""" + + class UnstructuredDiscoverySummary(BaseModel): """An unstructured discovery run's totals.""" @@ -113,7 +130,15 @@ class UnstructuredDiscoverySummary(BaseModel): files_found: int """How many files were found on the connection, summed over the folders.""" files_read: int + files_failed: int + """How many files the run tried to read and could not.""" + files_without_text: int + """How many of the files the run read held no text to search, as a scanned document may not.""" files_with_findings: int """How many of the files the run read hold a finding.""" bytes_opened: int """The total size, in bytes, of the files the run read.""" + files_over_run_limit: int + """How many files the run did not read because it reached the config's `files_per_run`.""" + labels: list[UnstructuredDiscoveryLabelTotal] + """Each label found, most found first; labels found equally often are listed by name.""" diff --git a/datamasque/client/unstructured_discovery.py b/datamasque/client/unstructured_discovery.py index c6694e6..5357b6b 100644 --- a/datamasque/client/unstructured_discovery.py +++ b/datamasque/client/unstructured_discovery.py @@ -111,7 +111,7 @@ def iter_unstructured_discovery_files( def get_unstructured_discovery_summary(self, run_id: RunId) -> UnstructuredDiscoverySummary: """ - Returns an unstructured discovery run's totals. + Returns an unstructured discovery run's totals, and how many times it found each label. They are counted from the run's stored results on each request, so they change while results are still being stored. @@ -124,6 +124,10 @@ def get_unstructured_report(self, run_id: RunId) -> Union[str, bytes]: """ Returns an unstructured discovery run's findings, one row for each label found in each file. + A file the run read and found nothing in has one row, + with a blank label, description and categories, and 0 occurrences. + A file the run could not read has no row. + Every row carries its file's `character_count`, blank where the run reported none. Returns CSV text (`str`) that starts with a byte-order mark, or a zip of numbered CSV parts as `bytes` when the server splits a report of more than 1,000,000 rows. The first row of each CSV names the columns. diff --git a/tests/test_unstructured_discovery.py b/tests/test_unstructured_discovery.py index 244a186..159e682 100644 --- a/tests/test_unstructured_discovery.py +++ b/tests/test_unstructured_discovery.py @@ -49,11 +49,19 @@ def _folder(directory: str) -> dict: } -def _file(path: str, *, status: str = "read", skip_reason: str = "", findings: Optional[list[dict]] = None) -> dict: +def _file( + path: str, + *, + status: str = "read", + skip_reason: str = "", + character_count: Optional[int] = 5120, + findings: Optional[list[dict]] = None, +) -> dict: return { "path": path, "file_extension": "pdf", "size_bytes": 204817, + "character_count": character_count, "status": status, "skip_reason": skip_reason, "findings": findings if findings is not None else [], @@ -136,7 +144,10 @@ def test_get_unstructured_discovery_file_page_parses_the_files_and_their_finding rows = [ _file("Finance/Invoices/0142.pdf", findings=[finding]), _file( - "Finance/Invoices/0143.pdf", status="unread", skip_reason="An error occurred while running data discovery" + "Finance/Invoices/0143.pdf", + status="unread", + skip_reason="An error occurred while running data discovery", + character_count=None, ), ] with requests_mock.Mocker(case_sensitive=True) as m: @@ -155,6 +166,7 @@ def test_get_unstructured_discovery_file_page_parses_the_files_and_their_finding assert read_file.path == "Finance/Invoices/0142.pdf" assert read_file.file_extension == "pdf" assert read_file.size_bytes == 204817 + assert read_file.character_count == 5120 assert read_file.status is UnstructuredDiscoveryFileStatus.read assert read_file.skip_reason == "" assert read_file.findings[0].label == "EMAIL" @@ -163,7 +175,10 @@ def test_get_unstructured_discovery_file_page_parses_the_files_and_their_finding assert read_file.findings[0].categories == ["PII"] assert unread_file.status is UnstructuredDiscoveryFileStatus.unread assert unread_file.skip_reason == "An error occurred while running data discovery" + assert unread_file.character_count is None assert unread_file.findings == [] + assert not read_file.model_extra + assert not unread_file.model_extra def test_get_unstructured_discovery_file_page_sends_a_blank_directory_for_the_root(client: DataMasqueClient) -> None: @@ -208,8 +223,26 @@ def test_iter_unstructured_discovery_files_asks_for_one_folder( } -def test_get_unstructured_discovery_summary(client: DataMasqueClient) -> None: - totals = {"run_id": 41, "files_found": 340, "files_read": 20, "files_with_findings": 14, "bytes_opened": 8123344} +def test_get_unstructured_discovery_summary_parses_the_totals_and_each_label(client: DataMasqueClient) -> None: + totals = { + "run_id": 41, + "files_found": 340, + "files_read": 20, + "files_failed": 2, + "files_without_text": 3, + "files_with_findings": 14, + "bytes_opened": 8123344, + "files_over_run_limit": 7, + "labels": [ + { + "label": "EMAIL", + "description": "Email address", + "categories": ["PII", "Contact"], + "occurrences": 62, + "files": 11, + } + ], + } with requests_mock.Mocker() as m: m.get(f"{RESULTS_URL}summary/", json=totals) summary = client.get_unstructured_discovery_summary(RunId(41)) @@ -217,15 +250,26 @@ def test_get_unstructured_discovery_summary(client: DataMasqueClient) -> None: assert summary.run_id == 41 assert summary.files_found == 340 assert summary.files_read == 20 + assert summary.files_failed == 2 + assert summary.files_without_text == 3 assert summary.files_with_findings == 14 assert summary.bytes_opened == 8123344 + assert summary.files_over_run_limit == 7 + label_total = summary.labels[0] + assert label_total.label == "EMAIL" + assert label_total.description == "Email address" + assert label_total.categories == ["PII", "Contact"] + assert label_total.occurrences == 62 + assert label_total.files == 11 assert not summary.model_extra + assert not label_total.model_extra def test_get_unstructured_report_returns_the_csv_text(client: DataMasqueClient) -> None: csv_text = ( - "\ufeffpath,directory,label,description,occurrences,categories\r\n" - "réunions/0142.pdf,réunions,EMAIL,Email address,4,PII;Contact\r\n" + "\ufeffpath,directory,label,description,occurrences,categories,character_count\r\n" + "réunions/0142.pdf,réunions,EMAIL,Email address,4,PII;Contact,5120\r\n" + "réunions/0143.pdf,réunions,,,0,,96\r\n" ) with requests_mock.Mocker() as m: m.get( From e1808f49153583e34af35c3fb766c6befa6b881c Mon Sep 17 00:00:00 2001 From: Timotius Date: Sat, 7 Nov 2026 19:02:13 +1300 Subject: [PATCH 06/23] feat: DM-4756: filter, search and sort folders and files --- HISTORY.rst | 5 +- datamasque/client/__init__.py | 4 + .../client/models/unstructured_discovery.py | 28 ++++ datamasque/client/unstructured_discovery.py | 129 +++++++++++++++--- tests/test_unstructured_discovery.py | 100 +++++++++++++- 5 files changed, 246 insertions(+), 20 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4cf7fe8..87be304 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -22,8 +22,11 @@ History * Added methods that read an unstructured discovery run's results: ``get_unstructured_discovery_folder_page`` and ``iter_unstructured_discovery_folders`` for the folders found on the connection, + optionally narrowed by ``search``, ``labels`` and ``categories`` + and sorted by ``UnstructuredDiscoveryFolderOrdering``, ``get_unstructured_discovery_file_page`` and ``iter_unstructured_discovery_files`` for the files of one folder, - optionally narrowed by ``UnstructuredDiscoveryFileStatus``, + optionally narrowed by ``UnstructuredDiscoveryFileStatus``, ``search``, ``labels``, ``categories`` + and ``UnstructuredDiscoveryFileOutcome``, ``get_unstructured_discovery_summary`` for its totals and how many times it found each label, and ``get_unstructured_report`` for its findings CSV, or a zip of CSV parts as ``bytes`` for a report of more than 1,000,000 rows. diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index e15b7f3..74f98ce 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -179,9 +179,11 @@ ) from datamasque.client.models.unstructured_discovery import ( UnstructuredDiscoveryFile, + UnstructuredDiscoveryFileOutcome, UnstructuredDiscoveryFileStatus, UnstructuredDiscoveryFinding, UnstructuredDiscoveryFolder, + UnstructuredDiscoveryFolderOrdering, UnstructuredDiscoveryLabelTotal, UnstructuredDiscoveryPage, UnstructuredDiscoverySampling, @@ -341,9 +343,11 @@ "TemporalSummaries", "UnfinishedRun", "UnstructuredDiscoveryFile", + "UnstructuredDiscoveryFileOutcome", "UnstructuredDiscoveryFileStatus", "UnstructuredDiscoveryFinding", "UnstructuredDiscoveryFolder", + "UnstructuredDiscoveryFolderOrdering", "UnstructuredDiscoveryLabelTotal", "UnstructuredDiscoveryPage", "UnstructuredDiscoverySampling", diff --git a/datamasque/client/models/unstructured_discovery.py b/datamasque/client/models/unstructured_discovery.py index a533f94..b55082b 100644 --- a/datamasque/client/models/unstructured_discovery.py +++ b/datamasque/client/models/unstructured_discovery.py @@ -20,6 +20,34 @@ class UnstructuredDiscoveryFileStatus(enum.Enum): """The run tried to read the file and could not; `skip_reason` says why.""" +class UnstructuredDiscoveryFileOutcome(enum.Enum): + """What an unstructured discovery run made of a file it tried to read.""" + + findings = "findings" + """The run found at least one label in the file.""" + no_text = "no_text" + """The run read the file and found no text in it, as in a scanned document.""" + nothing_found = "nothing_found" + """The run read the file's text and found no label in it.""" + failed = "failed" + """The run tried to read the file and could not.""" + + +class UnstructuredDiscoveryFolderOrdering(enum.Enum): + """A column the folders of an unstructured discovery run are sorted by; folders that tie are sorted by path.""" + + directory = "directory" + """The folder's path.""" + files_present = "files_present" + files_read = "files_read" + size_bytes = "size_bytes" + occurrences = "occurrences" + """ + The sum of the folder's `entity_counts`. + With a `labels` or `categories` filter, only the matching labels count. + """ + + class UnstructuredDiscoverySampling(enum.Enum): """How many files of each folder an unstructured discovery run read.""" diff --git a/datamasque/client/unstructured_discovery.py b/datamasque/client/unstructured_discovery.py index 5357b6b..29ce137 100644 --- a/datamasque/client/unstructured_discovery.py +++ b/datamasque/client/unstructured_discovery.py @@ -5,13 +5,17 @@ from datamasque.client.models.runs import RunId from datamasque.client.models.unstructured_discovery import ( UnstructuredDiscoveryFile, + UnstructuredDiscoveryFileOutcome, UnstructuredDiscoveryFileStatus, UnstructuredDiscoveryFolder, + UnstructuredDiscoveryFolderOrdering, UnstructuredDiscoveryPage, UnstructuredDiscoverySummary, ) SERVER_MAX_PAGE_SIZE = 1000 +LIST_SEPARATOR = "," +DESCENDING_PREFIX = "-" class UnstructuredDiscoveryClient(BaseClient): @@ -22,13 +26,47 @@ class UnstructuredDiscoveryClient(BaseClient): when the run is not an unstructured discovery run. """ + @staticmethod + def _build_label_filter_params( + search: Optional[str], labels: Optional[list[str]], categories: Optional[list[str]] + ) -> dict[str, str]: + params = {} + if search is not None: + params["search"] = search + if labels is not None: + params["labels"] = LIST_SEPARATOR.join(labels) + if categories is not None: + params["categories"] = LIST_SEPARATOR.join(categories) + + return params + + @staticmethod + def _build_unstructured_folder_page_params( + search: Optional[str], + labels: Optional[list[str]], + categories: Optional[list[str]], + ordering: UnstructuredDiscoveryFolderOrdering, + is_descending: bool, + ) -> dict[str, str]: + params = UnstructuredDiscoveryClient._build_label_filter_params(search, labels, categories) + params["ordering"] = f"{DESCENDING_PREFIX}{ordering.value}" if is_descending else ordering.value + return params + @staticmethod def _build_unstructured_file_page_params( - directory: str, status: Optional[UnstructuredDiscoveryFileStatus] + directory: str, + status: Optional[UnstructuredDiscoveryFileStatus], + search: Optional[str], + labels: Optional[list[str]], + categories: Optional[list[str]], + outcomes: Optional[list[UnstructuredDiscoveryFileOutcome]], ) -> dict[str, str]: params = {"directory": directory} if status is not None: params["status"] = status.value + params |= UnstructuredDiscoveryClient._build_label_filter_params(search, labels, categories) + if outcomes is not None: + params["outcome"] = LIST_SEPARATOR.join(outcome.value for outcome in outcomes) return params @@ -44,25 +82,62 @@ def get_last_unstructured_discovery_run_id(self, connection_id: ConnectionId) -> return RunId(run_id) if run_id is not None else None def get_unstructured_discovery_folder_page( - self, run_id: RunId, *, limit: int = 50, offset: int = 0 + self, + run_id: RunId, + *, + search: Optional[str] = None, + labels: Optional[list[str]] = None, + categories: Optional[list[str]] = None, + ordering: UnstructuredDiscoveryFolderOrdering = UnstructuredDiscoveryFolderOrdering.directory, + descending: bool = False, + limit: int = 50, + offset: int = 0, ) -> UnstructuredDiscoveryPage[UnstructuredDiscoveryFolder]: - """Returns one page of the folders an unstructured discovery run found on the connection, ordered by path.""" + """ + Returns one page of the folders an unstructured discovery run found on the connection. + + `search` keeps the folders whose path contains it, ignoring case, + and the folders holding a file whose name contains it. + `labels` keeps the folders holding any of those labels, + and `categories` the folders holding a label that carries any of those categories; + given both, a label must match both. + With `labels` or `categories`, a file matches `search` only if it holds a matching label, + or the run read no text from it or could not read it. + Each folder's `entity_counts` still counts every label it holds. + The folders come in `ordering`, descending when `descending` is set, + and folders that tie are sorted by path, ascending. + """ - response = self.make_request( - "GET", - f"/api/runs/{run_id}/unstructured-discovery/", - params={"limit": limit, "offset": offset}, - ) + params = self._build_unstructured_folder_page_params(search, labels, categories, ordering, descending) | { + "limit": limit, + "offset": offset, + } + response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-discovery/", params=params) return UnstructuredDiscoveryPage[UnstructuredDiscoveryFolder].model_validate(response.json()) # `iter_unstructured_discovery_folders` and `iter_unstructured_discovery_files` have no `list_` counterparts, # because a run's folder and file counts are unbounded. - def iter_unstructured_discovery_folders(self, run_id: RunId) -> Iterator[UnstructuredDiscoveryFolder]: - """Lazily iterate every folder an unstructured discovery run found on the connection, ordered by path.""" + def iter_unstructured_discovery_folders( + self, + run_id: RunId, + *, + search: Optional[str] = None, + labels: Optional[list[str]] = None, + categories: Optional[list[str]] = None, + ordering: UnstructuredDiscoveryFolderOrdering = UnstructuredDiscoveryFolderOrdering.directory, + descending: bool = False, + ) -> Iterator[UnstructuredDiscoveryFolder]: + """ + Lazily iterate every folder an unstructured discovery run found on the connection. + + `search`, `labels`, `categories`, `ordering` and `descending` + are as for `get_unstructured_discovery_folder_page`. + """ return self._iter_paginated( f"/api/runs/{run_id}/unstructured-discovery/", model=UnstructuredDiscoveryFolder, + params=self._build_unstructured_folder_page_params(search, labels, categories, ordering, descending), page_size=SERVER_MAX_PAGE_SIZE, ) @@ -72,20 +147,37 @@ def get_unstructured_discovery_file_page( directory: str, *, status: Optional[UnstructuredDiscoveryFileStatus] = None, + search: Optional[str] = None, + labels: Optional[list[str]] = None, + categories: Optional[list[str]] = None, + outcomes: Optional[list[UnstructuredDiscoveryFileOutcome]] = None, limit: int = 50, offset: int = 0, ) -> UnstructuredDiscoveryPage[UnstructuredDiscoveryFile]: """ - Returns one page of the files an unstructured discovery run tried to read in one folder, ordered by name. + Returns one page of the files an unstructured discovery run tried to read in one folder. `directory` is a folder's `directory` exactly as the folder pages name it; an empty string names the connection root. - `status` keeps only the files with that status. + `status` keeps only the files with that status, + and `outcomes` only the files with any of those outcomes. + `search` keeps the files whose name contains it, ignoring case. + `labels` keeps the files holding any of those labels, + and `categories` the files holding a label that carries any of those categories; + given both, a label must match both. + Either one also keeps the files the run read and found no text in, which may hold any label. + The files with findings come first, most occurrences first, + then the files with no text, those with nothing found and those the run could not read, + each group sorted by name. + With `labels` or `categories`, a file's occurrences count only the matching labels. Files the run did not try to read have no row; the folder's `unread` counts them. """ - params = self._build_unstructured_file_page_params(directory, status) | {"limit": limit, "offset": offset} + params = self._build_unstructured_file_page_params(directory, status, search, labels, categories, outcomes) | { + "limit": limit, + "offset": offset, + } response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-discovery/", params=params) return UnstructuredDiscoveryPage[UnstructuredDiscoveryFile].model_validate(response.json()) @@ -95,17 +187,22 @@ def iter_unstructured_discovery_files( directory: str, *, status: Optional[UnstructuredDiscoveryFileStatus] = None, + search: Optional[str] = None, + labels: Optional[list[str]] = None, + categories: Optional[list[str]] = None, + outcomes: Optional[list[UnstructuredDiscoveryFileOutcome]] = None, ) -> Iterator[UnstructuredDiscoveryFile]: """ - Lazily iterate every file an unstructured discovery run tried to read in one folder, ordered by name. + Lazily iterate every file an unstructured discovery run tried to read in one folder. - `directory` and `status` are as for `get_unstructured_discovery_file_page`. + `directory`, `status`, `search`, `labels`, `categories` and `outcomes` + are as for `get_unstructured_discovery_file_page`, and so is the order of the files. """ return self._iter_paginated( f"/api/runs/{run_id}/unstructured-discovery/", model=UnstructuredDiscoveryFile, - params=self._build_unstructured_file_page_params(directory, status), + params=self._build_unstructured_file_page_params(directory, status, search, labels, categories, outcomes), page_size=SERVER_MAX_PAGE_SIZE, ) diff --git a/tests/test_unstructured_discovery.py b/tests/test_unstructured_discovery.py index 159e682..be217f8 100644 --- a/tests/test_unstructured_discovery.py +++ b/tests/test_unstructured_discovery.py @@ -12,7 +12,9 @@ from datamasque.client.models.connection import ConnectionId from datamasque.client.models.runs import RunId from datamasque.client.models.unstructured_discovery import ( + UnstructuredDiscoveryFileOutcome, UnstructuredDiscoveryFileStatus, + UnstructuredDiscoveryFolderOrdering, UnstructuredDiscoverySampling, ) @@ -107,7 +109,7 @@ def test_get_unstructured_discovery_folder_page_parses_the_run_and_its_folders(c m.get(RESULTS_URL, json=_envelope([_folder("finance/invoices")])) page = client.get_unstructured_discovery_folder_page(RunId(41), limit=10, offset=20) - assert m.last_request.qs == {"limit": ["10"], "offset": ["20"]} + assert m.last_request.qs == {"ordering": ["directory"], "limit": ["10"], "offset": ["20"]} assert page.run_id == 41 assert page.connection.id == CONNECTION_ID assert page.connection.name == "invoices" @@ -132,13 +134,70 @@ def test_get_unstructured_discovery_folder_page_asks_for_the_first_page_by_defau m.get(RESULTS_URL, json=_envelope([], sampling="all", files_per_directory=None, finished_at=None)) page = client.get_unstructured_discovery_folder_page(RunId(41)) - assert m.last_request.qs == {"limit": ["50"], "offset": ["0"]} + assert m.last_request.qs == {"ordering": ["directory"], "limit": ["50"], "offset": ["0"]} assert page.results == [] assert page.sampling is UnstructuredDiscoverySampling.all assert page.files_per_directory is None assert page.finished_at is None +@pytest.mark.parametrize( + "call, expected_limit", + [ + (lambda client, **filters: client.get_unstructured_discovery_folder_page(RunId(41), **filters), "50"), + (lambda client, **filters: list(client.iter_unstructured_discovery_folders(RunId(41), **filters)), "1000"), + ], + ids=["page", "iterator"], +) +def test_unstructured_discovery_folders_send_the_search_and_label_filters( + client: DataMasqueClient, call: Callable[..., object], expected_limit: str +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(RESULTS_URL, json=_envelope([_folder("Finance/Invoices")])) + call(client, search="Invoices", labels=["EMAIL", "CREDIT_CARD"], categories=["PII", "Financial"]) + + assert m.last_request.qs == { + "search": ["Invoices"], + "labels": ["EMAIL,CREDIT_CARD"], + "categories": ["PII,Financial"], + "ordering": ["directory"], + "limit": [expected_limit], + "offset": ["0"], + } + + +@pytest.mark.parametrize( + "ordering, descending, expected_ordering", + [ + (UnstructuredDiscoveryFolderOrdering.directory, True, "-directory"), + (UnstructuredDiscoveryFolderOrdering.files_present, False, "files_present"), + (UnstructuredDiscoveryFolderOrdering.files_read, True, "-files_read"), + (UnstructuredDiscoveryFolderOrdering.size_bytes, False, "size_bytes"), + (UnstructuredDiscoveryFolderOrdering.occurrences, True, "-occurrences"), + ], +) +@pytest.mark.parametrize( + "call", + [ + lambda client, **ordering: client.get_unstructured_discovery_folder_page(RunId(41), **ordering), + lambda client, **ordering: list(client.iter_unstructured_discovery_folders(RunId(41), **ordering)), + ], + ids=["page", "iterator"], +) +def test_unstructured_discovery_folders_send_the_ordering( + client: DataMasqueClient, + call: Callable[..., object], + ordering: UnstructuredDiscoveryFolderOrdering, + descending: bool, + expected_ordering: str, +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(RESULTS_URL, json=_envelope([_folder("Finance/Invoices")])) + call(client, ordering=ordering, descending=descending) + + assert m.last_request.qs["ordering"] == [expected_ordering] + + def test_get_unstructured_discovery_file_page_parses_the_files_and_their_findings(client: DataMasqueClient) -> None: finding = {"label": "EMAIL", "description": "Email address", "occurrences": 4, "categories": ["PII"]} rows = [ @@ -197,7 +256,7 @@ def test_iter_unstructured_discovery_folders_follows_the_next_page(client: DataM folders = list(client.iter_unstructured_discovery_folders(RunId(41))) assert [folder.directory for folder in folders] == ["a", "b"] - assert m.request_history[0].qs == {"limit": ["1000"], "offset": ["0"]} + assert m.request_history[0].qs == {"ordering": ["directory"], "limit": ["1000"], "offset": ["0"]} assert m.request_history[1].url == next_url @@ -223,6 +282,41 @@ def test_iter_unstructured_discovery_files_asks_for_one_folder( } +@pytest.mark.parametrize( + "call, expected_limit", + [ + (lambda client, **filters: client.get_unstructured_discovery_file_page(RunId(41), "claims", **filters), "50"), + ( + lambda client, **filters: list(client.iter_unstructured_discovery_files(RunId(41), "claims", **filters)), + "1000", + ), + ], + ids=["page", "iterator"], +) +def test_unstructured_discovery_files_send_the_search_label_and_outcome_filters( + client: DataMasqueClient, call: Callable[..., object], expected_limit: str +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(RESULTS_URL, json=_envelope([_file("claims/Claim-1.pdf")])) + call( + client, + search="Claim", + labels=["EMAIL", "CREDIT_CARD"], + categories=["PII", "Financial"], + outcomes=[UnstructuredDiscoveryFileOutcome.no_text, UnstructuredDiscoveryFileOutcome.failed], + ) + + assert m.last_request.qs == { + "directory": ["claims"], + "search": ["Claim"], + "labels": ["EMAIL,CREDIT_CARD"], + "categories": ["PII,Financial"], + "outcome": ["no_text,failed"], + "limit": [expected_limit], + "offset": ["0"], + } + + def test_get_unstructured_discovery_summary_parses_the_totals_and_each_label(client: DataMasqueClient) -> None: totals = { "run_id": 41, From 6aed8f4c9575d23ac5a03d8d00bb06bb3c3dfa90 Mon Sep 17 00:00:00 2001 From: Timotius Date: Sat, 7 Nov 2026 20:53:53 +1300 Subject: [PATCH 07/23] docs: DM-4756: describe the AI Engine run fields as the API does --- HISTORY.rst | 3 ++- datamasque/client/models/runs.py | 12 ++++++++---- tests/test_runs.py | 6 +++++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 87be304..c604ee5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -37,7 +37,8 @@ History or ``None`` when it has none. * Added ``ai_engine_model`` and ``ai_engine_version`` to ``RunInfo``: the AI Engine model and engine version the run's ``ai_detect`` matchers use, - blank for a run that uses no AI Engine, and ``None`` on a server that does not report them. + blank when the run recorded none, such as a run that uses no AI Engine, + and ``None`` on a server that does not report them. Requires server version 3.26.20 diff --git a/datamasque/client/models/runs.py b/datamasque/client/models/runs.py index 631cf19..a220054 100644 --- a/datamasque/client/models/runs.py +++ b/datamasque/client/models/runs.py @@ -150,13 +150,17 @@ class RunInfo(BaseModel): """ ai_engine_model: Optional[str] = None """ - The AI Engine model the run's `ai_detect` matchers use, as the engine reports it. - Blank for a run that uses no AI Engine, and `None` on a server that does not report it. + The AI Engine model the run's `ai_detect` matchers use: + for a Local AI Engine, the name of its model file or folder, + and for a Bedrock AI Engine, the Bedrock model set in its configuration. + Blank when the run recorded none, such as a run that uses no AI Engine, + and `None` on a server that does not report it. """ ai_engine_version: Optional[str] = None """ - The version of the AI Engine the run's `ai_detect` matchers use, as the engine reports it. - Blank for a run that uses no AI Engine, and `None` on a server that does not report it. + The version the AI Engine reported when the run started. + Blank when the run recorded none, such as a run that uses no AI Engine, + and `None` on a server that does not report it. """ @model_validator(mode="before") diff --git a/tests/test_runs.py b/tests/test_runs.py index ed11f68..db45d85 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -345,7 +345,11 @@ def test_get_run_info(client): @pytest.mark.parametrize( "ai_engine_fields, expected_model, expected_version", [ - ({"ai_engine_model": "base", "ai_engine_version": "1.26.11"}, "base", "1.26.11"), + ( + {"ai_engine_model": "dm-ai-ner-small-1.0.0-20260601.tar.enc", "ai_engine_version": "1.26.11"}, + "dm-ai-ner-small-1.0.0-20260601.tar.enc", + "1.26.11", + ), ({"ai_engine_model": "", "ai_engine_version": ""}, "", ""), ({}, None, None), ], From c67b15747b2c7900d50b615f13322efa68d975a6 Mon Sep 17 00:00:00 2001 From: Timotius Date: Sun, 8 Nov 2026 01:48:01 +1300 Subject: [PATCH 08/23] feat: DM-4756: call a file the run could not read failed --- .../client/models/unstructured_discovery.py | 2 +- tests/test_unstructured_discovery.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/datamasque/client/models/unstructured_discovery.py b/datamasque/client/models/unstructured_discovery.py index b55082b..9f2172b 100644 --- a/datamasque/client/models/unstructured_discovery.py +++ b/datamasque/client/models/unstructured_discovery.py @@ -16,7 +16,7 @@ class UnstructuredDiscoveryFileStatus(enum.Enum): """Whether an unstructured discovery run read a file's text.""" read = "read" - unread = "unread" + failed = "failed" """The run tried to read the file and could not; `skip_reason` says why.""" diff --git a/tests/test_unstructured_discovery.py b/tests/test_unstructured_discovery.py index be217f8..b4ad5c8 100644 --- a/tests/test_unstructured_discovery.py +++ b/tests/test_unstructured_discovery.py @@ -204,7 +204,7 @@ def test_get_unstructured_discovery_file_page_parses_the_files_and_their_finding _file("Finance/Invoices/0142.pdf", findings=[finding]), _file( "Finance/Invoices/0143.pdf", - status="unread", + status="failed", skip_reason="An error occurred while running data discovery", character_count=None, ), @@ -221,7 +221,7 @@ def test_get_unstructured_discovery_file_page_parses_the_files_and_their_finding "limit": ["1000"], "offset": ["5"], } - read_file, unread_file = page.results + read_file, failed_file = page.results assert read_file.path == "Finance/Invoices/0142.pdf" assert read_file.file_extension == "pdf" assert read_file.size_bytes == 204817 @@ -232,12 +232,12 @@ def test_get_unstructured_discovery_file_page_parses_the_files_and_their_finding assert read_file.findings[0].description == "Email address" assert read_file.findings[0].occurrences == 4 assert read_file.findings[0].categories == ["PII"] - assert unread_file.status is UnstructuredDiscoveryFileStatus.unread - assert unread_file.skip_reason == "An error occurred while running data discovery" - assert unread_file.character_count is None - assert unread_file.findings == [] + assert failed_file.status is UnstructuredDiscoveryFileStatus.failed + assert failed_file.skip_reason == "An error occurred while running data discovery" + assert failed_file.character_count is None + assert failed_file.findings == [] assert not read_file.model_extra - assert not unread_file.model_extra + assert not failed_file.model_extra def test_get_unstructured_discovery_file_page_sends_a_blank_directory_for_the_root(client: DataMasqueClient) -> None: @@ -262,7 +262,7 @@ def test_iter_unstructured_discovery_folders_follows_the_next_page(client: DataM @pytest.mark.parametrize( "status, expected_status_query", - [(None, {}), (UnstructuredDiscoveryFileStatus.unread, {"status": ["unread"]})], + [(None, {}), (UnstructuredDiscoveryFileStatus.failed, {"status": ["failed"]})], ) def test_iter_unstructured_discovery_files_asks_for_one_folder( client: DataMasqueClient, From 395bc436e93410fd19c3d3123005d15b256df876 Mon Sep 17 00:00:00 2001 From: Timotius Date: Sun, 8 Nov 2026 01:48:01 +1300 Subject: [PATCH 09/23] docs: DM-4756: say a run without an AI Engine reports None --- HISTORY.rst | 4 ++-- datamasque/client/models/runs.py | 8 ++++---- tests/test_runs.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c604ee5..91b643f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -37,8 +37,8 @@ History or ``None`` when it has none. * Added ``ai_engine_model`` and ``ai_engine_version`` to ``RunInfo``: the AI Engine model and engine version the run's ``ai_detect`` matchers use, - blank when the run recorded none, such as a run that uses no AI Engine, - and ``None`` on a server that does not report them. + ``None`` when the run recorded none, such as a run that uses no AI Engine, + or on a server that does not report them. Requires server version 3.26.20 diff --git a/datamasque/client/models/runs.py b/datamasque/client/models/runs.py index a220054..b77095a 100644 --- a/datamasque/client/models/runs.py +++ b/datamasque/client/models/runs.py @@ -153,14 +153,14 @@ class RunInfo(BaseModel): The AI Engine model the run's `ai_detect` matchers use: for a Local AI Engine, the name of its model file or folder, and for a Bedrock AI Engine, the Bedrock model set in its configuration. - Blank when the run recorded none, such as a run that uses no AI Engine, - and `None` on a server that does not report it. + `None` when the run recorded none, such as a run that uses no AI Engine, + or on a server that does not report it. """ ai_engine_version: Optional[str] = None """ The version the AI Engine reported when the run started. - Blank when the run recorded none, such as a run that uses no AI Engine, - and `None` on a server that does not report it. + `None` when the run recorded none, such as a run that uses no AI Engine, + or on a server that does not report it. """ @model_validator(mode="before") diff --git a/tests/test_runs.py b/tests/test_runs.py index db45d85..7dc6915 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -350,7 +350,7 @@ def test_get_run_info(client): "dm-ai-ner-small-1.0.0-20260601.tar.enc", "1.26.11", ), - ({"ai_engine_model": "", "ai_engine_version": ""}, "", ""), + ({"ai_engine_model": None, "ai_engine_version": None}, None, None), ({}, None, None), ], ids=["ai_detect_run", "run_without_ai_detect", "server_without_the_fields"], From c10700338d45699681a9b8b7356d2dad11f66732 Mon Sep 17 00:00:00 2001 From: Timotius Date: Mon, 9 Nov 2026 17:39:56 +1300 Subject: [PATCH 10/23] feat: DM-4756: read the findings report from its new URL --- datamasque/client/unstructured_discovery.py | 2 +- tests/test_unstructured_discovery.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datamasque/client/unstructured_discovery.py b/datamasque/client/unstructured_discovery.py index 29ce137..00ed72e 100644 --- a/datamasque/client/unstructured_discovery.py +++ b/datamasque/client/unstructured_discovery.py @@ -231,7 +231,7 @@ def get_unstructured_report(self, run_id: RunId) -> Union[str, bytes]: A label's categories share one cell, joined with `;`. """ - response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-report/") + response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-discovery/report/") if response.headers.get("Content-Type", "").startswith("application/zip"): return response.content return response.text diff --git a/tests/test_unstructured_discovery.py b/tests/test_unstructured_discovery.py index b4ad5c8..107c35c 100644 --- a/tests/test_unstructured_discovery.py +++ b/tests/test_unstructured_discovery.py @@ -19,7 +19,7 @@ ) RESULTS_URL = "http://test-server/api/runs/41/unstructured-discovery/" -REPORT_URL = "http://test-server/api/runs/41/unstructured-report/" +REPORT_URL = "http://test-server/api/runs/41/unstructured-discovery/report/" CONNECTION_ID = "aaaaaaaa-1111-2222-3333-444444444444" ClientCall = Callable[[DataMasqueClient], object] From ff721e2afb85866619474b76ebd7ccd1dcb6e7ed Mon Sep 17 00:00:00 2001 From: Timotius Date: Mon, 9 Nov 2026 17:39:56 +1300 Subject: [PATCH 11/23] docs: DM-4756: describe sampling and files without text as the server does --- datamasque/client/models/unstructured_discovery.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datamasque/client/models/unstructured_discovery.py b/datamasque/client/models/unstructured_discovery.py index 9f2172b..2601b15 100644 --- a/datamasque/client/models/unstructured_discovery.py +++ b/datamasque/client/models/unstructured_discovery.py @@ -26,7 +26,7 @@ class UnstructuredDiscoveryFileOutcome(enum.Enum): findings = "findings" """The run found at least one label in the file.""" no_text = "no_text" - """The run read the file and found no text in it, as in a scanned document.""" + """The run read the file and found no text in it.""" nothing_found = "nothing_found" """The run read the file's text and found no label in it.""" failed = "failed" @@ -54,7 +54,7 @@ class UnstructuredDiscoverySampling(enum.Enum): sampled = "sampled" """At most `files_per_directory` files of each folder.""" all = "all" - """Every file of each folder that the run could read.""" + """The run set no limit on how many files of each folder it read.""" unknown = "unknown" """The run's discovery config cannot be read.""" @@ -110,7 +110,7 @@ class UnstructuredDiscoveryFile(BaseModel): file_extension: str size_bytes: int character_count: Optional[int] = None - """How many characters of the file's text the run searched, `None` where the run reported none.""" + """How many characters of the file's text the run searched; `None` for a file the run could not read.""" status: UnstructuredDiscoveryFileStatus skip_reason: str """Why the run could not read the file, blank if it did.""" @@ -161,7 +161,7 @@ class UnstructuredDiscoverySummary(BaseModel): files_failed: int """How many files the run tried to read and could not.""" files_without_text: int - """How many of the files the run read held no text to search, as a scanned document may not.""" + """How many of the files the run read held no text to search.""" files_with_findings: int """How many of the files the run read hold a finding.""" bytes_opened: int From 796cabe9331aec51a21628df52e2a44d74f3ce96 Mon Sep 17 00:00:00 2001 From: Timotius Date: Mon, 9 Nov 2026 20:23:34 +1300 Subject: [PATCH 12/23] docs: DM-4756: say a folder path search lists all its files --- datamasque/client/unstructured_discovery.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datamasque/client/unstructured_discovery.py b/datamasque/client/unstructured_discovery.py index 00ed72e..6eace1d 100644 --- a/datamasque/client/unstructured_discovery.py +++ b/datamasque/client/unstructured_discovery.py @@ -161,7 +161,8 @@ def get_unstructured_discovery_file_page( an empty string names the connection root. `status` keeps only the files with that status, and `outcomes` only the files with any of those outcomes. - `search` keeps the files whose name contains it, ignoring case. + `search` keeps the files whose name contains it, ignoring case, + or every file when the folder's path contains it. `labels` keeps the files holding any of those labels, and `categories` the files holding a label that carries any of those categories; given both, a label must match both. From 6e3e9ef6240d0cda12692dc07997d356e8c129e0 Mon Sep 17 00:00:00 2001 From: Timotius Date: Tue, 10 Nov 2026 10:42:40 +1300 Subject: [PATCH 13/23] docs: DM-4756: say a label filter drops failed files --- datamasque/client/unstructured_discovery.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/datamasque/client/unstructured_discovery.py b/datamasque/client/unstructured_discovery.py index 6eace1d..4bfc488 100644 --- a/datamasque/client/unstructured_discovery.py +++ b/datamasque/client/unstructured_discovery.py @@ -97,12 +97,15 @@ def get_unstructured_discovery_folder_page( Returns one page of the folders an unstructured discovery run found on the connection. `search` keeps the folders whose path contains it, ignoring case, - and the folders holding a file whose name contains it. + and the folders holding a file the run tried to read whose name contains it. `labels` keeps the folders holding any of those labels, and `categories` the folders holding a label that carries any of those categories; given both, a label must match both. + Files with no text and files the run could not read hold no label, + so they do not keep a folder. With `labels` or `categories`, a file matches `search` only if it holds a matching label, - or the run read no text from it or could not read it. + or the run read it and found no text in it; + a file the run could not read never matches. Each folder's `entity_counts` still counts every label it holds. The folders come in `ordering`, descending when `descending` is set, and folders that tie are sorted by path, ascending. @@ -166,7 +169,8 @@ def get_unstructured_discovery_file_page( `labels` keeps the files holding any of those labels, and `categories` the files holding a label that carries any of those categories; given both, a label must match both. - Either one also keeps the files the run read and found no text in, which may hold any label. + Either one also keeps the files the run read and found no text in, which may hold any label, + and drops the files the run could not read. The files with findings come first, most occurrences first, then the files with no text, those with nothing found and those the run could not read, each group sorted by name. From d4719794869e3b539f9a28268b467cd78f73953d Mon Sep 17 00:00:00 2001 From: Timotius Date: Tue, 10 Nov 2026 10:43:00 +1300 Subject: [PATCH 14/23] docs: DM-4756: list the findings report's columns --- datamasque/client/unstructured_discovery.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/datamasque/client/unstructured_discovery.py b/datamasque/client/unstructured_discovery.py index 4bfc488..155ac85 100644 --- a/datamasque/client/unstructured_discovery.py +++ b/datamasque/client/unstructured_discovery.py @@ -229,10 +229,12 @@ def get_unstructured_report(self, run_id: RunId) -> Union[str, bytes]: A file the run read and found nothing in has one row, with a blank label, description and categories, and 0 occurrences. A file the run could not read has no row. - Every row carries its file's `character_count`, blank where the run reported none. + Every row carries its file's `character_count`, + how many characters of the file's text the run searched. Returns CSV text (`str`) that starts with a byte-order mark, or a zip of numbered CSV parts as `bytes` when the server splits a report of more than 1,000,000 rows. - The first row of each CSV names the columns. + The first row of each CSV names the columns: + `path`, `directory`, `label`, `description`, `occurrences`, `categories` and `character_count`. A label's categories share one cell, joined with `;`. """ From a4176a2da01711c455cf255aada3faafac7f00c7 Mon Sep 17 00:00:00 2001 From: Timotius Date: Tue, 10 Nov 2026 10:43:14 +1300 Subject: [PATCH 15/23] docs: DM-4756: say files_read excludes failed files --- datamasque/client/models/unstructured_discovery.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datamasque/client/models/unstructured_discovery.py b/datamasque/client/models/unstructured_discovery.py index 2601b15..67ec430 100644 --- a/datamasque/client/models/unstructured_discovery.py +++ b/datamasque/client/models/unstructured_discovery.py @@ -16,6 +16,7 @@ class UnstructuredDiscoveryFileStatus(enum.Enum): """Whether an unstructured discovery run read a file's text.""" read = "read" + """The run read the file; its `character_count` is 0 when it held no text.""" failed = "failed" """The run tried to read the file and could not; `skip_reason` says why.""" @@ -81,6 +82,7 @@ class UnstructuredDiscoveryFolder(BaseModel): files_present: int """How many files were found in the folder, read or not.""" files_read: int + """How many of the folder's files the run read, including those with no text but not those it could not read.""" size_bytes: int """The total size, in bytes, of the folder's files, read or not.""" entity_counts: dict[str, int] @@ -158,6 +160,7 @@ class UnstructuredDiscoverySummary(BaseModel): files_found: int """How many files were found on the connection, summed over the folders.""" files_read: int + """How many files the run read, including those with no text but not those it could not read.""" files_failed: int """How many files the run tried to read and could not.""" files_without_text: int From 5993ee79363a42db30d9665fafde6f06c9051f02 Mon Sep 17 00:00:00 2001 From: Timotius Date: Tue, 10 Nov 2026 10:44:00 +1300 Subject: [PATCH 16/23] refactor: DM-4756: share the CSV-or-zip report read --- datamasque/client/base.py | 8 ++++++++ datamasque/client/runs.py | 4 +--- datamasque/client/unstructured_discovery.py | 4 +--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/datamasque/client/base.py b/datamasque/client/base.py index 673356d..f377106 100644 --- a/datamasque/client/base.py +++ b/datamasque/client/base.py @@ -313,6 +313,14 @@ def _delete_if_exists(self, path: str, *, params: Optional[dict] = None) -> None self._raise_for_status(response) + @staticmethod + def _read_csv_or_zip(response: Response) -> Union[str, bytes]: + """Returns a report's CSV text, or as `bytes` the zip of CSV parts the server splits a large report into.""" + + if response.headers.get("Content-Type", "").startswith("application/zip"): + return response.content + return response.text + def _iter_paginated( self, path: str, diff --git a/datamasque/client/runs.py b/datamasque/client/runs.py index 9f4c87c..1b85d1d 100644 --- a/datamasque/client/runs.py +++ b/datamasque/client/runs.py @@ -59,9 +59,7 @@ def get_db_discovery_result_report(self, run_id: RunId, include_selection_column params = None if include_selection_column else {"include_selection_column": "false"} response = self.make_request("GET", url, params=params) - if response.headers.get("Content-Type", "").startswith("application/zip"): - return response.content - return response.text + return self._read_csv_or_zip(response) def get_unfinished_runs(self) -> dict[str, UnfinishedRun]: """Queries the DM instance for unfinished runs, and returns them organised by connection name.""" diff --git a/datamasque/client/unstructured_discovery.py b/datamasque/client/unstructured_discovery.py index 155ac85..90d570a 100644 --- a/datamasque/client/unstructured_discovery.py +++ b/datamasque/client/unstructured_discovery.py @@ -239,6 +239,4 @@ def get_unstructured_report(self, run_id: RunId) -> Union[str, bytes]: """ response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-discovery/report/") - if response.headers.get("Content-Type", "").startswith("application/zip"): - return response.content - return response.text + return self._read_csv_or_zip(response) From 29ef9d369c93eb02c8ce7d8f49baa6bb3d9fd8b2 Mon Sep 17 00:00:00 2001 From: Timotius Date: Tue, 10 Nov 2026 10:44:42 +1300 Subject: [PATCH 17/23] test: DM-4756: assert the keyword-only TypeError message --- tests/test_discovery_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_discovery_configs.py b/tests/test_discovery_configs.py index a5378d6..70be617 100644 --- a/tests/test_discovery_configs.py +++ b/tests/test_discovery_configs.py @@ -532,7 +532,7 @@ def test_get_default_discovery_config_yaml_raises_when_the_server_refuses_the_ki def test_default_discovery_config_parameters_are_keyword_only(client: DataMasqueClient) -> None: - with pytest.raises(TypeError): + with pytest.raises(TypeError, match="takes 1 positional argument but 2 were given"): client.get_default_discovery_config_yaml(DiscoveryConfigType.file) From 1315ed23435018dbdad8dbd5456c9ecb6a7ef60b Mon Sep 17 00:00:00 2001 From: Timotius Date: Tue, 10 Nov 2026 10:44:53 +1300 Subject: [PATCH 18/23] docs: DM-4756: call what a document run finds labels --- HISTORY.rst | 2 +- datamasque/client/models/discovery_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 91b643f..8aa0e1d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,7 +18,7 @@ History typed by ``DiscoveryConfigType`` and the new ``DiscoveryKind`` enum. ``discovery_kind=DiscoveryKind.unstructured`` with ``config_type=DiscoveryConfigType.file`` returns the template of an unstructured discovery config, - which counts entities in the text of documents. + which counts the occurrences of its labels in the text of documents. * Added methods that read an unstructured discovery run's results: ``get_unstructured_discovery_folder_page`` and ``iter_unstructured_discovery_folders`` for the folders found on the connection, diff --git a/datamasque/client/models/discovery_config.py b/datamasque/client/models/discovery_config.py index 051dcf8..3c403b9 100644 --- a/datamasque/client/models/discovery_config.py +++ b/datamasque/client/models/discovery_config.py @@ -17,7 +17,7 @@ class DiscoveryConfigType(enum.Enum): class DiscoveryKind(enum.Enum): - """What a file discovery config looks for: sensitive locators, or entities in the text of documents.""" + """What a file discovery config looks for: sensitive locators, or its labels in the text of documents.""" structured = "structured" unstructured = "unstructured" From 52be6bb1b176094883fd756016059b4d781315d9 Mon Sep 17 00:00:00 2001 From: Timotius Date: Tue, 10 Nov 2026 10:46:13 +1300 Subject: [PATCH 19/23] docs: DM-4756: say no-text files have no findings --- datamasque/client/unstructured_discovery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datamasque/client/unstructured_discovery.py b/datamasque/client/unstructured_discovery.py index 90d570a..ddf26ee 100644 --- a/datamasque/client/unstructured_discovery.py +++ b/datamasque/client/unstructured_discovery.py @@ -101,8 +101,8 @@ def get_unstructured_discovery_folder_page( `labels` keeps the folders holding any of those labels, and `categories` the folders holding a label that carries any of those categories; given both, a label must match both. - Files with no text and files the run could not read hold no label, - so they do not keep a folder. + Files with no text and files the run could not read have no findings, + so neither filter keeps a folder for them. With `labels` or `categories`, a file matches `search` only if it holds a matching label, or the run read it and found no text in it; a file the run could not read never matches. From 34e400a7846757c5cb9f82eaab2705498782dd03 Mon Sep 17 00:00:00 2001 From: Timotius Date: Tue, 10 Nov 2026 20:42:35 +1300 Subject: [PATCH 20/23] feat: DM-4756: follow the split unstructured discovery API --- datamasque/client/__init__.py | 10 +- datamasque/client/models/runs.py | 6 + .../client/models/unstructured_discovery.py | 88 +++++--- datamasque/client/unstructured_discovery.py | 39 ++-- tests/test_runs.py | 28 +++ tests/test_unstructured_discovery.py | 208 +++++++++++------- 6 files changed, 256 insertions(+), 123 deletions(-) diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index 74f98ce..6e03e7c 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -178,6 +178,7 @@ TableReferenceOptions, ) from datamasque.client.models.unstructured_discovery import ( + UnstructuredDiscoveryAiEngine, UnstructuredDiscoveryFile, UnstructuredDiscoveryFileOutcome, UnstructuredDiscoveryFileStatus, @@ -185,8 +186,8 @@ UnstructuredDiscoveryFolder, UnstructuredDiscoveryFolderOrdering, UnstructuredDiscoveryLabelTotal, - UnstructuredDiscoveryPage, - UnstructuredDiscoverySampling, + UnstructuredDiscoverySampleOrder, + UnstructuredDiscoverySettings, UnstructuredDiscoverySummary, UnstructuredDiscoveryUnreadCount, ) @@ -349,8 +350,9 @@ "UnstructuredDiscoveryFolder", "UnstructuredDiscoveryFolderOrdering", "UnstructuredDiscoveryLabelTotal", - "UnstructuredDiscoveryPage", - "UnstructuredDiscoverySampling", + "UnstructuredDiscoveryAiEngine", + "UnstructuredDiscoverySampleOrder", + "UnstructuredDiscoverySettings", "UnstructuredDiscoverySummary", "UnstructuredDiscoveryUnreadCount", "UnsupportedPreview", diff --git a/datamasque/client/models/runs.py b/datamasque/client/models/runs.py index b77095a..936a005 100644 --- a/datamasque/client/models/runs.py +++ b/datamasque/client/models/runs.py @@ -7,6 +7,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from datamasque.client.models.connection import ConnectionConfig, ConnectionId, unwrap_connection_id +from datamasque.client.models.discovery_config import DiscoveryKind from datamasque.client.models.ruleset import Ruleset, RulesetId, unwrap_ruleset_id from datamasque.client.models.status import MaskingRunStatus @@ -162,6 +163,11 @@ class RunInfo(BaseModel): `None` when the run recorded none, such as a run that uses no AI Engine, or on a server that does not report it. """ + discovery_kind: Optional[DiscoveryKind] = None + """ + What a file discovery run looked for: sensitive locators, or labels in the text of documents. + `None` for a run that ran no discovery task, or on a server that does not report it. + """ @model_validator(mode="before") @classmethod diff --git a/datamasque/client/models/unstructured_discovery.py b/datamasque/client/models/unstructured_discovery.py index 67ec430..68b6047 100644 --- a/datamasque/client/models/unstructured_discovery.py +++ b/datamasque/client/models/unstructured_discovery.py @@ -2,15 +2,12 @@ import enum from datetime import datetime -from typing import Generic, Optional, TypeVar +from typing import Optional from pydantic import BaseModel, ConfigDict -from datamasque.client.models.pagination import Page from datamasque.client.models.runs import RunConnectionRef, RunId -T = TypeVar("T") - class UnstructuredDiscoveryFileStatus(enum.Enum): """Whether an unstructured discovery run read a file's text.""" @@ -49,15 +46,15 @@ class UnstructuredDiscoveryFolderOrdering(enum.Enum): """ -class UnstructuredDiscoverySampling(enum.Enum): - """How many files of each folder an unstructured discovery run read.""" +class UnstructuredDiscoverySampleOrder(enum.Enum): + """The order an unstructured discovery run takes a folder's files in when it reads only some of them.""" - sampled = "sampled" - """At most `files_per_directory` files of each folder.""" - all = "all" - """The run set no limit on how many files of each folder it read.""" - unknown = "unknown" - """The run's discovery config cannot be read.""" + hash = "hash" + """An order that is fixed for each file but looks random.""" + modified_desc = "modified_desc" + """The most recently modified files first.""" + name_asc = "name_asc" + """By name, ascending.""" class UnstructuredDiscoveryUnreadCount(BaseModel): @@ -67,6 +64,9 @@ class UnstructuredDiscoveryUnreadCount(BaseModel): # A string rather than an enum, because each server release can add reasons. skip_reason: str + """Why the run did not try to read these files, as a code such as `over_sample_limit`.""" + skip_reason_text: str + """The same reason as a sentence, as the run report words it.""" file_extension: str """The extension without a leading dot, blank for files that have none.""" count: int @@ -97,10 +97,11 @@ class UnstructuredDiscoveryFinding(BaseModel): model_config = ConfigDict(extra="allow") label: str - """The label's `name` in the discovery config.""" - description: str + """ + The label's `name` in the discovery config. + `get_unstructured_discovery_summary` gives its description and categories. + """ occurrences: int - categories: list[str] class UnstructuredDiscoveryFile(BaseModel): @@ -114,26 +115,36 @@ class UnstructuredDiscoveryFile(BaseModel): character_count: Optional[int] = None """How many characters of the file's text the run searched; `None` for a file the run could not read.""" status: UnstructuredDiscoveryFileStatus + # A string rather than an enum, because each server release can add reasons. skip_reason: str - """Why the run could not read the file, blank if it did.""" + """Why the run could not read the file, as a code such as `unscannable_file`; blank if it read it.""" + skip_reason_text: str + """The same reason as a sentence, as the run report words it; blank if the run read the file.""" findings: list[UnstructuredDiscoveryFinding] -class UnstructuredDiscoveryPage(Page[T], Generic[T]): - """ - A page of an unstructured discovery run's folders, or of one folder's files. +class UnstructuredDiscoverySettings(BaseModel): + """The sampling and size settings an unstructured discovery run used.""" - Every page also carries the run it belongs to and how the run sampled each folder. - `get_unstructured_discovery_summary` returns the run's totals. - """ + model_config = ConfigDict(extra="allow") - run_id: RunId - connection: RunConnectionRef - finished_at: Optional[datetime] = None - """When the run ended, `None` until it has.""" - sampling: UnstructuredDiscoverySampling files_per_directory: Optional[int] = None - """The sample size when `sampling` is `UnstructuredDiscoverySampling.sampled`, `None` otherwise.""" + """How many files of each folder the run read at most, `None` when it read every file.""" + sample_order: UnstructuredDiscoverySampleOrder + files_per_run: Optional[int] = None + """How many files the run read at most in all, `None` when it set no limit.""" + max_bytes_per_file: int + """The size, in bytes, above which the run did not read a file.""" + + +class UnstructuredDiscoveryAiEngine(BaseModel): + """The AI Engine an unstructured discovery run's `ai_detect` matchers used.""" + + model_config = ConfigDict(extra="allow") + + model: str + version: Optional[str] = None + """The version the AI Engine reported, `None` when it reported none.""" class UnstructuredDiscoveryLabelTotal(BaseModel): @@ -152,11 +163,25 @@ class UnstructuredDiscoveryLabelTotal(BaseModel): class UnstructuredDiscoverySummary(BaseModel): - """An unstructured discovery run's totals.""" + """An unstructured discovery run, the settings it used and its totals.""" model_config = ConfigDict(extra="allow") run_id: RunId + connection: RunConnectionRef + status: str + """The run's status, such as `finished`.""" + finished_at: Optional[datetime] = None + """When the run ended, `None` until it has.""" + settings: Optional[UnstructuredDiscoverySettings] = None + """The settings the run used, `None` when it ended before it read any folder.""" + ai_engine: Optional[UnstructuredDiscoveryAiEngine] = None + """The AI Engine the run used, `None` when it used none.""" + report_rows: int + """ + How many rows the findings report has. + `get_unstructured_report` returns a zip of CSV parts above 1,000,000 rows. + """ files_found: int """How many files were found on the connection, summed over the folders.""" files_read: int @@ -172,4 +197,7 @@ class UnstructuredDiscoverySummary(BaseModel): files_over_run_limit: int """How many files the run did not read because it reached the config's `files_per_run`.""" labels: list[UnstructuredDiscoveryLabelTotal] - """Each label found, most found first; labels found equally often are listed by name.""" + """ + Each label of the run's discovery config, most found first; labels found equally often are listed by name. + A label the run never found is listed with 0 occurrences. + """ diff --git a/datamasque/client/unstructured_discovery.py b/datamasque/client/unstructured_discovery.py index ddf26ee..e8d3961 100644 --- a/datamasque/client/unstructured_discovery.py +++ b/datamasque/client/unstructured_discovery.py @@ -2,14 +2,16 @@ from datamasque.client.base import BaseClient from datamasque.client.models.connection import ConnectionId +from datamasque.client.models.discovery_config import DiscoveryKind +from datamasque.client.models.pagination import Page from datamasque.client.models.runs import RunId +from datamasque.client.models.status import MaskingRunStatus from datamasque.client.models.unstructured_discovery import ( UnstructuredDiscoveryFile, UnstructuredDiscoveryFileOutcome, UnstructuredDiscoveryFileStatus, UnstructuredDiscoveryFolder, UnstructuredDiscoveryFolderOrdering, - UnstructuredDiscoveryPage, UnstructuredDiscoverySummary, ) @@ -66,7 +68,7 @@ def _build_unstructured_file_page_params( params["status"] = status.value params |= UnstructuredDiscoveryClient._build_label_filter_params(search, labels, categories) if outcomes is not None: - params["outcome"] = LIST_SEPARATOR.join(outcome.value for outcome in outcomes) + params["outcomes"] = LIST_SEPARATOR.join(outcome.value for outcome in outcomes) return params @@ -75,11 +77,20 @@ def get_last_unstructured_discovery_run_id(self, connection_id: ConnectionId) -> Returns the ID of a connection's newest finished unstructured discovery run, with or without warnings. Returns `None` when no unstructured discovery run on the connection has finished. + A dry run stores no results, so it is never returned. """ - response = self.make_request("GET", f"/api/connections/{connection_id}/") - run_id = response.json()["last_unstructured_discovery_run_id"] - return RunId(run_id) if run_id is not None else None + params = { + "discovery_kind": DiscoveryKind.unstructured.value, + "source_connection": connection_id, + "dry_run": "false", + "run_status": [MaskingRunStatus.finished.value, MaskingRunStatus.finished_with_warnings.value], + "ordering": "-id", + "limit": 1, + } + response = self.make_request("GET", "/api/runs/", params=params) + runs = response.json()["results"] + return RunId(runs[0]["id"]) if runs else None def get_unstructured_discovery_folder_page( self, @@ -92,7 +103,7 @@ def get_unstructured_discovery_folder_page( descending: bool = False, limit: int = 50, offset: int = 0, - ) -> UnstructuredDiscoveryPage[UnstructuredDiscoveryFolder]: + ) -> Page[UnstructuredDiscoveryFolder]: """ Returns one page of the folders an unstructured discovery run found on the connection. @@ -115,8 +126,8 @@ def get_unstructured_discovery_folder_page( "limit": limit, "offset": offset, } - response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-discovery/", params=params) - return UnstructuredDiscoveryPage[UnstructuredDiscoveryFolder].model_validate(response.json()) + response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-discovery/directories/", params=params) + return Page[UnstructuredDiscoveryFolder].model_validate(response.json()) # `iter_unstructured_discovery_folders` and `iter_unstructured_discovery_files` have no `list_` counterparts, # because a run's folder and file counts are unbounded. @@ -138,7 +149,7 @@ def iter_unstructured_discovery_folders( """ return self._iter_paginated( - f"/api/runs/{run_id}/unstructured-discovery/", + f"/api/runs/{run_id}/unstructured-discovery/directories/", model=UnstructuredDiscoveryFolder, params=self._build_unstructured_folder_page_params(search, labels, categories, ordering, descending), page_size=SERVER_MAX_PAGE_SIZE, @@ -156,7 +167,7 @@ def get_unstructured_discovery_file_page( outcomes: Optional[list[UnstructuredDiscoveryFileOutcome]] = None, limit: int = 50, offset: int = 0, - ) -> UnstructuredDiscoveryPage[UnstructuredDiscoveryFile]: + ) -> Page[UnstructuredDiscoveryFile]: """ Returns one page of the files an unstructured discovery run tried to read in one folder. @@ -183,8 +194,8 @@ def get_unstructured_discovery_file_page( "limit": limit, "offset": offset, } - response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-discovery/", params=params) - return UnstructuredDiscoveryPage[UnstructuredDiscoveryFile].model_validate(response.json()) + response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-discovery/files/", params=params) + return Page[UnstructuredDiscoveryFile].model_validate(response.json()) def iter_unstructured_discovery_files( self, @@ -205,7 +216,7 @@ def iter_unstructured_discovery_files( """ return self._iter_paginated( - f"/api/runs/{run_id}/unstructured-discovery/", + f"/api/runs/{run_id}/unstructured-discovery/files/", model=UnstructuredDiscoveryFile, params=self._build_unstructured_file_page_params(directory, status, search, labels, categories, outcomes), page_size=SERVER_MAX_PAGE_SIZE, @@ -213,7 +224,7 @@ def iter_unstructured_discovery_files( def get_unstructured_discovery_summary(self, run_id: RunId) -> UnstructuredDiscoverySummary: """ - Returns an unstructured discovery run's totals, and how many times it found each label. + Returns an unstructured discovery run, the settings it used, its totals, and how many times it found each label. They are counted from the run's stored results on each request, so they change while results are still being stored. diff --git a/tests/test_runs.py b/tests/test_runs.py index 7dc6915..d2fab2a 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -13,6 +13,7 @@ RunNotCancellableError, ) from datamasque.client.models.connection import ConnectionId, DatabaseConnectionConfig, DatabaseType +from datamasque.client.models.discovery_config import DiscoveryKind from datamasque.client.models.ruleset import Ruleset, RulesetId, RulesetType from datamasque.client.models.runs import ( MaskingRunOptions, @@ -375,6 +376,33 @@ def test_run_info_reads_the_ai_engine_model_and_version( assert not run.model_extra +@pytest.mark.parametrize( + "discovery_kind_field, expected", + [ + ({"discovery_kind": "unstructured"}, DiscoveryKind.unstructured), + ({"discovery_kind": "structured"}, DiscoveryKind.structured), + ({"discovery_kind": None}, None), + ({}, None), + ], + ids=["unstructured_discovery_run", "structured_discovery_run", "masking_run", "server_without_the_field"], +) +def test_run_info_reads_the_discovery_kind(discovery_kind_field: dict, expected: Optional[DiscoveryKind]) -> None: + run = RunInfo.model_validate( + { + "id": 1, + "status": "finished", + "mask_type": "file_data_discovery", + "source_connection_name": "documents", + "ruleset_name": "", + "project": "1-default", + **discovery_kind_field, + } + ) + + assert run.discovery_kind is expected + assert not run.model_extra + + def test_masking_run_request_model_dump_minimal(): """A minimal request dumps with an empty `options` object (the server rejects missing `options`).""" req = MaskingRunRequest(connection="conn-1", ruleset="rs-1") diff --git a/tests/test_unstructured_discovery.py b/tests/test_unstructured_discovery.py index 107c35c..dfb94f9 100644 --- a/tests/test_unstructured_discovery.py +++ b/tests/test_unstructured_discovery.py @@ -15,29 +15,21 @@ UnstructuredDiscoveryFileOutcome, UnstructuredDiscoveryFileStatus, UnstructuredDiscoveryFolderOrdering, - UnstructuredDiscoverySampling, + UnstructuredDiscoverySampleOrder, ) RESULTS_URL = "http://test-server/api/runs/41/unstructured-discovery/" -REPORT_URL = "http://test-server/api/runs/41/unstructured-discovery/report/" +FOLDERS_URL = f"{RESULTS_URL}directories/" +FILES_URL = f"{RESULTS_URL}files/" +REPORT_URL = f"{RESULTS_URL}report/" +RUNS_URL = "http://test-server/api/runs/" CONNECTION_ID = "aaaaaaaa-1111-2222-3333-444444444444" ClientCall = Callable[[DataMasqueClient], object] -def _envelope(results: list[dict], **overrides: Any) -> dict: - return { - "run_id": 41, - "connection": {"id": CONNECTION_ID, "name": "invoices"}, - "finished_at": "2026-09-04T02:11:07Z", - "sampling": "sampled", - "files_per_directory": 3, - "count": len(results), - "next": None, - "previous": None, - "results": results, - **overrides, - } +def _page(results: list[dict], **overrides: Any) -> dict: + return {"count": len(results), "next": None, "previous": None, "results": results, **overrides} def _folder(directory: str) -> dict: @@ -47,7 +39,14 @@ def _folder(directory: str) -> dict: "files_read": 20, "size_bytes": 14399, "entity_counts": {"CREDIT_CARD": 3, "EMAIL": 62}, - "unread": [{"skip_reason": "Not selected by the per-directory sample", "file_extension": "pdf", "count": 318}], + "unread": [ + { + "skip_reason": "over_sample_limit", + "skip_reason_text": "Not selected by the per-folder sample", + "file_extension": "pdf", + "count": 318, + } + ], } @@ -56,6 +55,7 @@ def _file( *, status: str = "read", skip_reason: str = "", + skip_reason_text: str = "", character_count: Optional[int] = 5120, findings: Optional[list[dict]] = None, ) -> dict: @@ -66,57 +66,42 @@ def _file( "character_count": character_count, "status": status, "skip_reason": skip_reason, + "skip_reason_text": skip_reason_text, "findings": findings if findings is not None else [], } @pytest.mark.parametrize( - "last_run_field, expected", - [ - ({"last_unstructured_discovery_run_id": 19}, RunId(19)), - ({"last_unstructured_discovery_run_id": None}, None), - ], + "runs, expected", + [([{"id": 19, "status": "finished"}], RunId(19)), ([], None)], + ids=["finished_run", "no_finished_run"], ) -def test_get_last_unstructured_discovery_run_id_reads_the_connection( - client: DataMasqueClient, existing_connection_json: dict, last_run_field: dict, expected: Optional[RunId] +def test_get_last_unstructured_discovery_run_id_asks_the_run_list( + client: DataMasqueClient, runs: list[dict], expected: Optional[RunId] ) -> None: with requests_mock.Mocker() as m: - m.get( - f"http://test-server/api/connections/{CONNECTION_ID}/", - json=existing_connection_json | {"id": CONNECTION_ID} | last_run_field, - ) + m.get(RUNS_URL, json=_page(runs)) assert client.get_last_unstructured_discovery_run_id(ConnectionId(CONNECTION_ID)) == expected - -def test_get_last_unstructured_discovery_run_id_raises_on_a_server_without_the_field( - client: DataMasqueClient, existing_connection_json: dict -) -> None: - with requests_mock.Mocker() as m: - m.get( - f"http://test-server/api/connections/{CONNECTION_ID}/", - json=existing_connection_json | {"id": CONNECTION_ID}, - ) - - with pytest.raises(KeyError) as exc_info: - client.get_last_unstructured_discovery_run_id(ConnectionId(CONNECTION_ID)) - - assert exc_info.value.args == ("last_unstructured_discovery_run_id",) + assert m.last_request.qs == { + "discovery_kind": ["unstructured"], + "source_connection": [CONNECTION_ID], + "dry_run": ["false"], + "run_status": ["finished", "finished_with_warnings"], + "ordering": ["-id"], + "limit": ["1"], + } -def test_get_unstructured_discovery_folder_page_parses_the_run_and_its_folders(client: DataMasqueClient) -> None: +def test_get_unstructured_discovery_folder_page_parses_the_folders(client: DataMasqueClient) -> None: with requests_mock.Mocker() as m: - m.get(RESULTS_URL, json=_envelope([_folder("finance/invoices")])) + m.get(FOLDERS_URL, json=_page([_folder("finance/invoices")])) page = client.get_unstructured_discovery_folder_page(RunId(41), limit=10, offset=20) assert m.last_request.qs == {"ordering": ["directory"], "limit": ["10"], "offset": ["20"]} - assert page.run_id == 41 - assert page.connection.id == CONNECTION_ID - assert page.connection.name == "invoices" - assert page.finished_at == datetime(2026, 9, 4, 2, 11, 7, tzinfo=timezone.utc) - assert page.sampling is UnstructuredDiscoverySampling.sampled - assert page.files_per_directory == 3 assert page.count == 1 + assert not page.model_extra folder = page.results[0] assert folder.directory == "finance/invoices" @@ -124,21 +109,20 @@ def test_get_unstructured_discovery_folder_page_parses_the_run_and_its_folders(c assert folder.files_read == 20 assert folder.size_bytes == 14399 assert folder.entity_counts == {"CREDIT_CARD": 3, "EMAIL": 62} - assert folder.unread[0].skip_reason == "Not selected by the per-directory sample" + assert folder.unread[0].skip_reason == "over_sample_limit" + assert folder.unread[0].skip_reason_text == "Not selected by the per-folder sample" assert folder.unread[0].file_extension == "pdf" assert folder.unread[0].count == 318 + assert not folder.unread[0].model_extra def test_get_unstructured_discovery_folder_page_asks_for_the_first_page_by_default(client: DataMasqueClient) -> None: with requests_mock.Mocker() as m: - m.get(RESULTS_URL, json=_envelope([], sampling="all", files_per_directory=None, finished_at=None)) + m.get(FOLDERS_URL, json=_page([])) page = client.get_unstructured_discovery_folder_page(RunId(41)) assert m.last_request.qs == {"ordering": ["directory"], "limit": ["50"], "offset": ["0"]} assert page.results == [] - assert page.sampling is UnstructuredDiscoverySampling.all - assert page.files_per_directory is None - assert page.finished_at is None @pytest.mark.parametrize( @@ -153,7 +137,7 @@ def test_unstructured_discovery_folders_send_the_search_and_label_filters( client: DataMasqueClient, call: Callable[..., object], expected_limit: str ) -> None: with requests_mock.Mocker(case_sensitive=True) as m: - m.get(RESULTS_URL, json=_envelope([_folder("Finance/Invoices")])) + m.get(FOLDERS_URL, json=_page([_folder("Finance/Invoices")])) call(client, search="Invoices", labels=["EMAIL", "CREDIT_CARD"], categories=["PII", "Financial"]) assert m.last_request.qs == { @@ -192,25 +176,26 @@ def test_unstructured_discovery_folders_send_the_ordering( expected_ordering: str, ) -> None: with requests_mock.Mocker(case_sensitive=True) as m: - m.get(RESULTS_URL, json=_envelope([_folder("Finance/Invoices")])) + m.get(FOLDERS_URL, json=_page([_folder("Finance/Invoices")])) call(client, ordering=ordering, descending=descending) assert m.last_request.qs["ordering"] == [expected_ordering] def test_get_unstructured_discovery_file_page_parses_the_files_and_their_findings(client: DataMasqueClient) -> None: - finding = {"label": "EMAIL", "description": "Email address", "occurrences": 4, "categories": ["PII"]} + finding = {"label": "EMAIL", "occurrences": 4} rows = [ _file("Finance/Invoices/0142.pdf", findings=[finding]), _file( "Finance/Invoices/0143.pdf", status="failed", - skip_reason="An error occurred while running data discovery", + skip_reason="unscannable_file", + skip_reason_text="File could not be scanned", character_count=None, ), ] with requests_mock.Mocker(case_sensitive=True) as m: - m.get(RESULTS_URL, json=_envelope(rows)) + m.get(FILES_URL, json=_page(rows)) page = client.get_unstructured_discovery_file_page( RunId(41), "Finance/Invoices", status=UnstructuredDiscoveryFileStatus.read, limit=1000, offset=5 ) @@ -228,12 +213,13 @@ def test_get_unstructured_discovery_file_page_parses_the_files_and_their_finding assert read_file.character_count == 5120 assert read_file.status is UnstructuredDiscoveryFileStatus.read assert read_file.skip_reason == "" + assert read_file.skip_reason_text == "" assert read_file.findings[0].label == "EMAIL" - assert read_file.findings[0].description == "Email address" assert read_file.findings[0].occurrences == 4 - assert read_file.findings[0].categories == ["PII"] + assert not read_file.findings[0].model_extra assert failed_file.status is UnstructuredDiscoveryFileStatus.failed - assert failed_file.skip_reason == "An error occurred while running data discovery" + assert failed_file.skip_reason == "unscannable_file" + assert failed_file.skip_reason_text == "File could not be scanned" assert failed_file.character_count is None assert failed_file.findings == [] assert not read_file.model_extra @@ -242,7 +228,7 @@ def test_get_unstructured_discovery_file_page_parses_the_files_and_their_finding def test_get_unstructured_discovery_file_page_sends_a_blank_directory_for_the_root(client: DataMasqueClient) -> None: with requests_mock.Mocker(case_sensitive=True) as m: - m.get(RESULTS_URL, json=_envelope([_file("notes.pdf")])) + m.get(FILES_URL, json=_page([_file("notes.pdf")])) page = client.get_unstructured_discovery_file_page(RunId(41), "") assert m.last_request.qs == {"directory": [""], "limit": ["50"], "offset": ["0"]} @@ -250,9 +236,9 @@ def test_get_unstructured_discovery_file_page_sends_a_blank_directory_for_the_ro def test_iter_unstructured_discovery_folders_follows_the_next_page(client: DataMasqueClient) -> None: - next_url = f"{RESULTS_URL}?limit=1000&offset=1000" + next_url = f"{FOLDERS_URL}?limit=1000&offset=1000" with requests_mock.Mocker() as m: - m.get(RESULTS_URL, [{"json": _envelope([_folder("a")], next=next_url)}, {"json": _envelope([_folder("b")])}]) + m.get(FOLDERS_URL, [{"json": _page([_folder("a")], next=next_url)}, {"json": _page([_folder("b")])}]) folders = list(client.iter_unstructured_discovery_folders(RunId(41))) assert [folder.directory for folder in folders] == ["a", "b"] @@ -270,7 +256,7 @@ def test_iter_unstructured_discovery_files_asks_for_one_folder( expected_status_query: dict[str, list[str]], ) -> None: with requests_mock.Mocker(case_sensitive=True) as m: - m.get(RESULTS_URL, json=_envelope([_file("claims/1.pdf"), _file("claims/2.pdf")])) + m.get(FILES_URL, json=_page([_file("claims/1.pdf"), _file("claims/2.pdf")])) files = list(client.iter_unstructured_discovery_files(RunId(41), "claims", status=status)) assert [file.path for file in files] == ["claims/1.pdf", "claims/2.pdf"] @@ -297,7 +283,7 @@ def test_unstructured_discovery_files_send_the_search_label_and_outcome_filters( client: DataMasqueClient, call: Callable[..., object], expected_limit: str ) -> None: with requests_mock.Mocker(case_sensitive=True) as m: - m.get(RESULTS_URL, json=_envelope([_file("claims/Claim-1.pdf")])) + m.get(FILES_URL, json=_page([_file("claims/Claim-1.pdf")])) call( client, search="Claim", @@ -311,15 +297,26 @@ def test_unstructured_discovery_files_send_the_search_label_and_outcome_filters( "search": ["Claim"], "labels": ["EMAIL,CREDIT_CARD"], "categories": ["PII,Financial"], - "outcome": ["no_text,failed"], + "outcomes": ["no_text,failed"], "limit": [expected_limit], "offset": ["0"], } -def test_get_unstructured_discovery_summary_parses_the_totals_and_each_label(client: DataMasqueClient) -> None: - totals = { +def _summary(**overrides: Any) -> dict: + return { "run_id": 41, + "connection": {"id": CONNECTION_ID, "name": "invoices"}, + "status": "finished", + "finished_at": "2026-09-04T02:11:07Z", + "settings": { + "files_per_directory": 20, + "sample_order": "hash", + "files_per_run": 100000, + "max_bytes_per_file": 10000000, + }, + "ai_engine": {"model": "dm-ai-ner-small-1.0.0-20260601", "version": "1.26.11"}, + "report_rows": 15, "files_found": 340, "files_read": 20, "files_failed": 2, @@ -334,14 +331,34 @@ def test_get_unstructured_discovery_summary_parses_the_totals_and_each_label(cli "categories": ["PII", "Contact"], "occurrences": 62, "files": 11, - } + }, + {"label": "IBAN", "description": "Bank account", "categories": ["PCI"], "occurrences": 0, "files": 0}, ], + **overrides, } + + +def test_get_unstructured_discovery_summary_parses_the_run_its_totals_and_each_label( + client: DataMasqueClient, +) -> None: with requests_mock.Mocker() as m: - m.get(f"{RESULTS_URL}summary/", json=totals) + m.get(f"{RESULTS_URL}summary/", json=_summary()) summary = client.get_unstructured_discovery_summary(RunId(41)) assert summary.run_id == 41 + assert summary.connection.id == CONNECTION_ID + assert summary.connection.name == "invoices" + assert summary.status == "finished" + assert summary.finished_at == datetime(2026, 9, 4, 2, 11, 7, tzinfo=timezone.utc) + assert summary.settings is not None + assert summary.settings.files_per_directory == 20 + assert summary.settings.sample_order is UnstructuredDiscoverySampleOrder.hash + assert summary.settings.files_per_run == 100000 + assert summary.settings.max_bytes_per_file == 10000000 + assert summary.ai_engine is not None + assert summary.ai_engine.model == "dm-ai-ner-small-1.0.0-20260601" + assert summary.ai_engine.version == "1.26.11" + assert summary.report_rows == 15 assert summary.files_found == 340 assert summary.files_read == 20 assert summary.files_failed == 2 @@ -349,16 +366,57 @@ def test_get_unstructured_discovery_summary_parses_the_totals_and_each_label(cli assert summary.files_with_findings == 14 assert summary.bytes_opened == 8123344 assert summary.files_over_run_limit == 7 - label_total = summary.labels[0] + label_total, never_found = summary.labels assert label_total.label == "EMAIL" assert label_total.description == "Email address" assert label_total.categories == ["PII", "Contact"] assert label_total.occurrences == 62 assert label_total.files == 11 + assert never_found.occurrences == 0 assert not summary.model_extra + assert not summary.settings.model_extra + assert not summary.ai_engine.model_extra assert not label_total.model_extra +def test_get_unstructured_discovery_summary_reads_a_run_without_settings_or_ai_engine( + client: DataMasqueClient, +) -> None: + with requests_mock.Mocker() as m: + m.get( + f"{RESULTS_URL}summary/", + json=_summary( + status="failed", + finished_at=None, + settings=None, + ai_engine=None, + report_rows=0, + labels=[], + ), + ) + summary = client.get_unstructured_discovery_summary(RunId(41)) + + assert summary.finished_at is None + assert summary.settings is None + assert summary.ai_engine is None + + +def test_get_unstructured_discovery_summary_reads_all_and_an_engine_without_a_version( + client: DataMasqueClient, +) -> None: + settings = {"files_per_directory": None, "sample_order": "name_asc", "files_per_run": None, "max_bytes_per_file": 5} + with requests_mock.Mocker() as m: + m.get(f"{RESULTS_URL}summary/", json=_summary(settings=settings, ai_engine={"model": "m", "version": None})) + summary = client.get_unstructured_discovery_summary(RunId(41)) + + assert summary.settings is not None + assert summary.settings.files_per_directory is None + assert summary.settings.files_per_run is None + assert summary.settings.sample_order is UnstructuredDiscoverySampleOrder.name_asc + assert summary.ai_engine is not None + assert summary.ai_engine.version is None + + def test_get_unstructured_report_returns_the_csv_text(client: DataMasqueClient) -> None: csv_text = ( "\ufeffpath,directory,label,description,occurrences,categories,character_count\r\n" @@ -386,8 +444,8 @@ def test_get_unstructured_report_returns_zip_bytes_when_split(client: DataMasque @pytest.mark.parametrize( "url, call", [ - (RESULTS_URL, lambda client: client.get_unstructured_discovery_folder_page(RunId(41))), - (RESULTS_URL, lambda client: client.get_unstructured_discovery_file_page(RunId(41), "")), + (FOLDERS_URL, lambda client: client.get_unstructured_discovery_folder_page(RunId(41))), + (FILES_URL, lambda client: client.get_unstructured_discovery_file_page(RunId(41), "")), (f"{RESULTS_URL}summary/", lambda client: client.get_unstructured_discovery_summary(RunId(41))), (REPORT_URL, lambda client: client.get_unstructured_report(RunId(41))), ], From 1e85ed96a9d9f82dc3b3bc801fb67e98119d88ca Mon Sep 17 00:00:00 2001 From: Timotius Date: Tue, 10 Nov 2026 20:46:59 +1300 Subject: [PATCH 21/23] refactor: DM-4756: name the report method for discovery --- datamasque/client/models/unstructured_discovery.py | 2 +- datamasque/client/unstructured_discovery.py | 2 +- tests/test_unstructured_discovery.py | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/datamasque/client/models/unstructured_discovery.py b/datamasque/client/models/unstructured_discovery.py index 68b6047..02d6575 100644 --- a/datamasque/client/models/unstructured_discovery.py +++ b/datamasque/client/models/unstructured_discovery.py @@ -180,7 +180,7 @@ class UnstructuredDiscoverySummary(BaseModel): report_rows: int """ How many rows the findings report has. - `get_unstructured_report` returns a zip of CSV parts above 1,000,000 rows. + `get_unstructured_discovery_report` returns a zip of CSV parts above 1,000,000 rows. """ files_found: int """How many files were found on the connection, summed over the folders.""" diff --git a/datamasque/client/unstructured_discovery.py b/datamasque/client/unstructured_discovery.py index e8d3961..41d7274 100644 --- a/datamasque/client/unstructured_discovery.py +++ b/datamasque/client/unstructured_discovery.py @@ -233,7 +233,7 @@ def get_unstructured_discovery_summary(self, run_id: RunId) -> UnstructuredDisco response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-discovery/summary/") return UnstructuredDiscoverySummary.model_validate(response.json()) - def get_unstructured_report(self, run_id: RunId) -> Union[str, bytes]: + def get_unstructured_discovery_report(self, run_id: RunId) -> Union[str, bytes]: """ Returns an unstructured discovery run's findings, one row for each label found in each file. diff --git a/tests/test_unstructured_discovery.py b/tests/test_unstructured_discovery.py index dfb94f9..9662844 100644 --- a/tests/test_unstructured_discovery.py +++ b/tests/test_unstructured_discovery.py @@ -417,7 +417,7 @@ def test_get_unstructured_discovery_summary_reads_all_and_an_engine_without_a_ve assert summary.ai_engine.version is None -def test_get_unstructured_report_returns_the_csv_text(client: DataMasqueClient) -> None: +def test_get_unstructured_discovery_report_returns_the_csv_text(client: DataMasqueClient) -> None: csv_text = ( "\ufeffpath,directory,label,description,occurrences,categories,character_count\r\n" "réunions/0142.pdf,réunions,EMAIL,Email address,4,PII;Contact,5120\r\n" @@ -430,15 +430,15 @@ def test_get_unstructured_report_returns_the_csv_text(client: DataMasqueClient) headers={"Content-Type": "text/csv; charset=utf-8"}, ) - assert client.get_unstructured_report(RunId(41)) == csv_text + assert client.get_unstructured_discovery_report(RunId(41)) == csv_text -def test_get_unstructured_report_returns_zip_bytes_when_split(client: DataMasqueClient) -> None: +def test_get_unstructured_discovery_report_returns_zip_bytes_when_split(client: DataMasqueClient) -> None: zip_bytes = b"PK\x03\x04 split report zip bytes" with requests_mock.Mocker() as m: m.get(REPORT_URL, content=zip_bytes, headers={"Content-Type": "application/zip"}) - assert client.get_unstructured_report(RunId(41)) == zip_bytes + assert client.get_unstructured_discovery_report(RunId(41)) == zip_bytes @pytest.mark.parametrize( @@ -447,7 +447,7 @@ def test_get_unstructured_report_returns_zip_bytes_when_split(client: DataMasque (FOLDERS_URL, lambda client: client.get_unstructured_discovery_folder_page(RunId(41))), (FILES_URL, lambda client: client.get_unstructured_discovery_file_page(RunId(41), "")), (f"{RESULTS_URL}summary/", lambda client: client.get_unstructured_discovery_summary(RunId(41))), - (REPORT_URL, lambda client: client.get_unstructured_report(RunId(41))), + (REPORT_URL, lambda client: client.get_unstructured_discovery_report(RunId(41))), ], ) def test_reading_a_run_that_is_not_unstructured_discovery_raises( From 8e076101f62c47db24e238601d3c2ad6d38b284c Mon Sep 17 00:00:00 2001 From: Timotius Date: Tue, 10 Nov 2026 20:47:12 +1300 Subject: [PATCH 22/23] docs: DM-4756: describe the split results API in the history --- HISTORY.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8aa0e1d..c98e85b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,14 +27,16 @@ History ``get_unstructured_discovery_file_page`` and ``iter_unstructured_discovery_files`` for the files of one folder, optionally narrowed by ``UnstructuredDiscoveryFileStatus``, ``search``, ``labels``, ``categories`` and ``UnstructuredDiscoveryFileOutcome``, - ``get_unstructured_discovery_summary`` for its totals and how many times it found each label, - and ``get_unstructured_report`` for its findings CSV, + ``get_unstructured_discovery_summary`` for the run's connection, status, end time, + ``UnstructuredDiscoverySettings``, ``UnstructuredDiscoveryAiEngine``, totals and how many times it found each label, + and ``get_unstructured_discovery_report`` for its findings CSV, or a zip of CSV parts as ``bytes`` for a report of more than 1,000,000 rows. - Each page is an ``UnstructuredDiscoveryPage``, - which also carries the run's connection, end time and ``UnstructuredDiscoverySampling``. + A skip reason is a code such as ``over_sample_limit``, with its sentence in ``skip_reason_text``. * Added ``get_last_unstructured_discovery_run_id``, - which returns the ID of a connection's newest finished unstructured discovery run, + which returns the ID of a connection's newest finished unstructured discovery run that is not a dry run, or ``None`` when it has none. +* Added ``discovery_kind`` to ``RunInfo``: ``DiscoveryKind.structured`` or ``DiscoveryKind.unstructured`` + for a file discovery run, ``None`` for a run that ran no discovery task. * Added ``ai_engine_model`` and ``ai_engine_version`` to ``RunInfo``: the AI Engine model and engine version the run's ``ai_detect`` matchers use, ``None`` when the run recorded none, such as a run that uses no AI Engine, From 8f8e55c244d0ef6f4ad657b2cedc6f267911c50c Mon Sep 17 00:00:00 2001 From: Timotius Date: Tue, 10 Nov 2026 20:52:31 +1300 Subject: [PATCH 23/23] docs: DM-4756: word the sample orders as the server does --- datamasque/client/models/unstructured_discovery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datamasque/client/models/unstructured_discovery.py b/datamasque/client/models/unstructured_discovery.py index 02d6575..e9732ef 100644 --- a/datamasque/client/models/unstructured_discovery.py +++ b/datamasque/client/models/unstructured_discovery.py @@ -50,11 +50,11 @@ class UnstructuredDiscoverySampleOrder(enum.Enum): """The order an unstructured discovery run takes a folder's files in when it reads only some of them.""" hash = "hash" - """An order that is fixed for each file but looks random.""" + """An order fixed by each file's path, so every run takes the same files.""" modified_desc = "modified_desc" """The most recently modified files first.""" name_asc = "name_asc" - """By name, ascending.""" + """By path, ascending.""" class UnstructuredDiscoveryUnreadCount(BaseModel):