diff --git a/HISTORY.rst b/HISTORY.rst index 191e613..c98e85b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,33 @@ History * ``RunInfo`` and ``UnfinishedRun`` gain a required ``project`` field: the run's project, taken from its ruleset's project when the run starts. * Added ``connect_timeout_seconds`` and ``read_timeout_seconds`` to the instance config models. +* Added optional keyword-only ``config_type`` and ``discovery_kind`` arguments to ``get_default_discovery_config_yaml``, + typed by ``DiscoveryConfigType`` and the new ``DiscoveryKind`` enum. + ``discovery_kind=DiscoveryKind.unstructured`` with ``config_type=DiscoveryConfigType.file`` + returns the template of an unstructured discovery config, + which counts the occurrences of its labels in the text of documents. +* Added methods that read an unstructured discovery run's results: + ``get_unstructured_discovery_folder_page`` and ``iter_unstructured_discovery_folders`` + for the folders found on the connection, + optionally narrowed by ``search``, ``labels`` and ``categories`` + and sorted by ``UnstructuredDiscoveryFolderOrdering``, + ``get_unstructured_discovery_file_page`` and ``iter_unstructured_discovery_files`` for the files of one folder, + optionally narrowed by ``UnstructuredDiscoveryFileStatus``, ``search``, ``labels``, ``categories`` + and ``UnstructuredDiscoveryFileOutcome``, + ``get_unstructured_discovery_summary`` for the run's connection, status, end time, + ``UnstructuredDiscoverySettings``, ``UnstructuredDiscoveryAiEngine``, totals and how many times it found each label, + and ``get_unstructured_discovery_report`` for its findings CSV, + or a zip of CSV parts as ``bytes`` for a report of more than 1,000,000 rows. + A skip reason is a code such as ``over_sample_limit``, with its sentence in ``skip_reason_text``. +* Added ``get_last_unstructured_discovery_run_id``, + which returns the ID of a connection's newest finished unstructured discovery run that is not a dry run, + or ``None`` when it has none. +* Added ``discovery_kind`` to ``RunInfo``: ``DiscoveryKind.structured`` or ``DiscoveryKind.unstructured`` + for a file discovery run, ``None`` for a run that ran no discovery task. +* Added ``ai_engine_model`` and ``ai_engine_version`` to ``RunInfo``: + the AI Engine model and engine version the run's ``ai_detect`` matchers use, + ``None`` when the run recorded none, such as a run that uses no AI Engine, + or on a server that does not report them. Requires server version 3.26.20 diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index 2122c6d..6e03e7c 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -89,7 +89,12 @@ TableConstraints, ValueCountStatus, ) -from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType +from datamasque.client.models.discovery_config import ( + DiscoveryConfig, + DiscoveryConfigId, + DiscoveryConfigType, + DiscoveryKind, +) from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary, DiscoveryConfigLibraryId from datamasque.client.models.dm_instance import DataMasqueInstanceConfig from datamasque.client.models.files import ( @@ -172,6 +177,20 @@ TableReferenceId, TableReferenceOptions, ) +from datamasque.client.models.unstructured_discovery import ( + UnstructuredDiscoveryAiEngine, + UnstructuredDiscoveryFile, + UnstructuredDiscoveryFileOutcome, + UnstructuredDiscoveryFileStatus, + UnstructuredDiscoveryFinding, + UnstructuredDiscoveryFolder, + UnstructuredDiscoveryFolderOrdering, + UnstructuredDiscoveryLabelTotal, + UnstructuredDiscoverySampleOrder, + UnstructuredDiscoverySettings, + UnstructuredDiscoverySummary, + UnstructuredDiscoveryUnreadCount, +) from datamasque.client.models.user import User, UserId, UserRole __version__ = version("datamasque-python") @@ -209,6 +228,7 @@ "DiscoveryConfigLibraryId", "DiscoveryConfigNotFoundError", "DiscoveryConfigType", + "DiscoveryKind", "DiscoveryMatch", "CosmosDbConnectionConfig", "DocumentDbConnectionConfig", @@ -323,6 +343,18 @@ "TemporalStatistics", "TemporalSummaries", "UnfinishedRun", + "UnstructuredDiscoveryFile", + "UnstructuredDiscoveryFileOutcome", + "UnstructuredDiscoveryFileStatus", + "UnstructuredDiscoveryFinding", + "UnstructuredDiscoveryFolder", + "UnstructuredDiscoveryFolderOrdering", + "UnstructuredDiscoveryLabelTotal", + "UnstructuredDiscoveryAiEngine", + "UnstructuredDiscoverySampleOrder", + "UnstructuredDiscoverySettings", + "UnstructuredDiscoverySummary", + "UnstructuredDiscoveryUnreadCount", "UnsupportedPreview", "UnsupportedStatistics", "User", diff --git a/datamasque/client/base.py b/datamasque/client/base.py index 673356d..f377106 100644 --- a/datamasque/client/base.py +++ b/datamasque/client/base.py @@ -313,6 +313,14 @@ def _delete_if_exists(self, path: str, *, params: Optional[dict] = None) -> None self._raise_for_status(response) + @staticmethod + def _read_csv_or_zip(response: Response) -> Union[str, bytes]: + """Returns a report's CSV text, or as `bytes` the zip of CSV parts the server splits a large report into.""" + + if response.headers.get("Content-Type", "").startswith("application/zip"): + return response.content + return response.text + def _iter_paginated( self, path: str, diff --git a/datamasque/client/discovery_configs.py b/datamasque/client/discovery_configs.py index d19f900..6196b8a 100644 --- a/datamasque/client/discovery_configs.py +++ b/datamasque/client/discovery_configs.py @@ -3,7 +3,12 @@ from datamasque.client.base import BaseClient from datamasque.client.exceptions import DataMasqueApiError, DataMasqueException -from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType +from datamasque.client.models.discovery_config import ( + DiscoveryConfig, + DiscoveryConfigId, + DiscoveryConfigType, + DiscoveryKind, +) from datamasque.client.models.pagination import Page logger = logging.getLogger(__name__) @@ -163,8 +168,28 @@ def delete_discovery_config_by_name_if_exists(self, name: str, config_type: Disc self.delete_discovery_config_by_id_if_exists(config.id) - def get_default_discovery_config_yaml(self) -> str: - """Returns the server's built-in default discovery configuration as a YAML string.""" + def get_default_discovery_config_yaml( + self, + *, + config_type: Optional[DiscoveryConfigType] = None, + discovery_kind: Optional[DiscoveryKind] = None, + ) -> str: + """ + Returns the server's built-in default discovery configuration as a YAML string. + + `config_type` selects the database or the file default; + the server returns the database default when it is omitted. + `discovery_kind` selects what a file config looks for; + the server returns the structured default when it is omitted. + Asking for `DiscoveryKind.unstructured` without `config_type=DiscoveryConfigType.file` + raises `DataMasqueApiError` with a 400 response. + """ + + params: dict[str, str] = {} + if config_type is not None: + params["config_type"] = config_type.value + if discovery_kind is not None: + params["discovery_kind"] = discovery_kind.value - response = self.make_request("GET", "/api/discovery/configs/defaults/") + response = self.make_request("GET", "/api/discovery/configs/defaults/", params=params or None) return response.content.decode("utf-8") diff --git a/datamasque/client/dmclient.py b/datamasque/client/dmclient.py index 7e8cc7f..4c38302 100644 --- a/datamasque/client/dmclient.py +++ b/datamasque/client/dmclient.py @@ -11,6 +11,7 @@ from datamasque.client.runs import RunClient from datamasque.client.settings import SettingsClient from datamasque.client.table_references import TableReferenceClient +from datamasque.client.unstructured_discovery import UnstructuredDiscoveryClient from datamasque.client.users import UserClient __all__ = ["DataMasqueClient", "FileOrContent", "UploadFile"] @@ -24,6 +25,7 @@ class DataMasqueClient( FileClient, RunClient, DiscoveryClient, + UnstructuredDiscoveryClient, DiscoveryConfigClient, DiscoveryConfigLibraryClient, TableReferenceClient, diff --git a/datamasque/client/models/discovery_config.py b/datamasque/client/models/discovery_config.py index 086257f..3c403b9 100644 --- a/datamasque/client/models/discovery_config.py +++ b/datamasque/client/models/discovery_config.py @@ -16,6 +16,13 @@ class DiscoveryConfigType(enum.Enum): file = "file" +class DiscoveryKind(enum.Enum): + """What a file discovery config looks for: sensitive locators, or its labels in the text of documents.""" + + structured = "structured" + unstructured = "unstructured" + + def unwrap_discovery_config_id(value: Any) -> Any: """ Coerce a `DiscoveryConfig` to its `id`; pass other values through unchanged. diff --git a/datamasque/client/models/runs.py b/datamasque/client/models/runs.py index 09bae55..936a005 100644 --- a/datamasque/client/models/runs.py +++ b/datamasque/client/models/runs.py @@ -7,6 +7,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from datamasque.client.models.connection import ConnectionConfig, ConnectionId, unwrap_connection_id +from datamasque.client.models.discovery_config import DiscoveryKind from datamasque.client.models.ruleset import Ruleset, RulesetId, unwrap_ruleset_id from datamasque.client.models.status import MaskingRunStatus @@ -148,6 +149,25 @@ class RunInfo(BaseModel): Taken from the ruleset's project when the run starts, and never changes after that. """ + ai_engine_model: Optional[str] = None + """ + The AI Engine model the run's `ai_detect` matchers use: + for a Local AI Engine, the name of its model file or folder, + and for a Bedrock AI Engine, the Bedrock model set in its configuration. + `None` when the run recorded none, such as a run that uses no AI Engine, + or on a server that does not report it. + """ + ai_engine_version: Optional[str] = None + """ + The version the AI Engine reported when the run started. + `None` when the run recorded none, such as a run that uses no AI Engine, + or on a server that does not report it. + """ + discovery_kind: Optional[DiscoveryKind] = None + """ + What a file discovery run looked for: sensitive locators, or labels in the text of documents. + `None` for a run that ran no discovery task, or on a server that does not report it. + """ @model_validator(mode="before") @classmethod diff --git a/datamasque/client/models/unstructured_discovery.py b/datamasque/client/models/unstructured_discovery.py new file mode 100644 index 0000000..e9732ef --- /dev/null +++ b/datamasque/client/models/unstructured_discovery.py @@ -0,0 +1,203 @@ +"""Typed response shapes for an unstructured discovery run's results.""" + +import enum +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel, ConfigDict + +from datamasque.client.models.runs import RunConnectionRef, RunId + + +class UnstructuredDiscoveryFileStatus(enum.Enum): + """Whether an unstructured discovery run read a file's text.""" + + read = "read" + """The run read the file; its `character_count` is 0 when it held no text.""" + failed = "failed" + """The run tried to read the file and could not; `skip_reason` says why.""" + + +class UnstructuredDiscoveryFileOutcome(enum.Enum): + """What an unstructured discovery run made of a file it tried to read.""" + + findings = "findings" + """The run found at least one label in the file.""" + no_text = "no_text" + """The run read the file and found no text in it.""" + nothing_found = "nothing_found" + """The run read the file's text and found no label in it.""" + failed = "failed" + """The run tried to read the file and could not.""" + + +class UnstructuredDiscoveryFolderOrdering(enum.Enum): + """A column the folders of an unstructured discovery run are sorted by; folders that tie are sorted by path.""" + + directory = "directory" + """The folder's path.""" + files_present = "files_present" + files_read = "files_read" + size_bytes = "size_bytes" + occurrences = "occurrences" + """ + The sum of the folder's `entity_counts`. + With a `labels` or `categories` filter, only the matching labels count. + """ + + +class UnstructuredDiscoverySampleOrder(enum.Enum): + """The order an unstructured discovery run takes a folder's files in when it reads only some of them.""" + + hash = "hash" + """An order fixed by each file's path, so every run takes the same files.""" + modified_desc = "modified_desc" + """The most recently modified files first.""" + name_asc = "name_asc" + """By path, ascending.""" + + +class UnstructuredDiscoveryUnreadCount(BaseModel): + """How many of a folder's files the run did not try to read for one reason, at one extension.""" + + model_config = ConfigDict(extra="allow") + + # A string rather than an enum, because each server release can add reasons. + skip_reason: str + """Why the run did not try to read these files, as a code such as `over_sample_limit`.""" + skip_reason_text: str + """The same reason as a sentence, as the run report words it.""" + file_extension: str + """The extension without a leading dot, blank for files that have none.""" + count: int + + +class UnstructuredDiscoveryFolder(BaseModel): + """One folder an unstructured discovery run found on the connection, and what its files add up to.""" + + model_config = ConfigDict(extra="allow") + + directory: str + """The folder's path, blank at the connection root.""" + files_present: int + """How many files were found in the folder, read or not.""" + files_read: int + """How many of the folder's files the run read, including those with no text but not those it could not read.""" + size_bytes: int + """The total size, in bytes, of the folder's files, read or not.""" + entity_counts: dict[str, int] + """Each label's occurrences, summed over the folder's files.""" + unread: list[UnstructuredDiscoveryUnreadCount] + """How many of the folder's files the run did not try to read, by reason and extension.""" + + +class UnstructuredDiscoveryFinding(BaseModel): + """How many times one label was found in one file.""" + + model_config = ConfigDict(extra="allow") + + label: str + """ + The label's `name` in the discovery config. + `get_unstructured_discovery_summary` gives its description and categories. + """ + occurrences: int + + +class UnstructuredDiscoveryFile(BaseModel): + """One file an unstructured discovery run tried to read, and how many times each label was found in it.""" + + model_config = ConfigDict(extra="allow") + + path: str + file_extension: str + size_bytes: int + character_count: Optional[int] = None + """How many characters of the file's text the run searched; `None` for a file the run could not read.""" + status: UnstructuredDiscoveryFileStatus + # A string rather than an enum, because each server release can add reasons. + skip_reason: str + """Why the run could not read the file, as a code such as `unscannable_file`; blank if it read it.""" + skip_reason_text: str + """The same reason as a sentence, as the run report words it; blank if the run read the file.""" + findings: list[UnstructuredDiscoveryFinding] + + +class UnstructuredDiscoverySettings(BaseModel): + """The sampling and size settings an unstructured discovery run used.""" + + model_config = ConfigDict(extra="allow") + + files_per_directory: Optional[int] = None + """How many files of each folder the run read at most, `None` when it read every file.""" + sample_order: UnstructuredDiscoverySampleOrder + files_per_run: Optional[int] = None + """How many files the run read at most in all, `None` when it set no limit.""" + max_bytes_per_file: int + """The size, in bytes, above which the run did not read a file.""" + + +class UnstructuredDiscoveryAiEngine(BaseModel): + """The AI Engine an unstructured discovery run's `ai_detect` matchers used.""" + + model_config = ConfigDict(extra="allow") + + model: str + version: Optional[str] = None + """The version the AI Engine reported, `None` when it reported none.""" + + +class UnstructuredDiscoveryLabelTotal(BaseModel): + """How many times one label was found over an unstructured discovery run's files.""" + + model_config = ConfigDict(extra="allow") + + label: str + """The label's `name` in the discovery config.""" + description: str + categories: list[str] + occurrences: int + """How many times the label was found, summed over the files.""" + files: int + """How many files hold the label.""" + + +class UnstructuredDiscoverySummary(BaseModel): + """An unstructured discovery run, the settings it used and its totals.""" + + model_config = ConfigDict(extra="allow") + + run_id: RunId + connection: RunConnectionRef + status: str + """The run's status, such as `finished`.""" + finished_at: Optional[datetime] = None + """When the run ended, `None` until it has.""" + settings: Optional[UnstructuredDiscoverySettings] = None + """The settings the run used, `None` when it ended before it read any folder.""" + ai_engine: Optional[UnstructuredDiscoveryAiEngine] = None + """The AI Engine the run used, `None` when it used none.""" + report_rows: int + """ + How many rows the findings report has. + `get_unstructured_discovery_report` returns a zip of CSV parts above 1,000,000 rows. + """ + files_found: int + """How many files were found on the connection, summed over the folders.""" + files_read: int + """How many files the run read, including those with no text but not those it could not read.""" + files_failed: int + """How many files the run tried to read and could not.""" + files_without_text: int + """How many of the files the run read held no text to search.""" + files_with_findings: int + """How many of the files the run read hold a finding.""" + bytes_opened: int + """The total size, in bytes, of the files the run read.""" + files_over_run_limit: int + """How many files the run did not read because it reached the config's `files_per_run`.""" + labels: list[UnstructuredDiscoveryLabelTotal] + """ + Each label of the run's discovery config, most found first; labels found equally often are listed by name. + A label the run never found is listed with 0 occurrences. + """ diff --git a/datamasque/client/runs.py b/datamasque/client/runs.py index 9f4c87c..1b85d1d 100644 --- a/datamasque/client/runs.py +++ b/datamasque/client/runs.py @@ -59,9 +59,7 @@ def get_db_discovery_result_report(self, run_id: RunId, include_selection_column params = None if include_selection_column else {"include_selection_column": "false"} response = self.make_request("GET", url, params=params) - if response.headers.get("Content-Type", "").startswith("application/zip"): - return response.content - return response.text + return self._read_csv_or_zip(response) def get_unfinished_runs(self) -> dict[str, UnfinishedRun]: """Queries the DM instance for unfinished runs, and returns them organised by connection name.""" diff --git a/datamasque/client/unstructured_discovery.py b/datamasque/client/unstructured_discovery.py new file mode 100644 index 0000000..41d7274 --- /dev/null +++ b/datamasque/client/unstructured_discovery.py @@ -0,0 +1,253 @@ +from typing import Iterator, Optional, Union + +from datamasque.client.base import BaseClient +from datamasque.client.models.connection import ConnectionId +from datamasque.client.models.discovery_config import DiscoveryKind +from datamasque.client.models.pagination import Page +from datamasque.client.models.runs import RunId +from datamasque.client.models.status import MaskingRunStatus +from datamasque.client.models.unstructured_discovery import ( + UnstructuredDiscoveryFile, + UnstructuredDiscoveryFileOutcome, + UnstructuredDiscoveryFileStatus, + UnstructuredDiscoveryFolder, + UnstructuredDiscoveryFolderOrdering, + UnstructuredDiscoverySummary, +) + +SERVER_MAX_PAGE_SIZE = 1000 +LIST_SEPARATOR = "," +DESCENDING_PREFIX = "-" + + +class UnstructuredDiscoveryClient(BaseClient): + """ + Unstructured discovery result API methods. Mixed into `DataMasqueClient`. + + Each method that takes a run ID raises `DataMasqueApiError` with a 404 response + when the run is not an unstructured discovery run. + """ + + @staticmethod + def _build_label_filter_params( + search: Optional[str], labels: Optional[list[str]], categories: Optional[list[str]] + ) -> dict[str, str]: + params = {} + if search is not None: + params["search"] = search + if labels is not None: + params["labels"] = LIST_SEPARATOR.join(labels) + if categories is not None: + params["categories"] = LIST_SEPARATOR.join(categories) + + return params + + @staticmethod + def _build_unstructured_folder_page_params( + search: Optional[str], + labels: Optional[list[str]], + categories: Optional[list[str]], + ordering: UnstructuredDiscoveryFolderOrdering, + is_descending: bool, + ) -> dict[str, str]: + params = UnstructuredDiscoveryClient._build_label_filter_params(search, labels, categories) + params["ordering"] = f"{DESCENDING_PREFIX}{ordering.value}" if is_descending else ordering.value + return params + + @staticmethod + def _build_unstructured_file_page_params( + directory: str, + status: Optional[UnstructuredDiscoveryFileStatus], + search: Optional[str], + labels: Optional[list[str]], + categories: Optional[list[str]], + outcomes: Optional[list[UnstructuredDiscoveryFileOutcome]], + ) -> dict[str, str]: + params = {"directory": directory} + if status is not None: + params["status"] = status.value + params |= UnstructuredDiscoveryClient._build_label_filter_params(search, labels, categories) + if outcomes is not None: + params["outcomes"] = LIST_SEPARATOR.join(outcome.value for outcome in outcomes) + + return params + + def get_last_unstructured_discovery_run_id(self, connection_id: ConnectionId) -> Optional[RunId]: + """ + Returns the ID of a connection's newest finished unstructured discovery run, with or without warnings. + + Returns `None` when no unstructured discovery run on the connection has finished. + A dry run stores no results, so it is never returned. + """ + + params = { + "discovery_kind": DiscoveryKind.unstructured.value, + "source_connection": connection_id, + "dry_run": "false", + "run_status": [MaskingRunStatus.finished.value, MaskingRunStatus.finished_with_warnings.value], + "ordering": "-id", + "limit": 1, + } + response = self.make_request("GET", "/api/runs/", params=params) + runs = response.json()["results"] + return RunId(runs[0]["id"]) if runs else None + + def get_unstructured_discovery_folder_page( + self, + run_id: RunId, + *, + search: Optional[str] = None, + labels: Optional[list[str]] = None, + categories: Optional[list[str]] = None, + ordering: UnstructuredDiscoveryFolderOrdering = UnstructuredDiscoveryFolderOrdering.directory, + descending: bool = False, + limit: int = 50, + offset: int = 0, + ) -> Page[UnstructuredDiscoveryFolder]: + """ + Returns one page of the folders an unstructured discovery run found on the connection. + + `search` keeps the folders whose path contains it, ignoring case, + and the folders holding a file the run tried to read whose name contains it. + `labels` keeps the folders holding any of those labels, + and `categories` the folders holding a label that carries any of those categories; + given both, a label must match both. + Files with no text and files the run could not read have no findings, + so neither filter keeps a folder for them. + With `labels` or `categories`, a file matches `search` only if it holds a matching label, + or the run read it and found no text in it; + a file the run could not read never matches. + Each folder's `entity_counts` still counts every label it holds. + The folders come in `ordering`, descending when `descending` is set, + and folders that tie are sorted by path, ascending. + """ + + params = self._build_unstructured_folder_page_params(search, labels, categories, ordering, descending) | { + "limit": limit, + "offset": offset, + } + response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-discovery/directories/", params=params) + return Page[UnstructuredDiscoveryFolder].model_validate(response.json()) + + # `iter_unstructured_discovery_folders` and `iter_unstructured_discovery_files` have no `list_` counterparts, + # because a run's folder and file counts are unbounded. + def iter_unstructured_discovery_folders( + self, + run_id: RunId, + *, + search: Optional[str] = None, + labels: Optional[list[str]] = None, + categories: Optional[list[str]] = None, + ordering: UnstructuredDiscoveryFolderOrdering = UnstructuredDiscoveryFolderOrdering.directory, + descending: bool = False, + ) -> Iterator[UnstructuredDiscoveryFolder]: + """ + Lazily iterate every folder an unstructured discovery run found on the connection. + + `search`, `labels`, `categories`, `ordering` and `descending` + are as for `get_unstructured_discovery_folder_page`. + """ + + return self._iter_paginated( + f"/api/runs/{run_id}/unstructured-discovery/directories/", + model=UnstructuredDiscoveryFolder, + params=self._build_unstructured_folder_page_params(search, labels, categories, ordering, descending), + page_size=SERVER_MAX_PAGE_SIZE, + ) + + def get_unstructured_discovery_file_page( + self, + run_id: RunId, + directory: str, + *, + status: Optional[UnstructuredDiscoveryFileStatus] = None, + search: Optional[str] = None, + labels: Optional[list[str]] = None, + categories: Optional[list[str]] = None, + outcomes: Optional[list[UnstructuredDiscoveryFileOutcome]] = None, + limit: int = 50, + offset: int = 0, + ) -> Page[UnstructuredDiscoveryFile]: + """ + Returns one page of the files an unstructured discovery run tried to read in one folder. + + `directory` is a folder's `directory` exactly as the folder pages name it; + an empty string names the connection root. + `status` keeps only the files with that status, + and `outcomes` only the files with any of those outcomes. + `search` keeps the files whose name contains it, ignoring case, + or every file when the folder's path contains it. + `labels` keeps the files holding any of those labels, + and `categories` the files holding a label that carries any of those categories; + given both, a label must match both. + Either one also keeps the files the run read and found no text in, which may hold any label, + and drops the files the run could not read. + The files with findings come first, most occurrences first, + then the files with no text, those with nothing found and those the run could not read, + each group sorted by name. + With `labels` or `categories`, a file's occurrences count only the matching labels. + Files the run did not try to read have no row; + the folder's `unread` counts them. + """ + + params = self._build_unstructured_file_page_params(directory, status, search, labels, categories, outcomes) | { + "limit": limit, + "offset": offset, + } + response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-discovery/files/", params=params) + return Page[UnstructuredDiscoveryFile].model_validate(response.json()) + + def iter_unstructured_discovery_files( + self, + run_id: RunId, + directory: str, + *, + status: Optional[UnstructuredDiscoveryFileStatus] = None, + search: Optional[str] = None, + labels: Optional[list[str]] = None, + categories: Optional[list[str]] = None, + outcomes: Optional[list[UnstructuredDiscoveryFileOutcome]] = None, + ) -> Iterator[UnstructuredDiscoveryFile]: + """ + Lazily iterate every file an unstructured discovery run tried to read in one folder. + + `directory`, `status`, `search`, `labels`, `categories` and `outcomes` + are as for `get_unstructured_discovery_file_page`, and so is the order of the files. + """ + + return self._iter_paginated( + f"/api/runs/{run_id}/unstructured-discovery/files/", + model=UnstructuredDiscoveryFile, + params=self._build_unstructured_file_page_params(directory, status, search, labels, categories, outcomes), + page_size=SERVER_MAX_PAGE_SIZE, + ) + + def get_unstructured_discovery_summary(self, run_id: RunId) -> UnstructuredDiscoverySummary: + """ + Returns an unstructured discovery run, the settings it used, its totals, and how many times it found each label. + + They are counted from the run's stored results on each request, + so they change while results are still being stored. + """ + + response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-discovery/summary/") + return UnstructuredDiscoverySummary.model_validate(response.json()) + + def get_unstructured_discovery_report(self, run_id: RunId) -> Union[str, bytes]: + """ + Returns an unstructured discovery run's findings, one row for each label found in each file. + + A file the run read and found nothing in has one row, + with a blank label, description and categories, and 0 occurrences. + A file the run could not read has no row. + Every row carries its file's `character_count`, + how many characters of the file's text the run searched. + Returns CSV text (`str`) that starts with a byte-order mark, + or a zip of numbered CSV parts as `bytes` when the server splits a report of more than 1,000,000 rows. + The first row of each CSV names the columns: + `path`, `directory`, `label`, `description`, `occurrences`, `categories` and `character_count`. + A label's categories share one cell, joined with `;`. + """ + + response = self.make_request("GET", f"/api/runs/{run_id}/unstructured-discovery/report/") + return self._read_csv_or_zip(response) diff --git a/docs/client.models.rst b/docs/client.models.rst index e98789b..37dcb1f 100644 --- a/docs/client.models.rst +++ b/docs/client.models.rst @@ -28,6 +28,14 @@ datamasque.client.models.discovery module :undoc-members: :show-inheritance: +datamasque.client.models.discovery_config module +------------------------------------------------ + +.. automodule:: datamasque.client.models.discovery_config + :members: + :undoc-members: + :show-inheritance: + datamasque.client.models.dm_instance module ------------------------------------------- @@ -100,6 +108,14 @@ datamasque.client.models.status module :undoc-members: :show-inheritance: +datamasque.client.models.unstructured_discovery module +------------------------------------------------------ + +.. automodule:: datamasque.client.models.unstructured_discovery + :members: + :undoc-members: + :show-inheritance: + datamasque.client.models.user module ------------------------------------ diff --git a/docs/client.rst b/docs/client.rst index f013c64..b3dc81d 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -36,6 +36,14 @@ datamasque.client.discovery module :undoc-members: :show-inheritance: +datamasque.client.discovery_configs module +------------------------------------------ + +.. automodule:: datamasque.client.discovery_configs + :members: + :undoc-members: + :show-inheritance: + datamasque.client.dmclient module --------------------------------- @@ -124,6 +132,14 @@ datamasque.client.settings module :undoc-members: :show-inheritance: +datamasque.client.unstructured_discovery module +----------------------------------------------- + +.. automodule:: datamasque.client.unstructured_discovery + :members: + :undoc-members: + :show-inheritance: + datamasque.client.users module ------------------------------ diff --git a/tests/test_discovery_configs.py b/tests/test_discovery_configs.py index d34c47d..70be617 100644 --- a/tests/test_discovery_configs.py +++ b/tests/test_discovery_configs.py @@ -13,6 +13,7 @@ DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType, + DiscoveryKind, unwrap_discovery_config_id, ) from datamasque.client.models.status import ValidationStatus @@ -478,6 +479,61 @@ def test_get_default_discovery_config_yaml(client: DataMasqueClient) -> None: result = client.get_default_discovery_config_yaml() assert result == yaml_body + assert m.last_request.qs == {} + + +def test_get_default_discovery_config_yaml_asks_for_the_type_and_kind(client: DataMasqueClient) -> None: + yaml_body = "discovery_kind: unstructured\nlabels: []\n" + + with requests_mock.Mocker(case_sensitive=True) as m: + m.get( + "http://test-server/api/discovery/configs/defaults/", + text=yaml_body, + status_code=200, + headers={"Content-Type": "application/x-yaml"}, + ) + result = client.get_default_discovery_config_yaml( + config_type=DiscoveryConfigType.file, discovery_kind=DiscoveryKind.unstructured + ) + + assert result == yaml_body + assert m.last_request.qs == {"config_type": ["file"], "discovery_kind": ["unstructured"]} + + +@pytest.mark.parametrize( + "kwargs, expected_query", + [ + ({"config_type": DiscoveryConfigType.database}, {"config_type": ["database"]}), + ({"discovery_kind": DiscoveryKind.structured}, {"discovery_kind": ["structured"]}), + ], +) +def test_get_default_discovery_config_yaml_sends_only_the_parameters_given( + client: DataMasqueClient, kwargs: dict[str, Any], expected_query: dict[str, list[str]] +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get("http://test-server/api/discovery/configs/defaults/", text="labels: []\n", status_code=200) + client.get_default_discovery_config_yaml(**kwargs) + + assert m.last_request.qs == expected_query + + +def test_get_default_discovery_config_yaml_raises_when_the_server_refuses_the_kind(client: DataMasqueClient) -> None: + refusal = {"discovery_kind": "`discovery_kind` `unstructured` is available for a `file` config only."} + with requests_mock.Mocker(case_sensitive=True) as m: + m.get("http://test-server/api/discovery/configs/defaults/", json=refusal, status_code=400) + with pytest.raises(DataMasqueApiError) as exc_info: + client.get_default_discovery_config_yaml( + config_type=DiscoveryConfigType.database, discovery_kind=DiscoveryKind.unstructured + ) + + assert m.last_request.qs == {"config_type": ["database"], "discovery_kind": ["unstructured"]} + assert exc_info.value.response.status_code == 400 + assert exc_info.value.response.json() == refusal + + +def test_default_discovery_config_parameters_are_keyword_only(client: DataMasqueClient) -> None: + with pytest.raises(TypeError, match="takes 1 positional argument but 2 were given"): + client.get_default_discovery_config_yaml(DiscoveryConfigType.file) def test_discovery_config_parses_validation_fields() -> None: diff --git a/tests/test_runs.py b/tests/test_runs.py index 295409a..d2fab2a 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -1,5 +1,7 @@ """Tests for `RunClient` (start, status, log, run-report endpoints).""" +from typing import Optional + import pytest import requests_mock @@ -11,6 +13,7 @@ RunNotCancellableError, ) from datamasque.client.models.connection import ConnectionId, DatabaseConnectionConfig, DatabaseType +from datamasque.client.models.discovery_config import DiscoveryKind from datamasque.client.models.ruleset import Ruleset, RulesetId, RulesetType from datamasque.client.models.runs import ( MaskingRunOptions, @@ -340,6 +343,66 @@ def test_get_run_info(client): assert result.name == "r1" +@pytest.mark.parametrize( + "ai_engine_fields, expected_model, expected_version", + [ + ( + {"ai_engine_model": "dm-ai-ner-small-1.0.0-20260601.tar.enc", "ai_engine_version": "1.26.11"}, + "dm-ai-ner-small-1.0.0-20260601.tar.enc", + "1.26.11", + ), + ({"ai_engine_model": None, "ai_engine_version": None}, None, None), + ({}, None, None), + ], + ids=["ai_detect_run", "run_without_ai_detect", "server_without_the_fields"], +) +def test_run_info_reads_the_ai_engine_model_and_version( + ai_engine_fields: dict, expected_model: Optional[str], expected_version: Optional[str] +) -> None: + run = RunInfo.model_validate( + { + "id": 1, + "status": "finished", + "mask_type": "file_data_discovery", + "source_connection_name": "documents", + "ruleset_name": "", + "project": "1-default", + **ai_engine_fields, + } + ) + + assert run.ai_engine_model == expected_model + assert run.ai_engine_version == expected_version + assert not run.model_extra + + +@pytest.mark.parametrize( + "discovery_kind_field, expected", + [ + ({"discovery_kind": "unstructured"}, DiscoveryKind.unstructured), + ({"discovery_kind": "structured"}, DiscoveryKind.structured), + ({"discovery_kind": None}, None), + ({}, None), + ], + ids=["unstructured_discovery_run", "structured_discovery_run", "masking_run", "server_without_the_field"], +) +def test_run_info_reads_the_discovery_kind(discovery_kind_field: dict, expected: Optional[DiscoveryKind]) -> None: + run = RunInfo.model_validate( + { + "id": 1, + "status": "finished", + "mask_type": "file_data_discovery", + "source_connection_name": "documents", + "ruleset_name": "", + "project": "1-default", + **discovery_kind_field, + } + ) + + assert run.discovery_kind is expected + assert not run.model_extra + + def test_masking_run_request_model_dump_minimal(): """A minimal request dumps with an empty `options` object (the server rejects missing `options`).""" req = MaskingRunRequest(connection="conn-1", ruleset="rs-1") diff --git a/tests/test_unstructured_discovery.py b/tests/test_unstructured_discovery.py new file mode 100644 index 0000000..9662844 --- /dev/null +++ b/tests/test_unstructured_discovery.py @@ -0,0 +1,461 @@ +"""Tests for reading an unstructured discovery run's results.""" + +from collections.abc import Callable +from datetime import datetime, timezone +from typing import Any, Optional + +import pytest +import requests_mock + +from datamasque.client import DataMasqueClient +from datamasque.client.exceptions import DataMasqueApiError +from datamasque.client.models.connection import ConnectionId +from datamasque.client.models.runs import RunId +from datamasque.client.models.unstructured_discovery import ( + UnstructuredDiscoveryFileOutcome, + UnstructuredDiscoveryFileStatus, + UnstructuredDiscoveryFolderOrdering, + UnstructuredDiscoverySampleOrder, +) + +RESULTS_URL = "http://test-server/api/runs/41/unstructured-discovery/" +FOLDERS_URL = f"{RESULTS_URL}directories/" +FILES_URL = f"{RESULTS_URL}files/" +REPORT_URL = f"{RESULTS_URL}report/" +RUNS_URL = "http://test-server/api/runs/" +CONNECTION_ID = "aaaaaaaa-1111-2222-3333-444444444444" + +ClientCall = Callable[[DataMasqueClient], object] + + +def _page(results: list[dict], **overrides: Any) -> dict: + return {"count": len(results), "next": None, "previous": None, "results": results, **overrides} + + +def _folder(directory: str) -> dict: + return { + "directory": directory, + "files_present": 340, + "files_read": 20, + "size_bytes": 14399, + "entity_counts": {"CREDIT_CARD": 3, "EMAIL": 62}, + "unread": [ + { + "skip_reason": "over_sample_limit", + "skip_reason_text": "Not selected by the per-folder sample", + "file_extension": "pdf", + "count": 318, + } + ], + } + + +def _file( + path: str, + *, + status: str = "read", + skip_reason: str = "", + skip_reason_text: str = "", + character_count: Optional[int] = 5120, + findings: Optional[list[dict]] = None, +) -> dict: + return { + "path": path, + "file_extension": "pdf", + "size_bytes": 204817, + "character_count": character_count, + "status": status, + "skip_reason": skip_reason, + "skip_reason_text": skip_reason_text, + "findings": findings if findings is not None else [], + } + + +@pytest.mark.parametrize( + "runs, expected", + [([{"id": 19, "status": "finished"}], RunId(19)), ([], None)], + ids=["finished_run", "no_finished_run"], +) +def test_get_last_unstructured_discovery_run_id_asks_the_run_list( + client: DataMasqueClient, runs: list[dict], expected: Optional[RunId] +) -> None: + with requests_mock.Mocker() as m: + m.get(RUNS_URL, json=_page(runs)) + + assert client.get_last_unstructured_discovery_run_id(ConnectionId(CONNECTION_ID)) == expected + + assert m.last_request.qs == { + "discovery_kind": ["unstructured"], + "source_connection": [CONNECTION_ID], + "dry_run": ["false"], + "run_status": ["finished", "finished_with_warnings"], + "ordering": ["-id"], + "limit": ["1"], + } + + +def test_get_unstructured_discovery_folder_page_parses_the_folders(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get(FOLDERS_URL, json=_page([_folder("finance/invoices")])) + page = client.get_unstructured_discovery_folder_page(RunId(41), limit=10, offset=20) + + assert m.last_request.qs == {"ordering": ["directory"], "limit": ["10"], "offset": ["20"]} + assert page.count == 1 + assert not page.model_extra + + folder = page.results[0] + assert folder.directory == "finance/invoices" + assert folder.files_present == 340 + assert folder.files_read == 20 + assert folder.size_bytes == 14399 + assert folder.entity_counts == {"CREDIT_CARD": 3, "EMAIL": 62} + assert folder.unread[0].skip_reason == "over_sample_limit" + assert folder.unread[0].skip_reason_text == "Not selected by the per-folder sample" + assert folder.unread[0].file_extension == "pdf" + assert folder.unread[0].count == 318 + assert not folder.unread[0].model_extra + + +def test_get_unstructured_discovery_folder_page_asks_for_the_first_page_by_default(client: DataMasqueClient) -> None: + with requests_mock.Mocker() as m: + m.get(FOLDERS_URL, json=_page([])) + page = client.get_unstructured_discovery_folder_page(RunId(41)) + + assert m.last_request.qs == {"ordering": ["directory"], "limit": ["50"], "offset": ["0"]} + assert page.results == [] + + +@pytest.mark.parametrize( + "call, expected_limit", + [ + (lambda client, **filters: client.get_unstructured_discovery_folder_page(RunId(41), **filters), "50"), + (lambda client, **filters: list(client.iter_unstructured_discovery_folders(RunId(41), **filters)), "1000"), + ], + ids=["page", "iterator"], +) +def test_unstructured_discovery_folders_send_the_search_and_label_filters( + client: DataMasqueClient, call: Callable[..., object], expected_limit: str +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(FOLDERS_URL, json=_page([_folder("Finance/Invoices")])) + call(client, search="Invoices", labels=["EMAIL", "CREDIT_CARD"], categories=["PII", "Financial"]) + + assert m.last_request.qs == { + "search": ["Invoices"], + "labels": ["EMAIL,CREDIT_CARD"], + "categories": ["PII,Financial"], + "ordering": ["directory"], + "limit": [expected_limit], + "offset": ["0"], + } + + +@pytest.mark.parametrize( + "ordering, descending, expected_ordering", + [ + (UnstructuredDiscoveryFolderOrdering.directory, True, "-directory"), + (UnstructuredDiscoveryFolderOrdering.files_present, False, "files_present"), + (UnstructuredDiscoveryFolderOrdering.files_read, True, "-files_read"), + (UnstructuredDiscoveryFolderOrdering.size_bytes, False, "size_bytes"), + (UnstructuredDiscoveryFolderOrdering.occurrences, True, "-occurrences"), + ], +) +@pytest.mark.parametrize( + "call", + [ + lambda client, **ordering: client.get_unstructured_discovery_folder_page(RunId(41), **ordering), + lambda client, **ordering: list(client.iter_unstructured_discovery_folders(RunId(41), **ordering)), + ], + ids=["page", "iterator"], +) +def test_unstructured_discovery_folders_send_the_ordering( + client: DataMasqueClient, + call: Callable[..., object], + ordering: UnstructuredDiscoveryFolderOrdering, + descending: bool, + expected_ordering: str, +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(FOLDERS_URL, json=_page([_folder("Finance/Invoices")])) + call(client, ordering=ordering, descending=descending) + + assert m.last_request.qs["ordering"] == [expected_ordering] + + +def test_get_unstructured_discovery_file_page_parses_the_files_and_their_findings(client: DataMasqueClient) -> None: + finding = {"label": "EMAIL", "occurrences": 4} + rows = [ + _file("Finance/Invoices/0142.pdf", findings=[finding]), + _file( + "Finance/Invoices/0143.pdf", + status="failed", + skip_reason="unscannable_file", + skip_reason_text="File could not be scanned", + character_count=None, + ), + ] + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(FILES_URL, json=_page(rows)) + page = client.get_unstructured_discovery_file_page( + RunId(41), "Finance/Invoices", status=UnstructuredDiscoveryFileStatus.read, limit=1000, offset=5 + ) + + assert m.last_request.qs == { + "directory": ["Finance/Invoices"], + "status": ["read"], + "limit": ["1000"], + "offset": ["5"], + } + read_file, failed_file = page.results + assert read_file.path == "Finance/Invoices/0142.pdf" + assert read_file.file_extension == "pdf" + assert read_file.size_bytes == 204817 + assert read_file.character_count == 5120 + assert read_file.status is UnstructuredDiscoveryFileStatus.read + assert read_file.skip_reason == "" + assert read_file.skip_reason_text == "" + assert read_file.findings[0].label == "EMAIL" + assert read_file.findings[0].occurrences == 4 + assert not read_file.findings[0].model_extra + assert failed_file.status is UnstructuredDiscoveryFileStatus.failed + assert failed_file.skip_reason == "unscannable_file" + assert failed_file.skip_reason_text == "File could not be scanned" + assert failed_file.character_count is None + assert failed_file.findings == [] + assert not read_file.model_extra + assert not failed_file.model_extra + + +def test_get_unstructured_discovery_file_page_sends_a_blank_directory_for_the_root(client: DataMasqueClient) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(FILES_URL, json=_page([_file("notes.pdf")])) + page = client.get_unstructured_discovery_file_page(RunId(41), "") + + assert m.last_request.qs == {"directory": [""], "limit": ["50"], "offset": ["0"]} + assert page.results[0].path == "notes.pdf" + + +def test_iter_unstructured_discovery_folders_follows_the_next_page(client: DataMasqueClient) -> None: + next_url = f"{FOLDERS_URL}?limit=1000&offset=1000" + with requests_mock.Mocker() as m: + m.get(FOLDERS_URL, [{"json": _page([_folder("a")], next=next_url)}, {"json": _page([_folder("b")])}]) + folders = list(client.iter_unstructured_discovery_folders(RunId(41))) + + assert [folder.directory for folder in folders] == ["a", "b"] + assert m.request_history[0].qs == {"ordering": ["directory"], "limit": ["1000"], "offset": ["0"]} + assert m.request_history[1].url == next_url + + +@pytest.mark.parametrize( + "status, expected_status_query", + [(None, {}), (UnstructuredDiscoveryFileStatus.failed, {"status": ["failed"]})], +) +def test_iter_unstructured_discovery_files_asks_for_one_folder( + client: DataMasqueClient, + status: Optional[UnstructuredDiscoveryFileStatus], + expected_status_query: dict[str, list[str]], +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(FILES_URL, json=_page([_file("claims/1.pdf"), _file("claims/2.pdf")])) + files = list(client.iter_unstructured_discovery_files(RunId(41), "claims", status=status)) + + assert [file.path for file in files] == ["claims/1.pdf", "claims/2.pdf"] + assert m.last_request.qs == { + "directory": ["claims"], + "limit": ["1000"], + "offset": ["0"], + **expected_status_query, + } + + +@pytest.mark.parametrize( + "call, expected_limit", + [ + (lambda client, **filters: client.get_unstructured_discovery_file_page(RunId(41), "claims", **filters), "50"), + ( + lambda client, **filters: list(client.iter_unstructured_discovery_files(RunId(41), "claims", **filters)), + "1000", + ), + ], + ids=["page", "iterator"], +) +def test_unstructured_discovery_files_send_the_search_label_and_outcome_filters( + client: DataMasqueClient, call: Callable[..., object], expected_limit: str +) -> None: + with requests_mock.Mocker(case_sensitive=True) as m: + m.get(FILES_URL, json=_page([_file("claims/Claim-1.pdf")])) + call( + client, + search="Claim", + labels=["EMAIL", "CREDIT_CARD"], + categories=["PII", "Financial"], + outcomes=[UnstructuredDiscoveryFileOutcome.no_text, UnstructuredDiscoveryFileOutcome.failed], + ) + + assert m.last_request.qs == { + "directory": ["claims"], + "search": ["Claim"], + "labels": ["EMAIL,CREDIT_CARD"], + "categories": ["PII,Financial"], + "outcomes": ["no_text,failed"], + "limit": [expected_limit], + "offset": ["0"], + } + + +def _summary(**overrides: Any) -> dict: + return { + "run_id": 41, + "connection": {"id": CONNECTION_ID, "name": "invoices"}, + "status": "finished", + "finished_at": "2026-09-04T02:11:07Z", + "settings": { + "files_per_directory": 20, + "sample_order": "hash", + "files_per_run": 100000, + "max_bytes_per_file": 10000000, + }, + "ai_engine": {"model": "dm-ai-ner-small-1.0.0-20260601", "version": "1.26.11"}, + "report_rows": 15, + "files_found": 340, + "files_read": 20, + "files_failed": 2, + "files_without_text": 3, + "files_with_findings": 14, + "bytes_opened": 8123344, + "files_over_run_limit": 7, + "labels": [ + { + "label": "EMAIL", + "description": "Email address", + "categories": ["PII", "Contact"], + "occurrences": 62, + "files": 11, + }, + {"label": "IBAN", "description": "Bank account", "categories": ["PCI"], "occurrences": 0, "files": 0}, + ], + **overrides, + } + + +def test_get_unstructured_discovery_summary_parses_the_run_its_totals_and_each_label( + client: DataMasqueClient, +) -> None: + with requests_mock.Mocker() as m: + m.get(f"{RESULTS_URL}summary/", json=_summary()) + summary = client.get_unstructured_discovery_summary(RunId(41)) + + assert summary.run_id == 41 + assert summary.connection.id == CONNECTION_ID + assert summary.connection.name == "invoices" + assert summary.status == "finished" + assert summary.finished_at == datetime(2026, 9, 4, 2, 11, 7, tzinfo=timezone.utc) + assert summary.settings is not None + assert summary.settings.files_per_directory == 20 + assert summary.settings.sample_order is UnstructuredDiscoverySampleOrder.hash + assert summary.settings.files_per_run == 100000 + assert summary.settings.max_bytes_per_file == 10000000 + assert summary.ai_engine is not None + assert summary.ai_engine.model == "dm-ai-ner-small-1.0.0-20260601" + assert summary.ai_engine.version == "1.26.11" + assert summary.report_rows == 15 + assert summary.files_found == 340 + assert summary.files_read == 20 + assert summary.files_failed == 2 + assert summary.files_without_text == 3 + assert summary.files_with_findings == 14 + assert summary.bytes_opened == 8123344 + assert summary.files_over_run_limit == 7 + label_total, never_found = summary.labels + assert label_total.label == "EMAIL" + assert label_total.description == "Email address" + assert label_total.categories == ["PII", "Contact"] + assert label_total.occurrences == 62 + assert label_total.files == 11 + assert never_found.occurrences == 0 + assert not summary.model_extra + assert not summary.settings.model_extra + assert not summary.ai_engine.model_extra + assert not label_total.model_extra + + +def test_get_unstructured_discovery_summary_reads_a_run_without_settings_or_ai_engine( + client: DataMasqueClient, +) -> None: + with requests_mock.Mocker() as m: + m.get( + f"{RESULTS_URL}summary/", + json=_summary( + status="failed", + finished_at=None, + settings=None, + ai_engine=None, + report_rows=0, + labels=[], + ), + ) + summary = client.get_unstructured_discovery_summary(RunId(41)) + + assert summary.finished_at is None + assert summary.settings is None + assert summary.ai_engine is None + + +def test_get_unstructured_discovery_summary_reads_all_and_an_engine_without_a_version( + client: DataMasqueClient, +) -> None: + settings = {"files_per_directory": None, "sample_order": "name_asc", "files_per_run": None, "max_bytes_per_file": 5} + with requests_mock.Mocker() as m: + m.get(f"{RESULTS_URL}summary/", json=_summary(settings=settings, ai_engine={"model": "m", "version": None})) + summary = client.get_unstructured_discovery_summary(RunId(41)) + + assert summary.settings is not None + assert summary.settings.files_per_directory is None + assert summary.settings.files_per_run is None + assert summary.settings.sample_order is UnstructuredDiscoverySampleOrder.name_asc + assert summary.ai_engine is not None + assert summary.ai_engine.version is None + + +def test_get_unstructured_discovery_report_returns_the_csv_text(client: DataMasqueClient) -> None: + csv_text = ( + "\ufeffpath,directory,label,description,occurrences,categories,character_count\r\n" + "réunions/0142.pdf,réunions,EMAIL,Email address,4,PII;Contact,5120\r\n" + "réunions/0143.pdf,réunions,,,0,,96\r\n" + ) + with requests_mock.Mocker() as m: + m.get( + REPORT_URL, + content=csv_text.encode("utf-8"), + headers={"Content-Type": "text/csv; charset=utf-8"}, + ) + + assert client.get_unstructured_discovery_report(RunId(41)) == csv_text + + +def test_get_unstructured_discovery_report_returns_zip_bytes_when_split(client: DataMasqueClient) -> None: + zip_bytes = b"PK\x03\x04 split report zip bytes" + with requests_mock.Mocker() as m: + m.get(REPORT_URL, content=zip_bytes, headers={"Content-Type": "application/zip"}) + + assert client.get_unstructured_discovery_report(RunId(41)) == zip_bytes + + +@pytest.mark.parametrize( + "url, call", + [ + (FOLDERS_URL, lambda client: client.get_unstructured_discovery_folder_page(RunId(41))), + (FILES_URL, lambda client: client.get_unstructured_discovery_file_page(RunId(41), "")), + (f"{RESULTS_URL}summary/", lambda client: client.get_unstructured_discovery_summary(RunId(41))), + (REPORT_URL, lambda client: client.get_unstructured_discovery_report(RunId(41))), + ], +) +def test_reading_a_run_that_is_not_unstructured_discovery_raises( + client: DataMasqueClient, url: str, call: ClientCall +) -> None: + with requests_mock.Mocker() as m: + m.get(url, json={"detail": "No Run matches the given query."}, status_code=404) + with pytest.raises(DataMasqueApiError) as exc_info: + call(client) + + assert exc_info.value.response.status_code == 404