Repository navigation
Complete Volume file operations through the kernel backend #1057
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
3a92da2
c54d091
edd7608
4fc9bf6
07dbcc4
dd8bc9c
56ff1a0
ea8e8a9
1e271bb
ab1efce
ccc5b46
64c9838
f4292d3
07538ed
ff77a54
cd42b4e
063389d
dfe3527
7f6e812
6995ed4
754a5da
d7c8907
b69d00d
273761b
b6042eb
5c188bd
dad82a6
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1 @@ | ||
| 0b5d0edfed32c0745d5abc2468fc3070e2eb2a17 | ||
| ab4f133620fce52f9932508f0b245db630a57c48 |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -16,8 +16,6 @@ | |
|
|
||
| - ``query_tags`` on execute is not supported (kernel exposes | ||
| ``statement_conf`` but PyO3 doesn't surface it). | ||
| - Volume PUT/GET (staging operations): kernel has no Volume API | ||
| yet. Users on Thrift-only paths. | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
@@ -35,6 +33,7 @@ | |
| Optional, | ||
| Set, | ||
| TYPE_CHECKING, | ||
| Tuple, | ||
| Union, | ||
| ) | ||
|
|
||
|
|
@@ -84,41 +83,6 @@ | |
| # per-request skip-and-warn. | ||
| _KERNEL_MANAGED_HEADERS = frozenset({"authorization", "x-databricks-org-id"}) | ||
|
|
||
| # Leading verbs of SQL volume/staging statements. Detected by the | ||
| # leading token (case-insensitive) so the kernel backend can fail loud | ||
| # on staging ops it can't service — see ``execute_command``. | ||
| _STAGING_VERBS = ("PUT", "GET", "REMOVE") | ||
|
|
||
|
|
||
| def _strip_leading_sql_comments(sql: str) -> str: | ||
| """Strip leading whitespace and SQL comments (``-- …`` line and | ||
| ``/* … */`` block, possibly several) from ``sql``, returning the | ||
| remainder. | ||
|
|
||
| Needed so staging detection sees the real leading verb: a | ||
| comment-prefixed staging op (``-- upload\\nPUT …`` or | ||
| ``/* c */ PUT …``, common in ETL scripts) must still be classified | ||
| as staging, or it would slip past the guard into the silent-no-op | ||
| bug. Block comments do not nest in Databricks SQL, so a simple | ||
| scan-to-``*/`` is correct. | ||
| """ | ||
| i = 0 | ||
| n = len(sql) | ||
| while i < n: | ||
| if sql[i].isspace(): | ||
| i += 1 | ||
| elif sql.startswith("--", i): | ||
| # Line comment: skip to end of line (or string). | ||
| nl = sql.find("\n", i) | ||
| i = n if nl == -1 else nl + 1 | ||
| elif sql.startswith("/*", i): | ||
| # Block comment: skip to closing */ (or end if unterminated). | ||
| close = sql.find("*/", i + 2) | ||
| i = n if close == -1 else close + 2 | ||
| else: | ||
| break | ||
| return sql[i:] | ||
|
|
||
|
|
||
| def _is_not_found(exc: BaseException) -> bool: | ||
| """True iff ``exc`` is a kernel ``NotFound`` error (HTTP 404 / | ||
|
|
@@ -155,21 +119,6 @@ def _catalog_or_none(value: Optional[str]) -> Optional[str]: | |
| return value | ||
|
|
||
|
|
||
| def _is_staging_statement(operation: str) -> bool: | ||
| """True iff ``operation`` is a volume/staging statement (PUT / GET / | ||
| REMOVE). | ||
|
|
||
| Strips leading whitespace + SQL comments first (so a comment- | ||
| prefixed staging op is still caught), then matches the leading token | ||
| only — so a normal query that merely *contains* the word (e.g. | ||
| ``SELECT 'GET' AS x``) isn't misflagged. | ||
| """ | ||
| stripped = _strip_leading_sql_comments(operation) | ||
| # First whitespace-delimited token, uppercased. | ||
| verb = stripped.split(None, 1)[0].upper() if stripped.strip() else "" | ||
| return verb in _STAGING_VERBS | ||
|
|
||
|
|
||
| class _KernelSessionParameterSupport(NamedTuple): | ||
| session_type: Any | ||
| parameters: FrozenSet[str] | ||
|
|
@@ -436,6 +385,8 @@ def __init__( | |
| # Guarded by its own lock — cancel can race execute teardown. | ||
| self._sync_cancellers: Dict[int, Any] = {} | ||
| self._sync_cancellers_lock = threading.RLock() | ||
| # Retain the mutable owner until its transfer completes; dropping it cancels the result. | ||
| self._volume_handles: Dict[str, Tuple[Any, Any]] = {} | ||
|
|
||
| # ── Session lifecycle ────────────────────────────────────────── | ||
|
|
||
|
|
@@ -565,6 +516,21 @@ def open_session( | |
| def close_session(self, session_id: SessionId) -> None: | ||
| if self._kernel_session is None: | ||
| return | ||
| with self._sync_cancellers_lock: | ||
| cancellers = list(self._sync_cancellers.values()) + [ | ||
| entry[1] | ||
| for entry in self._volume_handles.values() | ||
| if entry[1] is not None | ||
| ] | ||
| self._sync_cancellers.clear() | ||
| self._volume_handles.clear() | ||
| for canceller in cancellers: | ||
| try: | ||
| canceller.cancel() | ||
| except Exception: | ||
| logger.debug( | ||
| "Statement cancellation during session close failed", exc_info=True | ||
| ) | ||
| # Close any tracked async handles first so they fire their | ||
| # server-side CloseStatement before the session goes away. | ||
| with self._async_handles_lock: | ||
|
|
@@ -633,24 +599,6 @@ def execute_command( | |
| # ``_async_statements`` and closed by ``close_command``); the sync | ||
| # path drops it in finally. ``close_stmt`` is the post-success | ||
| # decision flag — it stays True on sync, flips to False on async. | ||
| # Volume/staging (PUT/GET/REMOVE) is not supported on the kernel | ||
| # path: the kernel returns the staging control row as a normal | ||
| # result set (``KernelResultSet.is_staging_operation`` is always | ||
| # False), so the connector's ``_handle_staging_operation`` never | ||
| # fires and NO file is transferred. Rather than silently no-op | ||
| # (the Thrift path performs the presigned-URL upload/download), | ||
| # fail loud at the call site so ETL scripts don't ingest | ||
| # stale/missing data. Detected by the leading SQL verb — the | ||
| # only signal available pre-execute, since the kernel exposes no | ||
| # staging marker today. | ||
| if _is_staging_statement(operation): | ||
| raise NotSupportedError( | ||
| "Volume / staging operations (PUT / GET / REMOVE) are not " | ||
| "supported on the kernel backend (use_kernel=True); the file " | ||
| "transfer would silently not happen. Use the Thrift backend " | ||
| "for staging operations." | ||
| ) | ||
|
|
||
| close_stmt = True | ||
| try: | ||
| try: | ||
|
|
@@ -692,6 +640,13 @@ def execute_command( | |
| # Canceller is best-effort; never block execute on it. | ||
| pass | ||
| executed = stmt.execute() | ||
| if executed.is_volume_operation: | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟠 High — If the resolver installs any wheel in the allowed range that lacks them, this isn't limited to Volume ops. Every
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The optional-canceller defect is corrected at The new Volume marker/completion API dependency is a valid merge prerequisite. The source build pins compatible The fetch_result subclaim is separate: released 1.4.0 exposes it, but the substitution is removed from this PR. Merged native #451 provides terminal-response reuse with the original await_result boundary. Actual public-cursor owner and reader/refetch tests show identical await/fetch requests, rows and metadata on the selected native; redundant #1056 is superseded without merge. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for confirming the Volume-marker dependency is a merge prerequisite, and for dropping the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟠 High — There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟠 High — Every synchronous kernel On a resolver-selected 1.4.x wheel, the attribute read raises Either bump the floor in this PR to the first kernel release that ships
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Confirmed; this remains a merge prerequisite, including ordinary queries and metadata, not just Volume statements. The published 1.4.0 wheel is not compatible with the new APIs. Delivery requires the actual compatible coordinated native release, the Python dependency floor/lock update, and clean installed-package ordinary-query/metadata/Volume validation. Source-built validation does not close this gate. We are requiring the coordinated new native API rather than adding an old-native fallback or inventing an unpublished version. The PR description retains the dependency as unresolved; this comment is acknowledged, not claimed fixed. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for confirming. Keeping this open as a merge blocker until it's fixed. The code still reads There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟠 High — Every synchronous The description says the floor bump will happen at release time, but nothing in the dependency graph enforces that ordering. Either bump the floor in this PR to the first kernel release that ships
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Confirmed: the compatible native release, dependency floor/lock update and clean installed ordinary-query/metadata/async/Volume proof remain prerequisites. Published 1.4.0 is incompatible; a source pin does not enforce compatibility for an installed wheel. No fallback or invented version is being used, and this finding is not claimed fixed. For the source-availability question, the pinned kernel defines the property on both public classes: There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for confirming. You're right that the pinned kernel source defines |
||
| command_id = CommandId.from_sea_statement_id(executed.statement_id) | ||
| with self._sync_cancellers_lock: | ||
| self._volume_handles[command_id.guid] = ( | ||
| stmt, | ||
| self._sync_cancellers.get(id(cursor)), | ||
| ) | ||
| # Execute succeeded: the kernel now owns the statement | ||
| # lifecycle. It auto-closes the server statement when the | ||
| # result stream is fully drained (``ExecutedStatement:: | ||
|
|
@@ -764,16 +719,22 @@ def execute_command( | |
| try: | ||
| return self._make_result_set(executed, cursor, command_id) | ||
| except Exception as exc: | ||
| self.finish_volume_operation(command_id) | ||
| raise _wrap_kernel_exception("execute_command", exc) from exc | ||
|
|
||
| def finish_volume_operation(self, command_id: CommandId) -> None: | ||
| with self._sync_cancellers_lock: | ||
| self._volume_handles.pop(command_id.guid, None) | ||
|
|
||
| def cancel_command(self, command_id: CommandId) -> None: | ||
| with self._async_handles_lock: | ||
| handle = self._async_handles.get(command_id.guid) | ||
| with self._sync_cancellers_lock: | ||
| volume = self._volume_handles.get(command_id.guid) | ||
| handle = volume[1] if volume is not None else None | ||
| if handle is None: | ||
| # Sync-execute paths fully materialise the result before | ||
| # ``execute_command`` returns, so by the time | ||
| # cancel_command can fire there's nothing in flight. | ||
| # Match the Thrift backend's tolerant behaviour. | ||
| with self._async_handles_lock: | ||
| handle = self._async_handles.get(command_id.guid) | ||
| if handle is None: | ||
| # Completed synchronous operations have no retained canceller. | ||
| logger.debug("cancel_command: no in-flight async handle for %s", command_id) | ||
| return | ||
| try: | ||
|
|
@@ -822,6 +783,13 @@ def cancel_running_cursor(self, cursor: "Cursor") -> bool: | |
| return True | ||
|
|
||
| def close_command(self, command_id: CommandId) -> None: | ||
| with self._sync_cancellers_lock: | ||
| volume = self._volume_handles.pop(command_id.guid, None) | ||
| if volume is not None and volume[1] is not None: | ||
| try: | ||
| volume[1].cancel() | ||
| except Exception: | ||
| logger.debug("Volume cancellation during close failed", exc_info=True) | ||
| with self._async_handles_lock: | ||
| handle = self._async_handles.pop(command_id.guid, None) | ||
| stmt = self._async_statements.pop(command_id.guid, None) | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.