Skip to content

Commit b1978d4

Browse files
authored
feat: add opt-in metadata catalog name escaping (#985)
* feat: add opt-in metadata catalog name escaping Signed-off-by: Vu Anh Phung <vu.phung@databricks.com> * docs: note metadata catalog escaping configuration in changelog Signed-off-by: Vu Anh Phung <vu.phung@databricks.com> * test: document JDBC-compatible metadata backslash escaping Signed-off-by: Vu Anh Phung <vu.phung@databricks.com> --------- Signed-off-by: Vu Anh Phung <vu.phung@databricks.com>
1 parent 9eb024c commit b1978d4

9 files changed

Lines changed: 233 additions & 4 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
# Release History
22

3+
# Unreleased
4+
- Add the `escape_catalog_name` connection parameter (default `False`) to opt in to escaping unescaped `_` in metadata catalog names on Thrift and kernel connections. Existing escapes and `%` wildcards are preserved.
5+
36
# 4.6.0 (2026-09-24)
47
- Upgrade Databricks SQL Kernel to 1.1.0; the kernel dependency is now stable and no longer experimental.
58
- Transparently auto-recover Thrift connections to Reyden / Real-Time warehouses: when a warehouse rejects the default Thrift protocol (SQLSTATE `KP001`), the session is re-opened on the kernel backend and the warehouse is remembered so later connections skip Thrift. Applies only when no backend was chosen explicitly.

‎CONNECTION_PARAMETERS.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ to change without notice.
166166
| `schema` | `str` | ✅ | ✅ | `None` | Initial schema for the session (DBR 9.0+). |
167167
| `query_tags` | `Dict[str, Optional[str]]` | ✅ | ✅ | `None` | Connection-level key/value tags serialized into the reserved `QUERY_TAGS` session configuration on both backends. Cursor-level `query_tags` are applied per statement separately. |
168168
| `enable_metric_view_metadata` | `bool` | ✅ | ✅ | `False` | Sets `spark.sql.thriftserver.metadata.metricview.enabled` via session config so metric-view metadata surfaces. |
169+
| `escape_catalog_name` | `bool` | ✅ | ✅ | `False` | Escape unescaped `_` in metadata catalog names, preserving existing `\_` escapes and `%` wildcards. Applies to `schemas()`, `tables()`, and `columns()` on Thrift and `tables()` on Kernel; Kernel schemas/columns already use exact catalog identifiers. |
169170
| `use_inline_params` | `bool` \| `"silent"` | ✅ | ✅ | `False` | Render parameters inline (legacy) vs. native bound params (DBR 14.1+). Shared cursor code performs the rendering before dispatch, so both backends honor it identically. |
170171
| `ignore_transactions` | `bool` | ✅ | ✅ | `True` | When `True`: `commit()` is a no-op, `rollback()` raises `NotSupportedError`, and setting `autocommit` is a no-op. |
171172
| `fetch_autocommit_from_server`| `bool` | ✅ | ✅ | `False` | Query the server (`SET AUTOCOMMIT`) for autocommit state instead of returning the cached value. |

‎src/databricks/sql/backend/kernel/client.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
ProgrammingError,
5050
)
5151
from databricks.sql.telemetry.telemetry_client import TelemetryHelper
52+
from databricks.sql.utils import escape_metadata_catalog_name
5253

5354
if TYPE_CHECKING:
5455
from databricks.sql.client import Cursor
@@ -267,6 +268,7 @@ def __init__(
267268
self._telemetry_options = kwargs.get("telemetry_options") or {}
268269
self._catalog = catalog
269270
self._schema = schema
271+
self._escape_catalog_name = kwargs.get("escape_catalog_name", False)
270272
# ``_use_arrow_native_complex_types`` is the connector-side
271273
# toggle for whether complex columns (ARRAY / MAP / STRUCT)
272274
# are surfaced as native Arrow shapes or as compact JSON
@@ -1019,7 +1021,9 @@ def get_tables(
10191021
# do the work — no connector-side drain + refilter. Passing it
10201022
# through preserves streaming for large schemas.
10211023
stream = self._kernel_session.metadata().list_tables(
1022-
catalog=_catalog_or_none(catalog_name),
1024+
catalog=escape_metadata_catalog_name(
1025+
_catalog_or_none(catalog_name), self._escape_catalog_name
1026+
),
10231027
schema_pattern=schema_name,
10241028
table_pattern=table_name,
10251029
table_types=table_types if table_types else None,

‎src/databricks/sql/backend/thrift_backend.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
convert_arrow_based_set_to_arrow_table,
5353
convert_decimals_in_arrow_table,
5454
convert_column_based_set_to_arrow_table,
55+
escape_metadata_catalog_name,
5556
serialize_query_tags,
5657
)
5758
from databricks.sql.types import SSLOptions
@@ -177,6 +178,7 @@ def __init__(
177178
raise ValueError("No valid connection settings.")
178179

179180
self._host = server_hostname
181+
self._escape_catalog_name = kwargs.get("escape_catalog_name", False)
180182
self._initialize_retry_args(kwargs)
181183
self._use_arrow_native_complex_types = kwargs.get(
182184
"_use_arrow_native_complex_types", True
@@ -1193,7 +1195,9 @@ def get_schemas(
11931195
getDirectResults=ttypes.TSparkGetDirectResults(
11941196
maxRows=max_rows, maxBytes=max_bytes
11951197
),
1196-
catalogName=catalog_name,
1198+
catalogName=escape_metadata_catalog_name(
1199+
catalog_name, self._escape_catalog_name
1200+
),
11971201
schemaName=schema_name,
11981202
)
11991203
resp = self.make_request(self._client.GetSchemas, req)
@@ -1239,7 +1243,9 @@ def get_tables(
12391243
getDirectResults=ttypes.TSparkGetDirectResults(
12401244
maxRows=max_rows, maxBytes=max_bytes
12411245
),
1242-
catalogName=catalog_name,
1246+
catalogName=escape_metadata_catalog_name(
1247+
catalog_name, self._escape_catalog_name
1248+
),
12431249
schemaName=schema_name,
12441250
tableName=table_name,
12451251
tableTypes=table_types,
@@ -1287,7 +1293,9 @@ def get_columns(
12871293
getDirectResults=ttypes.TSparkGetDirectResults(
12881294
maxRows=max_rows, maxBytes=max_bytes
12891295
),
1290-
catalogName=catalog_name,
1296+
catalogName=escape_metadata_catalog_name(
1297+
catalog_name, self._escape_catalog_name
1298+
),
12911299
schemaName=schema_name,
12921300
tableName=table_name,
12931301
columnName=column_name,

‎src/databricks/sql/client.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -315,6 +315,11 @@ def read(self) -> Optional[OAuthToken]:
315315
This allows
316316
1. cursor.tables() to return METRIC_VIEW table type
317317
2. cursor.columns() to return "measure" column type
318+
:param escape_catalog_name: `bool`, optional (default is False)
319+
Escape unescaped underscores in catalog names for schemas(), tables(),
320+
and columns() metadata requests. Existing escapes and % wildcards are
321+
preserved. Kernel schemas() and columns() already use exact catalog
322+
identifiers, so their catalog names are passed through unchanged.
318323
:param fetch_autocommit_from_server: `bool`, optional (default is False)
319324
When True, the connection.autocommit property queries the server for current state
320325
using SET AUTOCOMMIT instead of returning cached value.

‎src/databricks/sql/session.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,7 @@ def _create_backend(
333333
request_timeout_secs=kwargs.get("_socket_timeout"),
334334
max_connections=kwargs.get("_pool_maxsize") or None,
335335
telemetry_options=kernel_telemetry_options,
336+
escape_catalog_name=kwargs.get("escape_catalog_name", False),
336337
)
337338

338339
# These reference the lazily-resolved module attributes defined via

‎src/databricks/sql/utils.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,15 @@
4646
logger = logging.getLogger(__name__)
4747

4848

49+
def escape_metadata_catalog_name(
50+
catalog_name: Optional[str], enabled: bool
51+
) -> Optional[str]:
52+
r"""Match JDBC: escape _ unless immediately preceded by \; preserve %."""
53+
if not enabled or catalog_name is None:
54+
return catalog_name
55+
return re.sub(r"(?<!\\)_", r"\\_", catalog_name)
56+
57+
4958
def get_session_config_value(
5059
session_configuration: Optional[Dict[str, Any]], key: str
5160
) -> Optional[str]:

‎tests/unit/test_kernel_client.py‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1812,6 +1812,102 @@ def test_sync_execute_leaves_rowcount_default_when_num_modified_rows_none():
18121812
# ---------------------------------------------------------------------------
18131813

18141814

1815+
@pytest.mark.parametrize(
1816+
"options",
1817+
[{}, {"escape_catalog_name": False}, {"escape_catalog_name": True}],
1818+
ids=["default", "disabled", "enabled"],
1819+
)
1820+
@pytest.mark.parametrize(
1821+
"method, filters, kernel_filters",
1822+
[
1823+
("schemas", {"schema_name": "schema_%"}, {"schema_pattern": "schema_%"}),
1824+
(
1825+
"tables",
1826+
{
1827+
"schema_name": "schema_%",
1828+
"table_name": "table_%",
1829+
"table_types": ["TABLE"],
1830+
},
1831+
{
1832+
"schema_pattern": "schema_%",
1833+
"table_pattern": "table_%",
1834+
"table_types": ["TABLE"],
1835+
},
1836+
),
1837+
(
1838+
"columns",
1839+
{
1840+
"schema_name": "schema_%",
1841+
"table_name": "table_%",
1842+
"column_name": "column_%",
1843+
},
1844+
{
1845+
"schema_pattern": "schema_%",
1846+
"table_pattern": "table_%",
1847+
"column_pattern": "column_%",
1848+
},
1849+
),
1850+
],
1851+
)
1852+
@pytest.mark.parametrize(
1853+
"catalog_name, escaped",
1854+
[
1855+
(None, None),
1856+
("", ""),
1857+
("%", "%"),
1858+
("*", "*"),
1859+
("simple", "simple"),
1860+
("my_catalog", r"my\_catalog"),
1861+
("a_b_c", r"a\_b\_c"),
1862+
(r"my\_catalog", r"my\_catalog"),
1863+
(r"a\_b_c%", r"a\_b\_c%"),
1864+
# Match JDBC: preserve the final \_ pair regardless of backslash parity.
1865+
pytest.param(
1866+
r"my\\_cat_tail",
1867+
r"my\\_cat\_tail",
1868+
id="jdbc-even-backslash-run",
1869+
),
1870+
pytest.param(
1871+
r"my\\\_cat_tail",
1872+
r"my\\\_cat\_tail",
1873+
id="jdbc-odd-backslash-run",
1874+
),
1875+
],
1876+
)
1877+
def test_metadata_catalog_escaping(
1878+
monkeypatch, options, method, filters, kernel_filters, catalog_name, escaped
1879+
):
1880+
from databricks import sql
1881+
1882+
kernel_session = MagicMock()
1883+
kernel_session.session_id = "session-id"
1884+
rpc = getattr(kernel_session.metadata.return_value, f"list_{method}")
1885+
rpc.return_value = _stream_with_schema()
1886+
monkeypatch.setattr(
1887+
kernel_client._kernel, "Session", lambda **kwargs: kernel_session
1888+
)
1889+
1890+
with sql.connect(
1891+
server_hostname="example.cloud.databricks.com",
1892+
http_path="/sql/1.0/warehouses/abc",
1893+
access_token="test-token",
1894+
use_kernel=True,
1895+
enable_telemetry=False,
1896+
**options,
1897+
) as connection:
1898+
cursor = connection.cursor()
1899+
assert getattr(cursor, method)(catalog_name=catalog_name, **filters) is cursor
1900+
1901+
expected = (
1902+
escaped
1903+
if method == "tables" and options.get("escape_catalog_name", False)
1904+
else catalog_name
1905+
)
1906+
if expected in ("%", "*"):
1907+
expected = None
1908+
rpc.assert_called_once_with(catalog=expected, **kernel_filters)
1909+
1910+
18151911
@pytest.mark.parametrize("catalog_wildcard", ["%", "*"])
18161912
def test_get_columns_normalizes_all_catalog_wildcard(catalog_wildcard):
18171913
c = _make_client()

‎tests/unit/test_thrift_backend.py‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,108 @@ def retry_policy_factory():
3232
}
3333

3434

35+
@pytest.mark.parametrize(
36+
"options",
37+
[{}, {"escape_catalog_name": False}, {"escape_catalog_name": True}],
38+
ids=["default", "disabled", "enabled"],
39+
)
40+
@pytest.mark.parametrize(
41+
"method, rpc_name, filters",
42+
[
43+
("schemas", "GetSchemas", {"schema_name": "schema_%"}),
44+
(
45+
"tables",
46+
"GetTables",
47+
{
48+
"schema_name": "schema_%",
49+
"table_name": "table_%",
50+
"table_types": ["TABLE"],
51+
},
52+
),
53+
(
54+
"columns",
55+
"GetColumns",
56+
{
57+
"schema_name": "schema_%",
58+
"table_name": "table_%",
59+
"column_name": "column_%",
60+
},
61+
),
62+
],
63+
)
64+
@pytest.mark.parametrize(
65+
"catalog_name, escaped",
66+
[
67+
(None, None),
68+
("", ""),
69+
("%", "%"),
70+
("*", "*"),
71+
("simple", "simple"),
72+
("my_catalog", r"my\_catalog"),
73+
("a_b_c", r"a\_b\_c"),
74+
(r"my\_catalog", r"my\_catalog"),
75+
(r"a\_b_c%", r"a\_b\_c%"),
76+
# Match JDBC: preserve the final \_ pair regardless of backslash parity.
77+
pytest.param(
78+
r"my\\_cat_tail",
79+
r"my\\_cat\_tail",
80+
id="jdbc-even-backslash-run",
81+
),
82+
pytest.param(
83+
r"my\\\_cat_tail",
84+
r"my\\\_cat\_tail",
85+
id="jdbc-odd-backslash-run",
86+
),
87+
],
88+
)
89+
def test_metadata_catalog_escaping(
90+
options, method, rpc_name, filters, catalog_name, escaped
91+
):
92+
with (
93+
patch(
94+
"databricks.sql.backend.thrift_backend.TCLIService.Client", autospec=True
95+
) as client_class,
96+
patch("databricks.sql.result_set.ThriftResultSet"),
97+
patch.object(
98+
ThriftDatabricksClient,
99+
"_handle_execute_response",
100+
return_value=(Mock(), False),
101+
),
102+
):
103+
rpc_client = client_class.return_value
104+
rpc_client.OpenSession.return_value = ThriftBackendTestSuite.open_session_resp
105+
rpc = getattr(rpc_client, rpc_name)
106+
rpc.return_value = Mock(
107+
status=ThriftBackendTestSuite.okay_status, directResults=None
108+
)
109+
110+
with databricks.sql.connect(
111+
server_hostname="example.cloud.databricks.com",
112+
http_path="/sql/1.0/warehouses/abc",
113+
access_token="test-token",
114+
enable_telemetry=False,
115+
**options,
116+
) as connection:
117+
cursor = connection.cursor()
118+
assert (
119+
getattr(cursor, method)(catalog_name=catalog_name, **filters) is cursor
120+
)
121+
122+
request = rpc.call_args.args[0]
123+
expected = (
124+
escaped if options.get("escape_catalog_name", False) else catalog_name
125+
)
126+
assert request.catalogName == expected
127+
for arg, value in filters.items():
128+
thrift_field = {
129+
"schema_name": "schemaName",
130+
"table_name": "tableName",
131+
"column_name": "columnName",
132+
"table_types": "tableTypes",
133+
}[arg]
134+
assert getattr(request, thrift_field) == value
135+
136+
35137
@pytest.mark.skipif(pyarrow is None, reason="PyArrow is not installed")
36138
class ThriftBackendTestSuite(unittest.TestCase):
37139
okay_status = ttypes.TStatus(statusCode=ttypes.TStatusCode.SUCCESS_STATUS)

0 commit comments

Comments
 (0)