DM-56186: Add metric tables to plot navigator

backend/lsst/plot_navigator/app_factory.py

@@ -32,7 +32,7 @@
 from fastapi.responses import FileResponse
 from fastapi.staticfiles import StaticFiles
 
-from . import cache, images, repos, summaries
+from . import cache, images, repos, summaries, tables
 from .config import Settings, get_settings
 
 logger = logging.getLogger("api.middleware")
@@ -66,6 +66,7 @@ def app_factory(settings: Settings | None = None) -> FastAPI:
     app.include_router(images.router, prefix=f"{settings.app_prefix}/api/v1/images")
     app.include_router(cache.router, prefix=f"{settings.app_prefix}/api/v1/cache")
     app.include_router(repos.router, prefix=f"{settings.app_prefix}/api/v1/repos")
+    app.include_router(tables.router, prefix=f"{settings.app_prefix}/api/v1/tables")
 
 
     # --- Static assets (JS, CSS, images from Vite build) ---

backend/lsst/plot_navigator/make_test_butler.py

@@ -48,6 +48,14 @@ def create_temp_butler(tracts: Iterable[int], repo_dir: Path) -> None:
         )
     )
 
+    butler.registry.registerDatasetType(
+        DatasetType(
+            "object_metrics_table",
+            butler.dimensions.conform(["skymap"]),
+            "ArrowAstropy",
+        )
+    )
+
     print(f"Created temporary Butler repo at: {repo_dir}")
 
 
@@ -135,8 +143,13 @@ def make_test_butler(repo_dir: Path, repo_name: str) -> None:
     collections = ["debug_collection",
                    "debug_collection_v2",
                    "debug_collection_v2_only",
+                   "big_debug_collection",
                    "LSSTCam/calib/DM-53399/3s_v1_dp2_gain_correction_20250720/gainCorrectionGen.20251124a/20251202T172458Z"]
 
+    # Create one collection with several hundred plots for testing (all the same plot PNG)
+    big_collection = "big_debug_collection"
+    assert big_collection in collections
+
     butler = Butler.from_config(repo_dir, writeable=True)
 
     for collection in collections:
@@ -163,8 +176,16 @@ def make_test_butler(repo_dir: Path, repo_name: str) -> None:
             {"skymap": "test_skymap"}
         )
 
+        ingest_to_temp_butler(
+            butler,
+            "test_assets/object_metrics_table_lsst_cells_v2_LSSTCam_runs_DRP_DP2_v30_0_6_rc1_DM-53881_stage3_20260412T000817Z.parq",
+            "object_metrics_table",
+            collection,
+            {"skymap": "test_skymap"},
+        )
+
         # Create a very large number of one type of plot.
-        if collection == collections[0]:
+        if collection == big_collection:
             for tract in range(1462, 11400):
                 ingest_to_temp_butler(
                     butler,

backend/lsst/plot_navigator/tables.py

@@ -0,0 +1,119 @@
+# This file is part of plot-navigator.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import logging
+from typing import Annotated
+
+import duckdb
+import lsst.daf.butler as dafButler
+from fastapi import APIRouter, Depends, HTTPException, Query
+from pydantic import BaseModel
+
+from .config import Settings, get_settings
+
+router = APIRouter(tags=["tables"])
+
+logger = logging.getLogger("tables")
+
+class TableList(BaseModel):
+    """API response for the list of available collections."""
+    repo: str
+    collection: str
+    tables: list[str]
+
+class GroupList(BaseModel):
+    """API response for the list of available metric groups in a table."""
+    repo: str
+    collection: str
+    table: str
+    groups: list[str]
+
+#TODO: can we validate the repo and collection argument somewhere outside of the function?
+#TODO: This is maybe kind of slow, can we cache it in memory?
+@router.get("/tables/{repo}/{collection:path}")
+async def get_tables(repo: str,
+                     collection: str,
+                     settings: Settings = Depends(get_settings)) -> TableList:
+    butler = dafButler.Butler.from_config(repo)
+    table_dataset_types = butler.registry.queryDatasetTypes("*metrics_table")
+    all_datasets = [
+        butler.query_datasets(dataset_type, collections=collection, explain=False)
+        for dataset_type in table_dataset_types
+    ]
+
+    # all_datasets is a list of lists, so this searches per candidate dataset,
+    # then picks the name out of each dataset ref in that candidate
+    dataset_names = [
+       ref.datasetType.name
+       for dataset_refs in all_datasets
+       for ref in dataset_refs
+    ]
+
+    return TableList(repo=repo, collection=collection, tables=list(set(dataset_names)))
+
+@router.get("/data/{table_name}/{repo}/{collection:path}")
+async def get_table(table_name: str,
+                    repo: str,
+                    collection: str,
+                    group_names: Annotated[str | None, Query(pattern="[a-zA-Z,]+")] = None,
+                    settings: Settings = Depends(get_settings)) -> list[dict]:
+  butler = dafButler.Butler(repo, collections=collection)
+  refs = butler.query_datasets(table_name)
+  if len(refs) < 1:
+    raise HTTPException(status_code=404, detail="Metric table not found")
+
+  parquet_filename = butler.getURI(refs[0])
+  con = duckdb.connect()
+
+  if group_names:
+    # Want to have a list of data ID dimensions that always get selected
+    column_string = ",".join(f"COLUMNS('{name}.*')" for name in group_names.split(","))
+    try:
+      res = con.execute(f"SELECT tract, {column_string} FROM read_parquet(\"{parquet_filename}\")")
+    except duckdb.BinderException as e:
+      raise HTTPException(status_code=404, detail="Invalid group names")
+  else:
+    res = con.execute(f"SELECT * FROM read_parquet(\"{parquet_filename}\")")
+
+  return res.to_arrow_table().to_pylist()
+
+@router.get("/groups/{table_name}/{repo}/{collection:path}")
+async def get_table_groups(table_name: str,
+                           repo: str,
+                           collection: str,
+                           settings: Settings = Depends(get_settings)) -> GroupList:
+
+  butler = dafButler.Butler(repo, collections=collection)
+  refs = butler.query_datasets(table_name)
+  if len(refs) < 1:
+    raise HTTPException(status_code=404, detail="Metric table not found")
+
+  parquet_filename = butler.getURI(refs[0])
+
+  con = duckdb.connect()
+
+  result = con.execute("SELECT split_part(column_name, '_', 1)  AS col_group "
+                       f"FROM (DESCRIBE SELECT * FROM read_parquet(\"{parquet_filename}\") ) "
+                       "GROUP BY col_group")
+
+  table = result.to_arrow_table()
+  return GroupList(repo=repo, collection=collection, table=table_name,
+                   groups=table["col_group"].to_pylist())

backend/pyproject.toml

@@ -21,6 +21,7 @@ dependencies = [
     "psycopg2-binary",
     "httpx>=0.28.1",
     "anyio>=4.12.1",
+    "duckdb>=1.5.5",
 ]
 
 [project.optional-dependencies]

backend/tests/test_routes.py

@@ -22,9 +22,24 @@
 from fastapi.testclient import TestClient
 
 from lsst.plot_navigator.repos import RepoList
+from lsst.plot_navigator.tables import TableList, GroupList
 
 
 def test_list_summaries(client: TestClient, test_butler):
     response = client.get("/api/v1/repos")
     repo_list = RepoList.model_validate(response.json())
-    assert "testing_butler" in repo_list.repos
+    assert "testing_butler" in repo_list.repos
+
+def test_list_metric_tables(client: TestClient, test_butler):
+    repo = "testing_butler"
+    collection = "debug_collection_v2"
+    response = client.get(f"/api/v1/tables/tables/{repo}/{collection}")
+    table_list = TableList.model_validate(response.json())
+    assert "object_metrics_table" in table_list.tables
+
+def test_list_groups_in_table(client: TestClient, test_butler):
+    repo = "testing_butler"
+    collection = "debug_collection_v2"
+    response = client.get(f"/api/v1/tables/groups/object_metrics_table/{repo}/{collection}")
+    group_list = GroupList.model_validate(response.json())
+    assert "shapeSizeFractionalDiff" in group_list.groups

frontend/package.json

@@ -16,6 +16,7 @@
     "@aws-sdk/client-s3": "^3.1010.0",
     "@fontsource-variable/inter": "^5.2.8",
     "@tailwindcss/vite": "^4.2.1",
+    "@tanstack/react-table": "^9.2.4",
     "@vitejs/plugin-react": "^5.2.0",
     "prettier": "^3.8.1",
     "react": "^19.2.4",