diff --git a/CLAUDE.md b/CLAUDE.md index e0e0a22..b6f3d4d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -169,5 +169,7 @@ docker-compose up --build - Widget keys must match parameter keys in `default-parameters.json` - Workflow names use lowercase with hyphens: "My Workflow" -> "my-workflow" - Use `show_fig()` and `show_table()` from `src/common/common.py` for consistent display +- Show DataFrames with `show_insight_table(df, key=...)` (or `show_table()`), not `st.dataframe()`: it uses OpenMS-Insight's server-side-paginated `Table`, so only the visible page is sent to the browser. `st.dataframe` serializes the whole frame and is extremely slow on full-scale datasets. Only use it for tiny fixed-size previews (e.g. `.head(10)`) +- Insight components request a rerun the first time they render, so never draw them only inside `if st.button(...):`. Store the result in `st.session_state` and render it outside the button block - Use `@st.fragment` on methods that should partially rerun (configure, results) - TOPP tool parameters use colon-separated paths: `"algorithm:section:param_name"` diff --git a/content/enrichment.py b/content/enrichment.py index 945821b..7c68aad 100644 --- a/content/enrichment.py +++ b/content/enrichment.py @@ -4,7 +4,7 @@ import pandas as pd import polars as pl import streamlit as st -from src.common.common import page_setup +from src.common.common import page_setup, show_insight_table from src.common.results_helpers import get_abundance_data, get_id_column # Import GO Enrichment modules from openms_insight engine from openms_insight.analysis.enrichment import calculate_go_enrichment @@ -136,6 +136,6 @@ st.plotly_chart(fig, use_container_width=True) st.subheader(f"📊 {go_type} Results Dataframe") - st.dataframe(df_go, use_container_width=True) + show_insight_table(df_go, key=f"enrichment-go-{go_type}", height=350) else: st.info(f"No statistically overrepresented terms identified for Category: **{go_type}**") \ No newline at end of file diff --git a/content/filtering.py b/content/filtering.py index cf593d0..3a86c68 100644 --- a/content/filtering.py +++ b/content/filtering.py @@ -4,7 +4,7 @@ import pandas as pd import polars as pl import streamlit as st -from src.common.common import page_setup +from src.common.common import page_setup, show_insight_table from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map # Import filtering functions from openms_insight package @@ -61,7 +61,7 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: st.markdown( f"Currently displaying **{pivot_df.shape[0]}** proteins and **{len(sample_cols)}** samples before filtering." ) -st.dataframe(pivot_df, use_container_width=True) +show_insight_table(pivot_df, key="filtering-original") st.markdown("---") @@ -153,9 +153,13 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: filtered_df = strip_stat_columns(filtered_lazy.collect().to_pandas()) st.session_state["filtered_df"] = filtered_df - # Layout response metrics and the filtered matrix st.success(f"Successfully applied **{filter_method}** filter!") +# Results are rendered from session state, outside the button block: the Insight +# table triggers a rerun the first time it is drawn, which would reset the button +# and wipe anything shown only while it is pressed. +filtered_df = st.session_state.get("filtered_df") +if filtered_df is not None: # Display dataset scale compression stats col1, col2, col3 = st.columns(3) col1.metric("Original Proteins", pivot_df.shape[0]) @@ -170,4 +174,4 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: "The filtered table is empty. Try relaxing the threshold constraints." ) else: - st.dataframe(filtered_df, use_container_width=True) \ No newline at end of file + show_insight_table(filtered_df, key="filtering-filtered") \ No newline at end of file diff --git a/content/imputation.py b/content/imputation.py index 1608b70..73715de 100644 --- a/content/imputation.py +++ b/content/imputation.py @@ -4,7 +4,7 @@ import pandas as pd import polars as pl import streamlit as st -from src.common.common import page_setup +from src.common.common import page_setup, show_insight_table from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map # Import imputation algorithms from openms_insight engine @@ -69,7 +69,7 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: st.markdown( f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples before imputation." ) -st.dataframe(base_df, use_container_width=True) +show_insight_table(base_df, key="imputation-input") st.markdown("---") @@ -140,6 +140,10 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: st.success(f"Successfully finalized **{impute_category}** imputation step!") - # Calculate and display a quick performance matrix check +# Results are rendered from session state, outside the button block: the Insight +# table triggers a rerun the first time it is drawn, which would reset the button +# and wipe anything shown only while it is pressed. +imputed_df = st.session_state.get("imputed_df") +if imputed_df is not None: st.subheader("Imputed Result Table") - st.dataframe(imputed_df, use_container_width=True) \ No newline at end of file + show_insight_table(imputed_df, key="imputation-result") \ No newline at end of file diff --git a/content/normalization.py b/content/normalization.py index c0e97e8..7f9ee49 100644 --- a/content/normalization.py +++ b/content/normalization.py @@ -4,7 +4,7 @@ import pandas as pd import polars as pl import streamlit as st -from src.common.common import page_setup +from src.common.common import page_setup, show_insight_table from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map # Import normalization engine functions from openms_insight from openms_insight.analysis.normalization import ( @@ -94,7 +94,7 @@ def strip_stat_columns(df: pd.DataFrame | None) -> pd.DataFrame | None: st.markdown( f"Currently displaying **{base_df.shape[0]}** rows and **{len(sample_cols)}** samples entering the normalization block." ) -st.dataframe(base_df, use_container_width=True) +show_insight_table(base_df, key="normalization-input") st.markdown("### Pipeline Overview") st.caption("Data flows in order: Filtering -> Imputation -> Normalization") @@ -231,12 +231,17 @@ def strip_stat_columns(df: pd.DataFrame | None) -> pd.DataFrame | None: st.success("Successfully executed all selected normalization pipelines!") - # Display the finalized transformation matrix view - st.subheader("Normalized Abundance Table") - st.dataframe(normalized_df, use_container_width=True) - except ValueError as val_err: # Gracefully handle validation failures raised from the engine layers (e.g., missing reference protein) st.error(f"Engine Configuration Error: {str(val_err)}") except Exception as e: - st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file + st.error(f"An unexpected pipeline error occurred: {str(e)}") + +# Results are rendered from session state, outside the button block: the Insight +# table triggers a rerun the first time it is drawn, which would reset the button +# and wipe anything shown only while it is pressed. +normalized_df = st.session_state.get("normalized_df") +if normalized_df is not None: + # Display the finalized transformation matrix view + st.subheader("Normalized Abundance Table") + show_insight_table(normalized_df, key="normalization-result") \ No newline at end of file diff --git a/content/results_pathway_analysis.py b/content/results_pathway_analysis.py index f5eb4c1..fb82116 100644 --- a/content/results_pathway_analysis.py +++ b/content/results_pathway_analysis.py @@ -8,7 +8,7 @@ from collections import defaultdict from scipy.stats import fisher_exact from pathlib import Path -from src.common.common import page_setup +from src.common.common import page_setup, show_insight_table from src.common.results_helpers import get_abundance_data # ================================ @@ -216,7 +216,7 @@ def run_go(go_type): st.info("No protein-level data available.") else: st.session_state["pivot_df"] = pivot_df - st.dataframe(pivot_df.sort_values("p-value"), width="stretch") + show_insight_table(pivot_df.sort_values("p-value"), key="pathway-protein") # ====================================================== # GO Enrichment Results @@ -255,4 +255,4 @@ def run_go(go_type): st.plotly_chart(fig, width="stretch") st.markdown(f"#### {go_type} Enrichment Results") - st.dataframe(df_go, width="stretch") \ No newline at end of file + show_insight_table(df_go, key=f"pathway-go-{go_type}", height=350) \ No newline at end of file diff --git a/content/results_pca.py b/content/results_pca.py index 29f441a..01262cb 100644 --- a/content/results_pca.py +++ b/content/results_pca.py @@ -2,7 +2,7 @@ import pandas as pd import polars as pl import streamlit as st -from src.common.common import page_setup +from src.common.common import page_setup, show_insight_table from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map from openms_insight import PCAPlot @@ -86,7 +86,7 @@ f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples " f"belonging to **{len(unique_groups)} groups** ({', '.join(unique_groups)})." ) -st.dataframe(base_df, use_container_width=True) +show_insight_table(base_df, key="pca-input") st.markdown("---") diff --git a/content/results_proteomicslfq.py b/content/results_proteomicslfq.py index fde2ab9..ad46e69 100644 --- a/content/results_proteomicslfq.py +++ b/content/results_proteomicslfq.py @@ -4,7 +4,7 @@ import numpy as np import plotly.express as px -from src.common.common import page_setup +from src.common.common import page_setup, show_insight_table from src.common.results_helpers import get_abundance_data # ================================ @@ -52,7 +52,7 @@ st.info("No protein-level data available.") else: st.session_state["pivot_df"] = pivot_df - st.dataframe(pivot_df, use_container_width=True) + show_insight_table(pivot_df, key="proteomicslfq-protein") # ====================================================== # GO Enrichment Results @@ -97,4 +97,4 @@ st.plotly_chart(fig, use_container_width=True) st.markdown(f"#### {go_type} Enrichment Results") - st.dataframe(df_go, use_container_width=True) \ No newline at end of file + show_insight_table(df_go, key=f"proteomicslfq-go-{go_type}", height=350) \ No newline at end of file diff --git a/content/statistical.py b/content/statistical.py index 07c77ea..57f732e 100644 --- a/content/statistical.py +++ b/content/statistical.py @@ -4,7 +4,7 @@ import pandas as pd import polars as pl import streamlit as st -from src.common.common import page_setup +from src.common.common import page_setup, show_insight_table from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map # Import statistics engine functions from openms_insight from openms_insight.analysis.statistics import calculate_statistical_tests, adjust_fdr_lazy @@ -80,7 +80,7 @@ st.markdown( f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples belonging to **{group_count} groups** ({', '.join(unique_groups)})." ) -st.dataframe(base_df, use_container_width=True) +show_insight_table(base_df, key="statistical-input") st.markdown("---") @@ -154,13 +154,18 @@ st.session_state["statistics_df"] = statistics_df st.success(f"Successfully calculated **{selected_method}** test with **{selected_fdr}** FDR correction!") - - # Display the finalized statistics table view - st.subheader("Statistical Analysis Results") - st.markdown(f"Generated framework containing columns: `{id_col}`, `log2FC`, `stat`, `p-value`, `p-adj`") - st.dataframe(statistics_df, use_container_width=True) - + except ValueError as val_err: st.error(f"Engine Validation Fallure: {str(val_err)}") except Exception as e: - st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file + st.error(f"An unexpected pipeline error occurred: {str(e)}") + +# Results are rendered from session state, outside the button block: the Insight +# table triggers a rerun the first time it is drawn, which would reset the button +# and wipe anything shown only while it is pressed. +statistics_df = st.session_state.get("statistics_df") +if statistics_df is not None: + # Display the finalized statistics table view + st.subheader("Statistical Analysis Results") + st.markdown(f"Generated framework containing columns: `{id_col}`, `log2FC`, `stat`, `p-value`, `p-adj`") + show_insight_table(statistics_df, key="statistical-result") \ No newline at end of file diff --git a/src/common/common.py b/src/common/common.py index c048064..b5b52ff 100644 --- a/src/common/common.py +++ b/src/common/common.py @@ -1,10 +1,13 @@ import os +import re import sys import uuid import json import time import psutil import shutil +import hashlib +import tempfile import pandas as pd import streamlit as st @@ -885,20 +888,155 @@ def get_current_chunk(df, chunk_size, chunk_index): return base_index + rows[0] -def show_table(df: pd.DataFrame, download_name: str = "") -> None: +# Name of the hidden unique row-id column used as the Insight Table index field. +_INSIGHT_ROW_ID = "__row_id__" +# Rows sampled when fingerprinting a dataframe for cache invalidation. +_FINGERPRINT_SAMPLE_ROWS = 2000 +# Bump when the generated Table configuration changes so old caches are rebuilt. +_INSIGHT_TABLE_VERSION = 1 + + +def _insight_table_cache_dir() -> Path: + """Directory holding OpenMS-Insight table caches for the active workspace.""" + workspace = st.session_state.get("workspace") + base = Path(workspace) if workspace else Path(tempfile.gettempdir(), "quantms-web") + return base / "insight_tables" + + +def _fingerprint_dataframe(df: pd.DataFrame) -> str: + """Cheap content hash of a dataframe, used to detect when a table must be re-cached. + + Hashing every cell on every Streamlit rerun would defeat the purpose for + large tables, so this combines the shape/schema, the column sums and an + evenly spaced sample of rows (always including the first and last row). + """ + digest = hashlib.sha256(str(_INSIGHT_TABLE_VERSION).encode()) + digest.update(repr((df.shape, [str(c) for c in df.columns], [str(t) for t in df.dtypes])).encode()) + digest.update(df.select_dtypes(include="number").sum().to_numpy(dtype="float64").tobytes()) + if len(df): + step = max(len(df) // _FINGERPRINT_SAMPLE_ROWS, 1) + rows = df.iloc[::step].astype(str) + digest.update(pd.util.hash_pandas_object(rows, index=False).values.tobytes()) + digest.update(df.iloc[[-1]].astype(str).to_csv(index=False).encode()) + return digest.hexdigest()[:16] + + +def _to_insight_frame(df: pd.DataFrame): + """Convert a pandas dataframe into a Polars frame plus Tabulator column definitions. + + - a named/non-default index becomes a regular column (it usually holds the IDs) + - column names are stringified and made unique; Tabulator treats "." in a + field name as a nested path, so it is replaced in the field (the title + keeps the original name) + - object columns with mixed types are stringified so Arrow can store them + - a hidden unique row-id column is added as the Table's index field + """ + import polars as pl + + if not isinstance(df.index, pd.RangeIndex): + df = df.reset_index() + + titles = [str(c) for c in df.columns] + fields, seen = [], set() + for title in titles: + field = title.replace(".", "_") + while field in seen or field == _INSIGHT_ROW_ID: + field += "_" + seen.add(field) + fields.append(field) + df = df.set_axis(fields, axis=1) + + try: + pl_df = pl.from_pandas(df) + except Exception: + object_cols = df.select_dtypes(include="object").columns + pl_df = pl.from_pandas(df.astype({c: str for c in object_cols})) + + column_definitions = [] + for title, field, dtype in zip(titles, fields, pl_df.dtypes): + definition = {"field": field, "title": title, "headerTooltip": True} + if dtype.is_numeric(): + definition.update(sorter="number", hozAlign="right") + else: + definition.update(sorter="string", hozAlign="left") + column_definitions.append(definition) + + return pl_df.with_row_index(_INSIGHT_ROW_ID), column_definitions + + + +def show_insight_table( + df: pd.DataFrame, + key: str, + height: int = 480, + page_size: int = 100, + cache_dir: Path | None = None, +) -> None: + """ + Displays a pandas dataframe with the OpenMS-Insight `Table` component. + + `st.dataframe` serializes the whole dataframe to the browser on every + rerun, which gets extremely slow for full-scale datasets. This component + caches the data on disk as parquet and only sends the requested page to the + browser (server-side pagination, sorting and filtering). + + The on-disk cache is keyed on the table's content: it is written once per + distinct dataframe and reused on every rerun, and caches of older versions + of the same table are removed when the data changes. + + Args: + df: The pandas dataframe to display. + key: Stable name for this table (unique per table on the app, e.g. + "filtering-original"). Must not contain "--". + height: Component height in pixels. + page_size: Rows fetched per page. + cache_dir: Cache location. Defaults to the active workspace. + """ + from openms_insight import Table + from openms_insight.core.cache import get_cache_dir + + cache_dir = Path(cache_dir) if cache_dir else _insight_table_cache_dir() + cache_dir.mkdir(parents=True, exist_ok=True) + + safe_key = re.sub(r"[^A-Za-z0-9_-]", "_", key) + cache_id = f"{safe_key}--{_fingerprint_dataframe(df)}" + + if (get_cache_dir(str(cache_dir), cache_id) / "manifest.json").exists(): + # Unchanged data: restore from the cache, no preprocessing. + table = Table(cache_id=cache_id, cache_path=str(cache_dir)) + else: + pl_df, column_definitions = _to_insight_frame(df) + table = Table( + cache_id=cache_id, + data=pl_df.lazy(), + cache_path=str(cache_dir), + column_definitions=column_definitions, + index_field=_INSIGHT_ROW_ID, + page_size=page_size, + ) + # Drop caches of previous versions of this table. + for stale in cache_dir.glob(f"{safe_key}--*"): + if stale.name != cache_id and stale.is_dir(): + shutil.rmtree(stale, ignore_errors=True) + + table(height=height) + + +def show_table(df: pd.DataFrame, download_name: str = "", key: str | None = None) -> None: """ - Displays a pandas dataframe using Streamlit's `dataframe` function and - provides a download button for the same table. + Displays a pandas dataframe with the OpenMS-Insight `Table` component + (server-side pagination, see `show_insight_table`) and provides a download + button for the same table. Args: df (pd.DataFrame): The pandas dataframe to display. download_name (str): The name to give to the downloaded file. Defaults to empty string. + key (str): Unique name of the table. Defaults to `download_name`. Returns: - df (pd.DataFrame): The possibly edited dataframe. + df (pd.DataFrame): The dataframe. """ - # Show dataframe using container width - st.dataframe(df, use_container_width=True) + show_insight_table(df, key=key or download_name or "table") # Show download button with the given download name for the table if name is given if download_name: st.download_button( diff --git a/tests/test_insight_table.py b/tests/test_insight_table.py new file mode 100644 index 0000000..a7e392d --- /dev/null +++ b/tests/test_insight_table.py @@ -0,0 +1,82 @@ +"""Tests for the OpenMS-Insight table helpers in src/common/common.py.""" +import numpy as np +import pandas as pd +import polars as pl + +from src.common.common import ( + _INSIGHT_ROW_ID, + _fingerprint_dataframe, + _to_insight_frame, +) + + +def _abundance_df(): + return pd.DataFrame( + { + "ProteinName": ["P1", "P2", "P3"], + "m/z": [1.5, np.nan, 3.5], + "sample1[126.1]": [1, 2, 3], + "PeptideSequence": ["AAK", "BBK", "CCK"], + } + ) + + +def test_to_insight_frame_adds_unique_hidden_row_id(): + pl_df, defs = _to_insight_frame(_abundance_df()) + assert pl_df[_INSIGHT_ROW_ID].n_unique() == 3 + assert _INSIGHT_ROW_ID not in [d["field"] for d in defs] + + +def test_to_insight_frame_keeps_titles_and_sanitizes_dotted_fields(): + pl_df, defs = _to_insight_frame(_abundance_df()) + by_title = {d["title"]: d for d in defs} + # Tabulator reads "." in a field name as a nested path, so it is replaced; + # the visible title keeps the original column name. + assert by_title["sample1[126.1]"]["field"] == "sample1[126_1]" + assert by_title["m/z"]["field"] == "m/z" + assert set(d["field"] for d in defs) <= set(pl_df.columns) + + +def test_to_insight_frame_makes_duplicate_column_names_unique(): + df = pd.DataFrame([[1, 2]], columns=["a.b", "a_b"]) + _, defs = _to_insight_frame(df) + assert len({d["field"] for d in defs}) == 2 + + +def test_to_insight_frame_sorters_follow_dtype(): + _, defs = _to_insight_frame(_abundance_df()) + by_title = {d["title"]: d for d in defs} + assert by_title["m/z"]["sorter"] == "number" + assert by_title["ProteinName"]["sorter"] == "string" + + +def test_to_insight_frame_turns_named_index_into_column(): + df = _abundance_df().set_index("ProteinName") + pl_df, defs = _to_insight_frame(df) + assert "ProteinName" in pl_df.columns + assert defs[0]["title"] == "ProteinName" + + +def test_to_insight_frame_nan_becomes_null_and_mixed_objects_are_stringified(): + df = pd.DataFrame({"x": [1.0, np.nan], "mixed": ["a", 1]}) + pl_df, _ = _to_insight_frame(df) + assert pl_df["x"].null_count() == 1 + assert pl_df["mixed"].dtype == pl.String + + +def test_fingerprint_is_stable_and_detects_changes(): + df = _abundance_df() + assert _fingerprint_dataframe(df) == _fingerprint_dataframe(df.copy()) + + changed_value = df.copy() + changed_value.loc[0, "sample1[126.1]"] = 99 + assert _fingerprint_dataframe(changed_value) != _fingerprint_dataframe(df) + + assert _fingerprint_dataframe(df.iloc[:2]) != _fingerprint_dataframe(df) + assert _fingerprint_dataframe(df.rename(columns={"m/z": "mz"})) != _fingerprint_dataframe(df) + + +def test_fingerprint_handles_duplicate_columns_and_list_cells(): + _fingerprint_dataframe(pd.DataFrame([[1, 2]], columns=["a", "a"])) + _fingerprint_dataframe(pd.DataFrame({"x": [[1, 2], [3]]})) + _fingerprint_dataframe(pd.DataFrame({"x": []}))