Use OpenMS-Insight Table for large tables

CLAUDE.md

@@ -169,5 +169,7 @@ docker-compose up --build
 - Widget keys must match parameter keys in `default-parameters.json`
 - Workflow names use lowercase with hyphens: "My Workflow" -> "my-workflow"
 - Use `show_fig()` and `show_table()` from `src/common/common.py` for consistent display
+- Show DataFrames with `show_insight_table(df, key=...)` (or `show_table()`), not `st.dataframe()`: it uses OpenMS-Insight's server-side-paginated `Table`, so only the visible page is sent to the browser. `st.dataframe` serializes the whole frame and is extremely slow on full-scale datasets. Only use it for tiny fixed-size previews (e.g. `.head(10)`)
+- Insight components request a rerun the first time they render, so never draw them only inside `if st.button(...):`. Store the result in `st.session_state` and render it outside the button block
 - Use `@st.fragment` on methods that should partially rerun (configure, results)
 - TOPP tool parameters use colon-separated paths: `"algorithm:section:param_name"`

content/enrichment.py

@@ -4,7 +4,7 @@
 import pandas as pd
 import polars as pl
 import streamlit as st
-from src.common.common import page_setup
+from src.common.common import page_setup, show_insight_table
 from src.common.results_helpers import get_abundance_data, get_id_column
 # Import GO Enrichment modules from openms_insight engine
 from openms_insight.analysis.enrichment import calculate_go_enrichment
@@ -136,6 +136,6 @@
                     st.plotly_chart(fig, use_container_width=True)
 
                     st.subheader(f"📊 {go_type} Results Dataframe")
-                    st.dataframe(df_go, use_container_width=True)
+                    show_insight_table(df_go, key=f"enrichment-go-{go_type}", height=350)
                 else:
                     st.info(f"No statistically overrepresented terms identified for Category: **{go_type}**")

content/filtering.py

@@ -4,7 +4,7 @@
 import pandas as pd
 import polars as pl
 import streamlit as st
-from src.common.common import page_setup
+from src.common.common import page_setup, show_insight_table
 from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map
 
 # Import filtering functions from openms_insight package
@@ -61,7 +61,7 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame:
 st.markdown(
     f"Currently displaying **{pivot_df.shape[0]}** proteins and **{len(sample_cols)}** samples before filtering."
 )
-st.dataframe(pivot_df, use_container_width=True)
+show_insight_table(pivot_df, key="filtering-original")
 
 st.markdown("---")
 
@@ -153,9 +153,13 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame:
     filtered_df = strip_stat_columns(filtered_lazy.collect().to_pandas())
     st.session_state["filtered_df"] = filtered_df
 
-    # Layout response metrics and the filtered matrix
     st.success(f"Successfully applied **{filter_method}** filter!")
 
+# Results are rendered from session state, outside the button block: the Insight
+# table triggers a rerun the first time it is drawn, which would reset the button
+# and wipe anything shown only while it is pressed.
+filtered_df = st.session_state.get("filtered_df")
+if filtered_df is not None:
     # Display dataset scale compression stats
     col1, col2, col3 = st.columns(3)
     col1.metric("Original Proteins", pivot_df.shape[0])
@@ -170,4 +174,4 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame:
             "The filtered table is empty. Try relaxing the threshold constraints."
         )
     else:
-        st.dataframe(filtered_df, use_container_width=True)
+        show_insight_table(filtered_df, key="filtering-filtered")

content/imputation.py

@@ -4,7 +4,7 @@
 import pandas as pd
 import polars as pl
 import streamlit as st
-from src.common.common import page_setup
+from src.common.common import page_setup, show_insight_table
 from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map
 
 # Import imputation algorithms from openms_insight engine
@@ -69,7 +69,7 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame:
 st.markdown(
     f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples before imputation."
 )
-st.dataframe(base_df, use_container_width=True)
+show_insight_table(base_df, key="imputation-input")
 
 st.markdown("---")
 
@@ -140,6 +140,10 @@ def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame:
 
     st.success(f"Successfully finalized **{impute_category}** imputation step!")
 
-    # Calculate and display a quick performance matrix check
+# Results are rendered from session state, outside the button block: the Insight
+# table triggers a rerun the first time it is drawn, which would reset the button
+# and wipe anything shown only while it is pressed.
+imputed_df = st.session_state.get("imputed_df")
+if imputed_df is not None:
     st.subheader("Imputed Result Table")
-    st.dataframe(imputed_df, use_container_width=True)
+    show_insight_table(imputed_df, key="imputation-result")

content/normalization.py

@@ -4,7 +4,7 @@
 import pandas as pd
 import polars as pl
 import streamlit as st
-from src.common.common import page_setup
+from src.common.common import page_setup, show_insight_table
 from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map
 # Import normalization engine functions from openms_insight
 from openms_insight.analysis.normalization import (
@@ -94,7 +94,7 @@ def strip_stat_columns(df: pd.DataFrame | None) -> pd.DataFrame | None:
 st.markdown(
     f"Currently displaying **{base_df.shape[0]}** rows and **{len(sample_cols)}** samples entering the normalization block."
 )
-st.dataframe(base_df, use_container_width=True)
+show_insight_table(base_df, key="normalization-input")
 
 st.markdown("### Pipeline Overview")
 st.caption("Data flows in order: Filtering -> Imputation -> Normalization")
@@ -231,12 +231,17 @@ def strip_stat_columns(df: pd.DataFrame | None) -> pd.DataFrame | None:
 
         st.success("Successfully executed all selected normalization pipelines!")
 
-        # Display the finalized transformation matrix view
-        st.subheader("Normalized Abundance Table")
-        st.dataframe(normalized_df, use_container_width=True)
-
     except ValueError as val_err:
         # Gracefully handle validation failures raised from the engine layers (e.g., missing reference protein)
         st.error(f"Engine Configuration Error: {str(val_err)}")
     except Exception as e:
-        st.error(f"An unexpected pipeline error occurred: {str(e)}")
+        st.error(f"An unexpected pipeline error occurred: {str(e)}")
+
+# Results are rendered from session state, outside the button block: the Insight
+# table triggers a rerun the first time it is drawn, which would reset the button
+# and wipe anything shown only while it is pressed.
+normalized_df = st.session_state.get("normalized_df")
+if normalized_df is not None:
+    # Display the finalized transformation matrix view
+    st.subheader("Normalized Abundance Table")
+    show_insight_table(normalized_df, key="normalization-result")

content/results_pathway_analysis.py

@@ -8,7 +8,7 @@
 from collections import defaultdict        
 from scipy.stats import fisher_exact
 from pathlib import Path
-from src.common.common import page_setup
+from src.common.common import page_setup, show_insight_table
 from src.common.results_helpers import get_abundance_data
 
 # ================================
@@ -216,7 +216,7 @@ def run_go(go_type):
         st.info("No protein-level data available.")
     else:
         st.session_state["pivot_df"] = pivot_df
-        st.dataframe(pivot_df.sort_values("p-value"), width="stretch")
+        show_insight_table(pivot_df.sort_values("p-value"), key="pathway-protein")
 
 # ======================================================
 # GO Enrichment Results 
@@ -255,4 +255,4 @@ def run_go(go_type):
                 st.plotly_chart(fig, width="stretch")
 
                 st.markdown(f"#### {go_type} Enrichment Results")
-                st.dataframe(df_go, width="stretch")
+                show_insight_table(df_go, key=f"pathway-go-{go_type}", height=350)

content/results_pca.py

@@ -2,7 +2,7 @@
 import pandas as pd
 import polars as pl
 import streamlit as st
-from src.common.common import page_setup
+from src.common.common import page_setup, show_insight_table
 from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map
 from openms_insight import PCAPlot
 
@@ -86,7 +86,7 @@
     f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples "
     f"belonging to **{len(unique_groups)} groups** ({', '.join(unique_groups)})."
 )
-st.dataframe(base_df, use_container_width=True)
+show_insight_table(base_df, key="pca-input")
 
 st.markdown("---")
 

content/results_proteomicslfq.py

@@ -4,7 +4,7 @@
 import numpy as np
 import plotly.express as px
 
-from src.common.common import page_setup
+from src.common.common import page_setup, show_insight_table
 from src.common.results_helpers import get_abundance_data
 
 # ================================
@@ -52,7 +52,7 @@
         st.info("No protein-level data available.")
     else:
         st.session_state["pivot_df"] = pivot_df
-        st.dataframe(pivot_df, use_container_width=True)
+        show_insight_table(pivot_df, key="proteomicslfq-protein")
 
 # ======================================================
 # GO Enrichment Results 
@@ -97,4 +97,4 @@
                 st.plotly_chart(fig, use_container_width=True)
 
                 st.markdown(f"#### {go_type} Enrichment Results")
-                st.dataframe(df_go, use_container_width=True)
+                show_insight_table(df_go, key=f"proteomicslfq-go-{go_type}", height=350)

content/statistical.py

@@ -4,7 +4,7 @@
 import pandas as pd
 import polars as pl
 import streamlit as st
-from src.common.common import page_setup
+from src.common.common import page_setup, show_insight_table
 from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map
 # Import statistics engine functions from openms_insight
 from openms_insight.analysis.statistics import calculate_statistical_tests, adjust_fdr_lazy
@@ -80,7 +80,7 @@
 st.markdown(
     f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples belonging to **{group_count} groups** ({', '.join(unique_groups)})."
 )
-st.dataframe(base_df, use_container_width=True)
+show_insight_table(base_df, key="statistical-input")
 
 st.markdown("---")
 
@@ -154,13 +154,18 @@
         st.session_state["statistics_df"] = statistics_df
 
         st.success(f"Successfully calculated **{selected_method}** test with **{selected_fdr}** FDR correction!")
-
-        # Display the finalized statistics table view
-        st.subheader("Statistical Analysis Results")
-        st.markdown(f"Generated framework containing columns: `{id_col}`, `log2FC`, `stat`, `p-value`, `p-adj`")
-        st.dataframe(statistics_df, use_container_width=True)
-
+
     except ValueError as val_err:
         st.error(f"Engine Validation Fallure: {str(val_err)}")
     except Exception as e:
-        st.error(f"An unexpected pipeline error occurred: {str(e)}")
+        st.error(f"An unexpected pipeline error occurred: {str(e)}")
+
+# Results are rendered from session state, outside the button block: the Insight
+# table triggers a rerun the first time it is drawn, which would reset the button
+# and wipe anything shown only while it is pressed.
+statistics_df = st.session_state.get("statistics_df")
+if statistics_df is not None:
+    # Display the finalized statistics table view
+    st.subheader("Statistical Analysis Results")
+    st.markdown(f"Generated framework containing columns: `{id_col}`, `log2FC`, `stat`, `p-value`, `p-adj`")
+    show_insight_table(statistics_df, key="statistical-result")