feat: LongMemEval-S dataset, grouped-corpus runner mode, and converter

.gitignore

@@ -16,3 +16,5 @@ benchmarks/datasets/locomo/locomo10.json
 benchmarks/datasets/locomo/locomo10.provenance.json
 .env
 benchmarks/bm-home/
+benchmarks/datasets/longmemeval/longmemeval_s.json
+benchmarks/datasets/longmemeval/longmemeval_s.provenance.json

README.md

@@ -98,6 +98,30 @@ uv run bm-bench run judge --run-dir benchmarks/runs/<run-id>
 uv run bm-bench publish --run-dir benchmarks/runs/<run-id>
 ```
 
+## LongMemEval-S
+
+LongMemEval (Wu et al., ICLR 2025) gives each of its 500 questions an
+independent haystack of ~50 chat sessions, so it runs in **grouped mode**: the
+converter writes one corpus per question under `groups/<question_id>/docs`,
+and the runner ingests + queries each group in isolation (fresh provider
+instance, group-suffixed run id namespacing the BM project / mem0 user).
+
+```bash
+just bench-prepare-longmemeval        # fetch (~278MB) + convert all 500
+just bench-convert-longmemeval-dev    # or: 25-question dev slice
+just bench-run-longmemeval-dev        # grouped retrieval, bm-local
+```
+
+Then score answers with the QA stage as usual (`run qa --run-dir ...`). The
+question's ask-date is carried in query metadata and appended to the question
+for both the answerer and the judge — temporal-reasoning questions are
+unanswerable without it.
+
+Anti-leakage: the raw dataset marks evidence sessions via an `answer_`
+session-id prefix and per-turn `has_answer` flags. The converter remaps all
+session ids to neutral positional ids (`<qid>-s012`) and drops turn flags, so
+ingested corpora carry no evidence markers.
+
 ## Basic Memory source policy
 
 By default this project tracks Basic Memory from `main`.

justfile

@@ -9,6 +9,9 @@ bm_local_path_flag := if bm_local_path != "" { "--bm-local-path " + bm_local_pat
 locomo_dataset_path := "benchmarks/datasets/locomo/locomo10.json"
 locomo_output_dir := "benchmarks/generated/locomo"
 locomo_c1_output_dir := "benchmarks/generated/locomo-c1"
+longmemeval_dataset_path := "benchmarks/datasets/longmemeval/longmemeval_s.json"
+longmemeval_output_dir := "benchmarks/generated/longmemeval-s"
+longmemeval_dev_output_dir := "benchmarks/generated/longmemeval-s-dev"
 
 # --- Repo maintenance ---
 
@@ -50,6 +53,40 @@ bench-prepare-short: bench-fetch-locomo bench-convert-locomo-c1 bench-make-quick
 
 bench-prepare-long: bench-fetch-locomo bench-convert-locomo
 
+bench-fetch-longmemeval:
+    uv run bm-bench datasets fetch --dataset longmemeval-s --output {{longmemeval_dataset_path}}
+
+bench-convert-longmemeval:
+    uv run bm-bench convert longmemeval --dataset-path {{longmemeval_dataset_path}} --output-dir {{longmemeval_output_dir}}
+
+# Dev slice: first 25 questions for fast iteration
+bench-convert-longmemeval-dev:
+    uv run bm-bench convert longmemeval --dataset-path {{longmemeval_dataset_path}} --output-dir {{longmemeval_dev_output_dir}} --max-questions 25
+
+bench-prepare-longmemeval: bench-fetch-longmemeval bench-convert-longmemeval
+
+# Grouped retrieval over the LongMemEval-S dev slice (bm-local only)
+bench-run-longmemeval-dev:
+    uv run bm-bench run retrieval \
+      --dataset-id longmemeval_s \
+      --dataset-path {{longmemeval_dataset_path}} \
+      --corpus-dir {{longmemeval_dev_output_dir}}/groups \
+      --queries-path {{longmemeval_dev_output_dir}}/queries.json \
+      --providers bm-local \
+      {{bm_local_path_flag}} \
+      --strict-providers
+
+# Grouped retrieval over full LongMemEval-S (slow: 500 isolated group corpora)
+bench-run-longmemeval:
+    uv run bm-bench run retrieval \
+      --dataset-id longmemeval_s \
+      --dataset-path {{longmemeval_dataset_path}} \
+      --corpus-dir {{longmemeval_output_dir}}/groups \
+      --queries-path {{longmemeval_output_dir}}/queries.json \
+      --providers bm-local,mem0-local \
+      {{bm_local_path_flag}} \
+      --allow-provider-skip
+
 # --- One-command pipelines ---
 
 # Full retrieval benchmark pipeline:

src/basic_memory_benchmarks/cli.py

@@ -10,9 +10,17 @@
 from rich.console import Console
 
 from basic_memory_benchmarks.converters.locomo_to_corpus import convert_locomo_to_corpus
+from basic_memory_benchmarks.converters.longmemeval_to_corpus import convert_longmemeval_to_corpus
 from basic_memory_benchmarks.datasets.locomo import LOCOMO_URL, fetch_locomo_dataset
+from basic_memory_benchmarks.datasets.longmemeval import (
+    LONGMEMEVAL_S_URL,
+    fetch_longmemeval_dataset,
+)
 from basic_memory_benchmarks.models import DatasetProvenance, RunConfig
-from basic_memory_benchmarks.reporting.compare import compare_provider_metric, load_retrieval_summary
+from basic_memory_benchmarks.reporting.compare import (
+    compare_provider_metric,
+    load_retrieval_summary,
+)
 from basic_memory_benchmarks.runner import run_judge, run_qa_stage, run_retrieval
 from basic_memory_benchmarks.utils import sha256_file
 
@@ -31,20 +39,29 @@
 @datasets_app.command("fetch")
 def datasets_fetch(
     dataset: str = typer.Option("locomo", "--dataset"),
-    output: Path = typer.Option(Path("benchmarks/datasets/locomo/locomo10.json"), "--output"),
-    url: str = typer.Option(LOCOMO_URL, "--url"),
+    output: Path | None = typer.Option(None, "--output"),
+    url: str | None = typer.Option(None, "--url"),
 ) -> None:
-    if dataset != "locomo":
-        raise typer.BadParameter("Only locomo is supported in v1")
-
-    provenance = fetch_locomo_dataset(output_path=output, url=url)
-    console.print(f"Downloaded {dataset} to [cyan]{output}[/cyan]")
+    if dataset == "locomo":
+        resolved_output = output or Path("benchmarks/datasets/locomo/locomo10.json")
+        provenance = fetch_locomo_dataset(output_path=resolved_output, url=url or LOCOMO_URL)
+    elif dataset == "longmemeval-s":
+        resolved_output = output or Path("benchmarks/datasets/longmemeval/longmemeval_s.json")
+        provenance = fetch_longmemeval_dataset(
+            output_path=resolved_output, url=url or LONGMEMEVAL_S_URL
+        )
+    else:
+        raise typer.BadParameter("Supported datasets: locomo, longmemeval-s")
+
+    console.print(f"Downloaded {dataset} to [cyan]{resolved_output}[/cyan]")
     console.print(f"SHA256: [green]{provenance.checksum_sha256}[/green]")
 
 
 @convert_app.command("locomo")
 def convert_locomo(
-    dataset_path: Path = typer.Option(Path("benchmarks/datasets/locomo/locomo10.json"), "--dataset-path"),
+    dataset_path: Path = typer.Option(
+        Path("benchmarks/datasets/locomo/locomo10.json"), "--dataset-path"
+    ),
     output_dir: Path = typer.Option(Path("benchmarks/generated/locomo"), "--output-dir"),
     max_conversations: int | None = typer.Option(None, "--max-conversations"),
 ) -> None:
@@ -57,13 +74,34 @@ def convert_locomo(
     console.print(f"Queries: [cyan]{queries_path}[/cyan] ({query_count})")
 
 
+@convert_app.command("longmemeval")
+def convert_longmemeval(
+    dataset_path: Path = typer.Option(
+        Path("benchmarks/datasets/longmemeval/longmemeval_s.json"), "--dataset-path"
+    ),
+    output_dir: Path = typer.Option(Path("benchmarks/generated/longmemeval-s"), "--output-dir"),
+    max_questions: int | None = typer.Option(None, "--max-questions"),
+) -> None:
+    groups_dir, queries_path, doc_count, query_count = convert_longmemeval_to_corpus(
+        dataset_path=dataset_path,
+        output_dir=output_dir,
+        max_questions=max_questions,
+    )
+    console.print(f"Groups: [cyan]{groups_dir}[/cyan] ({query_count} groups, {doc_count} docs)")
+    console.print(f"Queries: [cyan]{queries_path}[/cyan] ({query_count})")
+
+
 @run_app.command("retrieval")
 def run_retrieval_command(
     providers: str = typer.Option("bm-local,mem0-local", "--providers"),
     dataset_id: str = typer.Option("locomo", "--dataset-id"),
-    dataset_path: Path = typer.Option(Path("benchmarks/datasets/locomo/locomo10.json"), "--dataset-path"),
+    dataset_path: Path = typer.Option(
+        Path("benchmarks/datasets/locomo/locomo10.json"), "--dataset-path"
+    ),
     corpus_dir: Path = typer.Option(Path("benchmarks/generated/locomo/docs"), "--corpus-dir"),
-    queries_path: Path = typer.Option(Path("benchmarks/generated/locomo/queries.json"), "--queries-path"),
+    queries_path: Path = typer.Option(
+        Path("benchmarks/generated/locomo/queries.json"), "--queries-path"
+    ),
     output_root: Path = typer.Option(Path("benchmarks/runs"), "--output-root"),
     run_id: str | None = typer.Option(None, "--run-id"),
     top_k: int = typer.Option(10, "--top-k"),
@@ -148,9 +186,13 @@ def run_judge_command(
 def run_full_command(
     providers: str = typer.Option("bm-local,mem0-local", "--providers"),
     dataset_id: str = typer.Option("locomo", "--dataset-id"),
-    dataset_path: Path = typer.Option(Path("benchmarks/datasets/locomo/locomo10.json"), "--dataset-path"),
+    dataset_path: Path = typer.Option(
+        Path("benchmarks/datasets/locomo/locomo10.json"), "--dataset-path"
+    ),
     corpus_dir: Path = typer.Option(Path("benchmarks/generated/locomo/docs"), "--corpus-dir"),
-    queries_path: Path = typer.Option(Path("benchmarks/generated/locomo/queries.json"), "--queries-path"),
+    queries_path: Path = typer.Option(
+        Path("benchmarks/generated/locomo/queries.json"), "--queries-path"
+    ),
     output_root: Path = typer.Option(Path("benchmarks/runs"), "--output-root"),
     run_id: str | None = typer.Option(None, "--run-id"),
     top_k: int = typer.Option(10, "--top-k"),

src/basic_memory_benchmarks/converters/longmemeval_to_corpus.py

@@ -0,0 +1,134 @@
+"""Convert LongMemEval-S into grouped benchmark corpora and a query manifest.
+
+Each LongMemEval question carries its own haystack of chat sessions, so the
+output is one corpus directory per question (``groups/<question_id>/docs``)
+plus a single ``queries.json`` whose entries name their group. The runner
+ingests and queries each group in isolation, matching the official protocol.
+
+Anti-leakage: the raw dataset marks evidence sessions with an ``answer_``
+session-id prefix and evidence turns with ``has_answer`` flags. Session ids
+are remapped to neutral positional ids and turn flags are dropped, so nothing
+ingested by a provider distinguishes evidence from filler.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from basic_memory_benchmarks.datasets.longmemeval import load_longmemeval_dataset
+
+DATASET_ID = "longmemeval_s"
+
+
+def _render_session_doc(
+    doc_id: str,
+    session_date: str,
+    turns: list[dict],
+) -> str:
+    lines: list[str] = [
+        "---",
+        f"title: {doc_id} ({session_date})",
+        "type: note",
+        f"source_doc_id: {doc_id}",
+        f"dataset_id: {DATASET_ID}",
+        f"session_date: {session_date}",
+        "---",
+        "",
+        f"# Chat session on {session_date}",
+        "",
+        "## Conversation",
+    ]
+    for turn in turns:
+        role = str(turn.get("role", "unknown")).capitalize()
+        content = str(turn.get("content", "")).strip()
+        if not content:
+            continue
+        # Keep each turn on one line so bullet-level chunking stays intact.
+        lines.append(f"- **{role}:** {' '.join(content.split())}")
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def convert_longmemeval_to_corpus(
+    dataset_path: Path,
+    output_dir: Path,
+    max_questions: int | None = None,
+) -> tuple[Path, Path, int, int]:
+    """Convert LongMemEval-S into per-question corpora + query manifest.
+
+    Returns:
+        groups_dir, queries_path, doc_count, query_count
+    """
+    entries = load_longmemeval_dataset(dataset_path)
+    if max_questions is not None:
+        entries = entries[:max_questions]
+
+    groups_dir = output_dir / "groups"
+    groups_dir.mkdir(parents=True, exist_ok=True)
+
+    all_queries: list[dict] = []
+    doc_count = 0
+
+    for entry in entries:
+        question_id = str(entry["question_id"])
+        sessions = entry["haystack_sessions"]
+        session_ids = entry["haystack_session_ids"]
+        session_dates = entry["haystack_dates"]
+        if not (len(sessions) == len(session_ids) == len(session_dates)):
+            raise ValueError(
+                f"Question {question_id}: haystack arrays misaligned "
+                f"({len(sessions)} sessions, {len(session_ids)} ids, {len(session_dates)} dates)"
+            )
+
+        docs_dir = groups_dir / question_id / "docs"
+        docs_dir.mkdir(parents=True, exist_ok=True)
+
+        # Neutral positional doc ids; the raw ids leak evidence via the
+        # "answer_" prefix. A few haystacks repeat a session id — keep the
+        # first occurrence so each session is ingested once.
+        doc_id_by_session_id: dict[str, str] = {}
+        for index, (session, session_id, session_date) in enumerate(
+            zip(sessions, session_ids, session_dates)
+        ):
+            if str(session_id) in doc_id_by_session_id:
+                continue
+            doc_id = f"{question_id}-s{index:03d}"
+            doc_id_by_session_id[str(session_id)] = doc_id
+            doc_path = docs_dir / f"{doc_id}.md"
+            doc_path.write_text(
+                _render_session_doc(doc_id, str(session_date), session),
+                encoding="utf-8",
+            )
+            doc_count += 1
+
+        ground_truth: list[str] = []
+        for answer_session_id in entry["answer_session_ids"]:
+            mapped = doc_id_by_session_id.get(str(answer_session_id))
+            if mapped is None:
+                raise ValueError(
+                    f"Question {question_id}: answer session {answer_session_id!r} "
+                    "not present in haystack"
+                )
+            ground_truth.append(mapped)
+
+        is_abstention = question_id.endswith("_abs")
+        all_queries.append(
+            {
+                "id": question_id,
+                "query": str(entry["question"]).strip(),
+                "category": str(entry["question_type"]),
+                "group": question_id,
+                "ground_truth": sorted(ground_truth),
+                "expected_answer": str(entry["answer"]).strip(),
+                "metadata": {
+                    "dataset_id": DATASET_ID,
+                    "question_date": str(entry["question_date"]),
+                    "abstention": is_abstention,
+                },
+            }
+        )
+
+    queries_path = output_dir / "queries.json"
+    queries_path.write_text(json.dumps(all_queries, indent=2), encoding="utf-8")
+
+    return groups_dir, queries_path, doc_count, len(all_queries)