From 74269708d0e7619e11a0f15b16674cb02252cbcb Mon Sep 17 00:00:00 2001 From: Jukka Raimovaara Date: Fri, 16 Oct 2026 10:49:34 +0300 Subject: [PATCH 1/9] Fix lookup of committed fixture history The history regex captures `(^|\n)` before the path, so destructuring `[, relativePath]` read that group: every committed fixture was keyed by "" or "\n", none ever matched, and every fixture was included in every migration. Use the path group, and normalise it so `./x.sql` and `x.sql` are the same fixture. Co-Authored-By: Claude Opus 5.5 --- src/fixtures.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/fixtures.ts b/src/fixtures.ts index 7015d82..fec74f6 100644 --- a/src/fixtures.ts +++ b/src/fixtures.ts @@ -1,6 +1,6 @@ import * as crypto from "crypto"; import { promises as fsp } from "fs"; -import { sep } from "path"; +import { posix, sep } from "path"; import pgMinify from "pg-minify"; import { isNoTransactionDefined } from "./header"; @@ -102,10 +102,12 @@ async function _reallyGetCommittedFixturesCache( const re = /(^|\n)--! Included ([^\r\n]+)\r?\n/g; let match: RegExpExecArray | null; while ((match = re.exec(body)) != null) { - const [, relativePath] = match; - const i = body.indexOf(`--! EndIncluded ${relativePath}`, match.index); + const [, , includePath] = match; + const i = body.indexOf(`--! EndIncluded ${includePath}`, match.index); if (i < 0) throw new Error(`Found Included with no EndIncluded`); const content = body.substring(match.index + match[0].length, i).trim(); + // `./auto/x.sql` and `auto/x.sql` are the same fixture. + const relativePath = posix.normalize(includePath); cache.fixtures[relativePath] ??= []; const hash = hashFixture(content); cache.fixtures[relativePath].push({ filename, hash }); From 236a1e25a90b3cab9f596ffbb2c1d9de6dad653a Mon Sep 17 00:00:00 2001 From: Jukka Raimovaara Date: Fri, 16 Oct 2026 10:49:34 +0300 Subject: [PATCH 2/9] Only include fixtures from fixtures/auto automatically Fixtures elsewhere (indexes, constraints, anything else that should not be re-run on change) are only included manually. Fixtures in fixtures/auto must not use --!include, so none is run more than once through nesting and each one's hash comes straight from its file. Also record which fixtures are empty or only comments. Co-Authored-By: Claude Opus 5.5 --- src/fixtures.ts | 41 +++++++++++++++++++++++++++++------------ src/settings.ts | 16 +++++++++------- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/src/fixtures.ts b/src/fixtures.ts index fec74f6..26ae476 100644 --- a/src/fixtures.ts +++ b/src/fixtures.ts @@ -5,7 +5,7 @@ import pgMinify from "pg-minify"; import { isNoTransactionDefined } from "./header"; import { realpathOrNull, walk } from "./lib"; -import { compileIncludes, getAllMigrations } from "./migration"; +import { getAllMigrations } from "./migration"; import type { ParsedSettings } from "./settings"; interface FileFixture { @@ -25,8 +25,19 @@ interface FileFixture { hash: string; hasNoTransaction: boolean; + + /** + * True if the fixture is empty or only comments + */ + isEmpty: boolean; } +/** + * Only fixtures in this folder (inside the fixtures folder) are included + * automatically. They must be self-contained: no `--!include`. + */ +export const AUTO_FIXTURES_FOLDER = "auto"; + interface FixturesCache { fixtures: { [relativePath: string]: Array<{ @@ -44,29 +55,35 @@ function hashFixture(fixtureText: string) { .digest("hex"); } +const EMPTY_FIXTURE_HASH = hashFixture(""); + export async function getLocalFixtures( parsedSettings: ParsedSettings, ): Promise<{ [relativePath: string]: FileFixture }> { const allFixtures = Object.create(null) as { [relativePath: string]: FileFixture; }; - const rawFixturesPath = `${parsedSettings.migrationsFolder}/fixtures`; - const fixturesPath = await realpathOrNull(rawFixturesPath); - if (fixturesPath) { - await walk(fixturesPath, async (relativePath, ent) => { + const rawAutoPath = `${parsedSettings.migrationsFolder}/fixtures/${AUTO_FIXTURES_FOLDER}`; + const autoPath = await realpathOrNull(rawAutoPath); + if (autoPath) { + await walk(autoPath, async (pathInAuto, ent) => { if (ent.name.startsWith(".") || !ent.name.endsWith(".sql")) return; + const relativePath = `${AUTO_FIXTURES_FOLDER}/${pathInAuto}`; const fullPath = ent.parentPath + sep + ent.name; const contents = await fsp.readFile(fullPath, "utf8"); - const compiled = await compileIncludes( - parsedSettings, - contents, - new Set([fullPath]), - ); + // Nested includes would run shared fixtures more than once. + if (/^--![ \t]*include\b/m.test(contents)) { + throw new Error( + `Automatic fixture '${relativePath}' uses '--!include'; fixtures in '${AUTO_FIXTURES_FOLDER}/' must be self-contained.`, + ); + } + const hash = hashFixture(contents); allFixtures[relativePath] = { fullPath, relativePath, - hash: hashFixture(compiled), - hasNoTransaction: isNoTransactionDefined(compiled), + hash, + hasNoTransaction: isNoTransactionDefined(contents), + isEmpty: hash === EMPTY_FIXTURE_HASH, }; }); } diff --git a/src/settings.ts b/src/settings.ts index 0f2af83..0f4019b 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -85,14 +85,16 @@ export interface Settings { blankMigrationContent?: string; logger?: Logger; /** - * On `commit` (or when using `watch` mode), find fixtures that have changed - * since the last commit and include them in this migration. + * On `commit` (or when using `watch` mode), find fixtures in + * `migrations/fixtures/auto/` that have changed since the last commit and + * include them in this migration. Fixtures in that folder must not use + * `--!include`; fixtures elsewhere are only included manually. * - * NOTE: in this mode, fixtures should only be used for stateless database - * entities: functions, RLS policies, grants, views, etc - things that can be - * dropped and recreated almost instantly. You should be very wary of using - * them with stateful entities such as indexes, constraints and materialized - * views as dropping and recreating these takes significant time. + * NOTE: only use the `auto` folder for stateless database entities: + * functions, RLS policies, grants, views, etc - things that can be dropped + * and recreated almost instantly. Keep stateful entities such as indexes, + * constraints and materialized views in other fixtures, as dropping and + * recreating these takes significant time. * * @experimental */ From 9f1c949db96bd826e538de20571c56512a54ff50 Mon Sep 17 00:00:00 2001 From: Jukka Raimovaara Date: Fri, 16 Oct 2026 10:49:34 +0300 Subject: [PATCH 3/9] Keep headers first and skip fixtures the migration already includes - Put the automatic fixtures after the headers and `--! no-transaction`, so `commit` keeps the message and no-transaction migrations stay outside a transaction; strip only that section on uncommit, so headers such as the message survive it too. - Read already-included fixtures from the compiled migration, so other spellings (`./auto/x.sql`) and includes through another fixture count. - Ignore new fixtures that are empty or only comments. - Let uncommit check the current migration without automatic fixtures; they are derived again afterwards. Co-Authored-By: Claude Opus 5.5 --- src/commands/uncommit.ts | 3 +- src/current.ts | 80 ++++++++++++++++++++++++++-------------- 2 files changed, 55 insertions(+), 28 deletions(-) diff --git a/src/commands/uncommit.ts b/src/commands/uncommit.ts index d5aa27c..4b5ea38 100644 --- a/src/commands/uncommit.ts +++ b/src/commands/uncommit.ts @@ -26,11 +26,12 @@ export async function _uncommit(parsedSettings: ParsedSettings): Promise { throw new Error("There's no committed migration to uncommit"); } - // Check current.sql is blank + // Check current.sql is blank; automatic fixtures are derived again afterwards const currentLocation = await getCurrentMigrationLocation(parsedSettings); const { compiled: currentBody } = await readCurrentMigration( parsedSettings, currentLocation, + { autoIncludeFixtures: false }, ); const minifiedCurrentBody = pgMinify(currentBody); if (minifiedCurrentBody !== "") { diff --git a/src/current.ts b/src/current.ts index 8d0228d..47e0d6a 100644 --- a/src/current.ts +++ b/src/current.ts @@ -1,5 +1,6 @@ import * as assert from "assert"; import { promises as fsp } from "fs"; +import { posix } from "path"; import { getCommittedFixturesState, getLocalFixtures } from "./fixtures"; import { isNoTransactionDefined } from "./header"; @@ -56,13 +57,17 @@ export async function getCurrentMigrationLocation( export async function readCurrentMigration( parsedSettings: ParsedSettings, location: CurrentMigrationLocation, + { autoIncludeFixtures = parsedSettings.autoIncludeFixtures } = {}, ): Promise<{ raw: string; compiled: string }> { const { body: raw, paths } = await readCurrentMigrationRaw(location); - let compiled = raw; - if (parsedSettings.autoIncludeFixtures) { - const regex = /^--![ \t]*include[ \t]+(.*\.sql)[ \t]*$/gm; - const alreadyIncluded = [...compiled.matchAll(regex)].map( - (match) => match[1], + let compiled = await compileIncludes(parsedSettings, raw, new Set(paths)); + if (autoIncludeFixtures) { + // Everything the migration already includes, directly or through another + // fixture, whatever the spelling of its `--!include`. + const alreadyIncluded = new Set( + [...compiled.matchAll(/^--! Included (.+)$/gm)].map((match) => + posix.normalize(match[1]), + ), ); // Find any fixture that has changed since the last commit that hasn't been @@ -70,39 +75,57 @@ export async function readCurrentMigration( const committedState = await getCommittedFixturesState(parsedSettings); const currentFixtures = await getLocalFixtures(parsedSettings); const changedFixtures: string[] = []; - for (const [relativePath, { hash, hasNoTransaction }] of Object.entries( - currentFixtures, - )) { - if (committedState[relativePath] !== hash) { - // TODO: maybe be a bit more flexible about matches here: `./`, etc - if (!alreadyIncluded.includes(relativePath)) { - if (hasNoTransaction) { - parsedSettings.logger.warn( - `Not including modified fixture ${relativePath} because it includes --! no-transaction`, - { relativePath, noTransaction: true }, - ); - } else { - changedFixtures.push(relativePath); - } + for (const [ + relativePath, + { hash, hasNoTransaction, isEmpty }, + ] of Object.entries(currentFixtures)) { + if ( + committedState[relativePath] !== hash && + !alreadyIncluded.has(relativePath) && + // A new fixture with nothing to run is not a change. + !(isEmpty && committedState[relativePath] == null) + ) { + if (hasNoTransaction) { + parsedSettings.logger.warn( + `Not including modified fixture ${relativePath} because it includes --! no-transaction`, + { relativePath, noTransaction: true }, + ); + } else { + changedFixtures.push(relativePath); } } } - // Prefix the migration with the changed fixture includes if (changedFixtures.length > 0) { changedFixtures.sort((a, z) => (a < z ? -1 : a > z ? 1 : 0)); - compiled = + const autoIncludes = await compileIncludes( + parsedSettings, changedFixtures .map((relativePath) => `--!include ${relativePath}`) - .join("\n") + - "\n\n--!endAuto\n\n" + - compiled; + .join("\n"), + new Set(paths), + ); + // Headers and `--! no-transaction` must stay first; the fixtures follow + // them and the blank line that ends the headers. + const lines = compiled.split("\n"); + let start = 0; + while (start < lines.length && LEADING_DIRECTIVE.test(lines[start])) { + start++; + } + if (start > 0 && lines[start] === "") start++; + compiled = [ + ...lines.slice(0, start), + `${autoIncludes}\n\n--!endAuto\n\n${lines.slice(start).join("\n")}`, + ].join("\n"); } } - compiled = await compileIncludes(parsedSettings, compiled, new Set(paths)); return { raw, compiled }; } +/** A header line, or the `--! no-transaction` directive. */ +const LEADING_DIRECTIVE = + /^--!(?: [A-Z][a-zA-Z0-9_]*(?::.*)?|\s*no-transaction\b.*)$/; + async function readCurrentMigrationRaw( location: CurrentMigrationLocation, ): Promise<{ body: string; paths: readonly string[] }> { @@ -196,8 +219,11 @@ export async function writeCurrentMigration( location: CurrentMigrationLocation, rawBody: string, ): Promise { - // Strip the auto-included fixtures - const body = rawBody.replace(/^[\s\S]*\n--!endAuto\n\n/, ""); + // Strip the auto-included fixtures, keeping any headers before them + const body = rawBody.replace( + /^(?:--!include [^\n]*\n)+\n--!endAuto\n\n/m, + "", + ); if (body.trim() + "\n" !== body) { throw new Error( "graphile-migrate error - 'body' should be sanitized before being passed to 'writeCurrentMigration'", From d41743147f1ed5177fb550c405ffbda35f730639 Mon Sep 17 00:00:00 2001 From: Jukka Raimovaara Date: Fri, 16 Oct 2026 10:49:34 +0300 Subject: [PATCH 4/9] Test and document automatic fixtures Co-Authored-By: Claude Opus 5.5 --- README.md | 48 +++++ __tests__/autoIncludeFixtures.test.ts | 265 ++++++++++++++++++++++++++ 2 files changed, 313 insertions(+) create mode 100644 __tests__/autoIncludeFixtures.test.ts diff --git a/README.md b/README.md index dd4685b..6e80a7b 100644 --- a/README.md +++ b/README.md @@ -806,6 +806,54 @@ drop policy if exists access_by_numbers on mytable; create policy access_by_numbers on mytable for update using (myfunction(4, 2) < 42); ``` +#### Automatically including changed fixtures (experimental) + +Set `autoIncludeFixtures: true` in `.gmrc` to include changed fixtures from +`migrations/fixtures/auto/`, including its subdirectories, without writing +`--!include` yourself. Whenever the current migration is read (by `watch`, +`status`, `commit` and so on), every SQL file in that folder that differs from +the version last included in a committed migration is included at the start of +the migration, after any headers, in path order. Your current migration files +are not changed. Comment-only changes are ignored, as are new files that are +empty or only comments. + +Only `fixtures/auto/` is included automatically; other fixtures are included +manually as before. Use the folder for things that can be recreated quickly at +any time: functions, views, policies, grants and comments. Keep stateful objects +such as indexes, constraints and materialized views in other fixtures. Fixtures +in `fixtures/auto/` must be self-contained: `--!include` inside them is an +error. Fixtures with `--! no-transaction` are not included automatically; a +warning is logged instead. Deleting a fixture does not drop anything; write any +cleanup SQL yourself. + +Automatic fixtures run before the rest of the migration, in path order, so name +them to order their dependencies (for example `auto/010-functions/…` before +`auto/020-policies/…`). To run a fixture elsewhere, for example a view that +selects a column this migration adds, include it yourself with +`--!include auto/…`; a fixture that the migration already includes, directly or +through another fixture, is not added again. Some changes cannot be made by +running the new definition: adding a parameter to a function needs a +`drop function` first, and possibly dropping and recreating the policies that +use it. Make those changes in the migration yourself. + +A fixture that no committed migration has included counts as new. That includes +a fixture moved into `fixtures/auto/`, because its path changes, so moving +existing fixtures into the folder runs each of them once in the next migration. +Check that they are safe to run again: no one-off data, and no +`drop ... cascade` of objects other fixtures define. Recreating views and +policies locks them and their tables until the migration commits, so deploy such +a migration at a quiet time or with a `lock_timeout`. + +`watch` does not undo the previous run before running the migration again, so +objects left by earlier runs can hide a wrong order. To check the order on a +fresh database, run `graphile-migrate reset --shadow --erase` and then +`graphile-migrate watch --once --shadow`; `commit` also replays every migration +on the shadow database. + +Changes are detected with `pg-minify`, which does not understand dollar quoting: +text after `--` inside a `$tag$…$tag$` string is treated as a comment, so +editing only such text is not detected. + ### Committed migration(s) The files for migrations that you've committed with `graphile-migrate commit` diff --git a/__tests__/autoIncludeFixtures.test.ts b/__tests__/autoIncludeFixtures.test.ts new file mode 100644 index 0000000..bd41e41 --- /dev/null +++ b/__tests__/autoIncludeFixtures.test.ts @@ -0,0 +1,265 @@ +import "./helpers"; // Has side-effects; must come first + +import { promises as fsp } from "fs"; +import mockFs from "mock-fs"; + +import { commit, current, status, uncommit } from "../src"; +import { + getCurrentMigrationLocation, + readCurrentMigration, +} from "../src/current"; +import { calculateHash } from "../src/hash"; +import { isNoTransactionDefined } from "../src/header"; +import { + compileIncludes, + parseMigrationText, + serializeMigration, +} from "../src/migration"; +import { withClient } from "../src/pg"; +import { parseSettings } from "../src/settings"; +import { resetDb, settings as baseSettings } from "./helpers"; + +const settings = { ...baseSettings, autoIncludeFixtures: true }; + +beforeEach(() => { + mockFs({ + "migrations/current.sql": "-- current\n", + "migrations/fixtures/auto/example.sql": "select '--literal'; -- comment\n", + }); +}); +afterEach(() => mockFs.restore()); + +/** Write a committed migration that includes `paths`, as an earlier commit would have. */ +async function recordCommit( + paths = ["auto/example.sql"], + number = 1, + previous: string | null = null, +) { + const parsed = await parseSettings(settings); + const body = await compileIncludes( + parsed, + paths.map((path) => `--!include ${path}`).join("\n"), + new Set(), + ); + const hash = calculateHash(body, previous); + await fsp.mkdir("migrations/committed", { recursive: true }); + await fsp.writeFile( + `migrations/committed/${String(number).padStart(6, "0")}.sql`, + serializeMigration(body, { Previous: previous || "-", Hash: hash }), + ); + return hash; +} + +/** Read the current migration, listing the fixtures it includes automatically. */ +async function read(autoIncludeFixtures = true) { + const parsed = await parseSettings({ ...settings, autoIncludeFixtures }); + const location = await getCurrentMigrationLocation(parsed); + const { raw, compiled } = await readCurrentMigration(parsed, location); + const end = compiled.indexOf("--!endAuto"); + const auto = + end < 0 + ? [] + : [...compiled.slice(0, end).matchAll(/^--! Included (.+)$/gm)].map( + (match) => match[1], + ); + return { raw, compiled, auto }; +} + +async function useCurrentDirectory(content = "-- current\n") { + await fsp.rm("migrations/current.sql"); + await fsp.mkdir("migrations/current"); + await fsp.writeFile("migrations/current/001.sql", content); +} + +it("does nothing unless autoIncludeFixtures is enabled", async () => { + expect((await read(false)).auto).toEqual([]); +}); + +it("includes new fixtures from fixtures/auto only, in path order", async () => { + await fsp.mkdir("migrations/fixtures/auto/policies", { recursive: true }); + await fsp.mkdir("migrations/fixtures/auto/.drafts", { recursive: true }); + await fsp.writeFile("migrations/fixtures/auto/policies/p.sql", "select 2;"); + await fsp.writeFile("migrations/fixtures/auto/.drafts/d.sql", "select 3;"); + await fsp.writeFile("migrations/fixtures/auto/.hidden.sql", "select 4;"); + await fsp.writeFile("migrations/fixtures/manual.sql", "select 5;"); + expect((await read()).auto).toEqual([ + "auto/example.sql", + "auto/policies/p.sql", + ]); +}); + +it("ignores comment-only changes and follows the latest committed version", async () => { + const first = await recordCommit(); + expect((await read()).auto).toEqual([]); + await fsp.writeFile( + "migrations/fixtures/auto/example.sql", + "select '--literal'; -- another comment\n", + ); + expect((await read()).auto).toEqual([]); + await fsp.writeFile("migrations/fixtures/auto/example.sql", "select 2;"); + expect((await read()).auto).toEqual(["auto/example.sql"]); + await recordCommit(["auto/example.sql"], 2, first); + expect((await read()).auto).toEqual([]); + // Removing that migration, as uncommit does, makes the change pending again. + await fsp.unlink("migrations/committed/000002.sql"); + expect((await read()).auto).toEqual(["auto/example.sql"]); +}); + +it("ignores new fixtures that are empty or only comments", async () => { + await fsp.writeFile("migrations/fixtures/auto/empty.sql", "-- todo\n"); + expect((await read()).auto).toEqual(["auto/example.sql"]); +}); + +it("treats a fixture moved into fixtures/auto as new", async () => { + await fsp.rename( + "migrations/fixtures/auto/example.sql", + "migrations/fixtures/example.sql", + ); + await recordCommit(["example.sql"]); + expect((await read()).auto).toEqual([]); + await fsp.rename( + "migrations/fixtures/example.sql", + "migrations/fixtures/auto/example.sql", + ); + expect((await read()).auto).toEqual(["auto/example.sql"]); +}); + +it.each([ + ["directly", "--!include auto/example.sql\n", {}], + ["with another spelling", "--! include ./auto/example.sql\n", {}], + [ + "through another fixture", + "--!include wrapper.sql\n", + { "migrations/fixtures/wrapper.sql": "--!include auto/example.sql\n" }, + ], +])( + "does not add a fixture the migration already includes %s", + async (_, content, files: Record) => { + for (const [path, fileContent] of Object.entries(files)) { + await fsp.writeFile(path, fileContent); + } + await fsp.writeFile("migrations/current.sql", content); + const { auto, compiled } = await read(); + expect(auto).toEqual([]); + expect(compiled.match(/select '--literal'/g)).toHaveLength(1); + }, +); + +it("rejects --!include inside fixtures/auto", async () => { + await fsp.writeFile("migrations/fixtures/shared.sql", "select 2;"); + await fsp.writeFile( + "migrations/fixtures/auto/wrapper.sql", + "--!include shared.sql\n", + ); + await expect(read()).rejects.toThrow("must be self-contained"); +}); + +it.each(["current.sql", "current/"])( + "keeps headers before the automatic fixtures (%s)", + async (mode) => { + const content = "--! Message: add example\n\nselect 1;\n"; + if (mode === "current/") { + await useCurrentDirectory(content); + } else { + await fsp.writeFile("migrations/current.sql", content); + } + const { compiled, auto } = await read(); + expect(auto).toEqual(["auto/example.sql"]); + const { headers } = parseMigrationText("current.sql", compiled, false); + expect(headers).toEqual({ Message: "add example" }); + }, +); + +it("keeps --! no-transaction on the first line", async () => { + await fsp.writeFile( + "migrations/current.sql", + "--! no-transaction\nselect 1;\n", + ); + const { compiled, auto } = await read(); + expect(auto).toEqual(["auto/example.sql"]); + expect(isNoTransactionDefined(compiled)).toBe(true); +}); + +describe("database lifecycle", () => { + beforeEach(resetDb); + + it.each(["current.sql", "current/"])( + "runs, commits, uncommits and recommits fixtures (%s)", + async (mode) => { + if (mode === "current/") await useCurrentDirectory(); + await fsp.writeFile( + "migrations/fixtures/auto/example.sql", + "create table if not exists fixture_test (id integer);\ntruncate fixture_test;\ninsert into fixture_test values (42);", + ); + expect(await status(settings, { skipDatabase: true })).toMatchObject({ + hasCurrentMigration: true, + }); + await current(settings); + const parsed = await parseSettings(settings); + const rows = await withClient( + parsed.connectionString, + parsed, + async (client) => + (await client.query("select * from fixture_test")).rows, + ); + expect(rows).toEqual([{ id: 42 }]); + expect((await read()).auto).toEqual(["auto/example.sql"]); + + await commit(settings); + const committed = await fsp.readFile( + "migrations/committed/000001.sql", + "utf8", + ); + expect(committed).toContain("--! Included auto/example.sql"); + expect((await read()).auto).toEqual([]); + expect(await status(settings, { skipDatabase: true })).toMatchObject({ + hasCurrentMigration: false, + }); + + await uncommit(settings); + const { raw, auto } = await read(); + // The automatic includes are derived again, not restored as manual ones. + expect(raw).not.toContain("auto/example.sql"); + expect(raw).not.toContain("--!endAuto"); + expect(auto).toEqual(["auto/example.sql"]); + + await commit(settings); + expect( + await fsp.readFile("migrations/committed/000001.sql", "utf8"), + ).toBe(committed); + }, + ); + + it("keeps the commit message through commit and uncommit", async () => { + const content = "--! Message: add example\n\nselect 1;\n"; + await fsp.writeFile("migrations/current.sql", content); + await commit(settings); + expect(await fsp.readdir("migrations/committed")).toEqual([ + "000001-add-example.sql", + ]); + await uncommit(settings); + expect(await fsp.readFile("migrations/current.sql", "utf8")).toBe(content); + }); + + it("can uncommit while fixtures have pending changes", async () => { + await commit(settings); + await fsp.writeFile("migrations/fixtures/auto/example.sql", "select 2;"); + await uncommit(settings); + const { raw, auto, compiled } = await read(); + expect(raw).not.toContain("auto/example.sql"); + expect(auto).toEqual(["auto/example.sql"]); + expect(compiled.match(/select 2;/g)).toHaveLength(1); + }); + + it("restores what the user wrote after a failed commit", async () => { + await fsp.writeFile( + "migrations/fixtures/auto/example.sql", + "select * from missing_fixture_table;", + ); + await expect(commit(settings)).rejects.toThrow("does not exist"); + expect(await fsp.readFile("migrations/current.sql", "utf8")).toBe( + "-- current\n", + ); + expect(await fsp.readdir("migrations/committed")).toEqual([]); + }); +}); From 469a13b369bc311069a4f7671993a8e8b63ffd41 Mon Sep 17 00:00:00 2001 From: Jukka Raimovaara Date: Fri, 16 Oct 2026 10:57:06 +0300 Subject: [PATCH 5/9] Turn on automatic fixtures by creating fixtures/auto Remove the autoIncludeFixtures setting: the folder's presence is the opt-in. Without it, reading the current migration skips the committed fixture history entirely. Co-Authored-By: Claude Opus 5.5 --- README.md | 16 +++++------ __tests__/__snapshots__/settings.test.ts.snap | 5 ---- ...eFixtures.test.ts => autoFixtures.test.ts} | 26 +++++++++++++----- src/commands/uncommit.ts | 2 +- src/current.ts | 10 ++++--- src/fixtures.ts | 13 +++++++-- src/settings.ts | 27 ------------------- 7 files changed, 46 insertions(+), 53 deletions(-) rename __tests__/{autoIncludeFixtures.test.ts => autoFixtures.test.ts} (93%) diff --git a/README.md b/README.md index 6e80a7b..3adfbed 100644 --- a/README.md +++ b/README.md @@ -808,14 +808,14 @@ create policy access_by_numbers on mytable for update using (myfunction(4, 2) < #### Automatically including changed fixtures (experimental) -Set `autoIncludeFixtures: true` in `.gmrc` to include changed fixtures from -`migrations/fixtures/auto/`, including its subdirectories, without writing -`--!include` yourself. Whenever the current migration is read (by `watch`, -`status`, `commit` and so on), every SQL file in that folder that differs from -the version last included in a committed migration is included at the start of -the migration, after any headers, in path order. Your current migration files -are not changed. Comment-only changes are ignored, as are new files that are -empty or only comments. +Fixtures in `migrations/fixtures/auto/`, including its subdirectories, are +included automatically when they change, without writing `--!include` yourself. +Creating that folder turns this on; there is no setting. Whenever the current +migration is read (by `watch`, `status`, `commit` and so on), every SQL file in +that folder that differs from the version last included in a committed migration +is included at the start of the migration, after any headers, in path order. +Your current migration files are not changed. Comment-only changes are ignored, +as are new files that are empty or only comments. Only `fixtures/auto/` is included automatically; other fixtures are included manually as before. Use the folder for things that can be recreated quickly at diff --git a/__tests__/__snapshots__/settings.test.ts.snap b/__tests__/__snapshots__/settings.test.ts.snap index d438b85..68d7da5 100644 --- a/__tests__/__snapshots__/settings.test.ts.snap +++ b/__tests__/__snapshots__/settings.test.ts.snap @@ -22,7 +22,6 @@ exports[`actions is backwards-compatible with untagged command specs 1`] = ` ], "afterCurrent": [], "afterReset": [], - "autoIncludeFixtures": false, "beforeAllMigrations": [], "beforeCurrent": [], "beforeReset": [], @@ -64,7 +63,6 @@ exports[`actions parses SQL actions 1`] = ` "file": "foo.sql", }, ], - "autoIncludeFixtures": false, "beforeAllMigrations": [], "beforeCurrent": [], "beforeReset": [], @@ -101,7 +99,6 @@ exports[`actions parses command actions 1`] = ` ], "afterCurrent": [], "afterReset": [], - "autoIncludeFixtures": false, "beforeAllMigrations": [], "beforeCurrent": [], "beforeReset": [], @@ -146,7 +143,6 @@ exports[`actions parses mixed actions 1`] = ` ], "afterCurrent": [], "afterReset": [], - "autoIncludeFixtures": false, "beforeAllMigrations": [], "beforeCurrent": [], "beforeReset": [], @@ -188,7 +184,6 @@ exports[`actions parses string values into SQL actions 1`] = ` "file": "foo.sql", }, ], - "autoIncludeFixtures": false, "beforeAllMigrations": [], "beforeCurrent": [], "beforeReset": [], diff --git a/__tests__/autoIncludeFixtures.test.ts b/__tests__/autoFixtures.test.ts similarity index 93% rename from __tests__/autoIncludeFixtures.test.ts rename to __tests__/autoFixtures.test.ts index bd41e41..daf9acc 100644 --- a/__tests__/autoIncludeFixtures.test.ts +++ b/__tests__/autoFixtures.test.ts @@ -10,6 +10,7 @@ import { } from "../src/current"; import { calculateHash } from "../src/hash"; import { isNoTransactionDefined } from "../src/header"; +import * as migration from "../src/migration"; import { compileIncludes, parseMigrationText, @@ -17,9 +18,7 @@ import { } from "../src/migration"; import { withClient } from "../src/pg"; import { parseSettings } from "../src/settings"; -import { resetDb, settings as baseSettings } from "./helpers"; - -const settings = { ...baseSettings, autoIncludeFixtures: true }; +import { resetDb, settings } from "./helpers"; beforeEach(() => { mockFs({ @@ -51,8 +50,8 @@ async function recordCommit( } /** Read the current migration, listing the fixtures it includes automatically. */ -async function read(autoIncludeFixtures = true) { - const parsed = await parseSettings({ ...settings, autoIncludeFixtures }); +async function read() { + const parsed = await parseSettings(settings); const location = await getCurrentMigrationLocation(parsed); const { raw, compiled } = await readCurrentMigration(parsed, location); const end = compiled.indexOf("--!endAuto"); @@ -71,8 +70,21 @@ async function useCurrentDirectory(content = "-- current\n") { await fsp.writeFile("migrations/current/001.sql", content); } -it("does nothing unless autoIncludeFixtures is enabled", async () => { - expect((await read(false)).auto).toEqual([]); +it("does nothing, without reading history, if there is no fixtures/auto", async () => { + await fsp.rename( + "migrations/fixtures/auto/example.sql", + "migrations/fixtures/example.sql", + ); + await fsp.rmdir("migrations/fixtures/auto"); + const history = jest.spyOn(migration, "getAllMigrations"); + try { + const { auto, compiled } = await read(); + expect(auto).toEqual([]); + expect(compiled).toBe("-- current\n"); + expect(history).not.toHaveBeenCalled(); + } finally { + history.mockRestore(); + } }); it("includes new fixtures from fixtures/auto only, in path order", async () => { diff --git a/src/commands/uncommit.ts b/src/commands/uncommit.ts index 4b5ea38..d6a4153 100644 --- a/src/commands/uncommit.ts +++ b/src/commands/uncommit.ts @@ -31,7 +31,7 @@ export async function _uncommit(parsedSettings: ParsedSettings): Promise { const { compiled: currentBody } = await readCurrentMigration( parsedSettings, currentLocation, - { autoIncludeFixtures: false }, + { autoFixtures: false }, ); const minifiedCurrentBody = pgMinify(currentBody); if (minifiedCurrentBody !== "") { diff --git a/src/current.ts b/src/current.ts index 47e0d6a..8213440 100644 --- a/src/current.ts +++ b/src/current.ts @@ -57,11 +57,16 @@ export async function getCurrentMigrationLocation( export async function readCurrentMigration( parsedSettings: ParsedSettings, location: CurrentMigrationLocation, - { autoIncludeFixtures = parsedSettings.autoIncludeFixtures } = {}, + { autoFixtures = true }: { autoFixtures?: boolean } = {}, ): Promise<{ raw: string; compiled: string }> { const { body: raw, paths } = await readCurrentMigrationRaw(location); let compiled = await compileIncludes(parsedSettings, raw, new Set(paths)); - if (autoIncludeFixtures) { + // Creating `fixtures/auto/` turns automatic fixtures on; without it, skip + // reading the committed history. + const currentFixtures = autoFixtures + ? await getLocalFixtures(parsedSettings) + : {}; + if (Object.keys(currentFixtures).length > 0) { // Everything the migration already includes, directly or through another // fixture, whatever the spelling of its `--!include`. const alreadyIncluded = new Set( @@ -73,7 +78,6 @@ export async function readCurrentMigration( // Find any fixture that has changed since the last commit that hasn't been // specifically `--!include`'d const committedState = await getCommittedFixturesState(parsedSettings); - const currentFixtures = await getLocalFixtures(parsedSettings); const changedFixtures: string[] = []; for (const [ relativePath, diff --git a/src/fixtures.ts b/src/fixtures.ts index 26ae476..32c319a 100644 --- a/src/fixtures.ts +++ b/src/fixtures.ts @@ -33,8 +33,17 @@ interface FileFixture { } /** - * Only fixtures in this folder (inside the fixtures folder) are included - * automatically. They must be self-contained: no `--!include`. + * Fixtures in this folder (inside the fixtures folder) that have changed since + * the last commit are included automatically; creating the folder turns this + * on. They must be self-contained: no `--!include`. + * + * NOTE: only use it for stateless database entities: functions, RLS policies, + * grants, views, etc - things that can be dropped and recreated almost + * instantly. Keep stateful entities such as indexes, constraints and + * materialized views in other fixtures, as dropping and recreating these takes + * significant time. + * + * @experimental */ export const AUTO_FIXTURES_FOLDER = "auto"; diff --git a/src/settings.ts b/src/settings.ts index 0f4019b..79e665e 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -84,21 +84,6 @@ export interface Settings { afterCurrent?: Actions; blankMigrationContent?: string; logger?: Logger; - /** - * On `commit` (or when using `watch` mode), find fixtures in - * `migrations/fixtures/auto/` that have changed since the last commit and - * include them in this migration. Fixtures in that folder must not use - * `--!include`; fixtures elsewhere are only included manually. - * - * NOTE: only use the `auto` folder for stateless database entities: - * functions, RLS policies, grants, views, etc - things that can be dropped - * and recreated almost instantly. Keep stateful entities such as indexes, - * constraints and materialized views in other fixtures, as dropping and - * recreating these takes significant time. - * - * @experimental - */ - autoIncludeFixtures?: boolean; } // NOTE: only override values that differ (e.g. changing non-nullability) @@ -117,8 +102,6 @@ export interface ParsedSettings extends Settings { afterCurrent: ActionSpec[]; blankMigrationContent: string; logger: Logger; - /** @experimental */ - autoIncludeFixtures: boolean; } export async function parseSettings( @@ -306,15 +289,6 @@ export async function parseSettings( }, ); - const autoIncludeFixtures = await check( - "autoIncludeFixtures", - (rawAIF): boolean => { - if (typeof rawAIF === "boolean") return rawAIF; - if (rawAIF == null) return false; - throw new Error(`Invalid value for autoIncludeFixtures`); - }, - ); - const validateAction = makeValidateActionCallback(logger); const rootValidateAction = makeValidateActionCallback(logger, true); @@ -416,7 +390,6 @@ export async function parseSettings( placeholders, blankMigrationContent, logger, - autoIncludeFixtures, }; } From 8f675cfced61b95ee0ab51d3097e0be3cf274cc8 Mon Sep 17 00:00:00 2001 From: Jukka Raimovaara Date: Fri, 16 Oct 2026 11:04:45 +0300 Subject: [PATCH 6/9] Reject --! no-transaction in automatic fixtures The automatic section always runs inside the migration's transaction, so a fixture in fixtures/auto that declares --! no-transaction is now an error instead of being skipped with an easy-to-miss warning. Keep such fixtures in another folder and include them manually. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 ++++---- __tests__/autoFixtures.test.ts | 11 +++++++++++ src/current.ts | 16 ++++------------ src/fixtures.ts | 13 ++++++++----- 4 files changed, 27 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 3adfbed..1d0918f 100644 --- a/README.md +++ b/README.md @@ -821,10 +821,10 @@ Only `fixtures/auto/` is included automatically; other fixtures are included manually as before. Use the folder for things that can be recreated quickly at any time: functions, views, policies, grants and comments. Keep stateful objects such as indexes, constraints and materialized views in other fixtures. Fixtures -in `fixtures/auto/` must be self-contained: `--!include` inside them is an -error. Fixtures with `--! no-transaction` are not included automatically; a -warning is logged instead. Deleting a fixture does not drop anything; write any -cleanup SQL yourself. +in `fixtures/auto/` must be self-contained and run inside the migration's +transaction: `--!include` or `--! no-transaction` inside them is an error, so +keep such fixtures in another folder and include them manually. Deleting a +fixture does not drop anything; write any cleanup SQL yourself. Automatic fixtures run before the rest of the migration, in path order, so name them to order their dependencies (for example `auto/010-functions/…` before diff --git a/__tests__/autoFixtures.test.ts b/__tests__/autoFixtures.test.ts index daf9acc..be00938 100644 --- a/__tests__/autoFixtures.test.ts +++ b/__tests__/autoFixtures.test.ts @@ -166,6 +166,17 @@ it("rejects --!include inside fixtures/auto", async () => { await expect(read()).rejects.toThrow("must be self-contained"); }); +it.each([ + ["on the first line", "--! no-transaction\nselect 1;\n"], + ["on a later line", "select 1;\n--! no-transaction\n"], +])( + "rejects --! no-transaction inside fixtures/auto (%s)", + async (_, content) => { + await fsp.writeFile("migrations/fixtures/auto/concurrently.sql", content); + await expect(read()).rejects.toThrow("include it manually"); + }, +); + it.each(["current.sql", "current/"])( "keeps headers before the automatic fixtures (%s)", async (mode) => { diff --git a/src/current.ts b/src/current.ts index 8213440..dfd8535 100644 --- a/src/current.ts +++ b/src/current.ts @@ -79,24 +79,16 @@ export async function readCurrentMigration( // specifically `--!include`'d const committedState = await getCommittedFixturesState(parsedSettings); const changedFixtures: string[] = []; - for (const [ - relativePath, - { hash, hasNoTransaction, isEmpty }, - ] of Object.entries(currentFixtures)) { + for (const [relativePath, { hash, isEmpty }] of Object.entries( + currentFixtures, + )) { if ( committedState[relativePath] !== hash && !alreadyIncluded.has(relativePath) && // A new fixture with nothing to run is not a change. !(isEmpty && committedState[relativePath] == null) ) { - if (hasNoTransaction) { - parsedSettings.logger.warn( - `Not including modified fixture ${relativePath} because it includes --! no-transaction`, - { relativePath, noTransaction: true }, - ); - } else { - changedFixtures.push(relativePath); - } + changedFixtures.push(relativePath); } } diff --git a/src/fixtures.ts b/src/fixtures.ts index 32c319a..bb0ca7d 100644 --- a/src/fixtures.ts +++ b/src/fixtures.ts @@ -3,7 +3,6 @@ import { promises as fsp } from "fs"; import { posix, sep } from "path"; import pgMinify from "pg-minify"; -import { isNoTransactionDefined } from "./header"; import { realpathOrNull, walk } from "./lib"; import { getAllMigrations } from "./migration"; import type { ParsedSettings } from "./settings"; @@ -24,8 +23,6 @@ interface FileFixture { */ hash: string; - hasNoTransaction: boolean; - /** * True if the fixture is empty or only comments */ @@ -35,7 +32,8 @@ interface FileFixture { /** * Fixtures in this folder (inside the fixtures folder) that have changed since * the last commit are included automatically; creating the folder turns this - * on. They must be self-contained: no `--!include`. + * on. They must be self-contained (no `--!include`) and run inside the + * migration's transaction (no `--! no-transaction`). * * NOTE: only use it for stateless database entities: functions, RLS policies, * grants, views, etc - things that can be dropped and recreated almost @@ -86,12 +84,17 @@ export async function getLocalFixtures( `Automatic fixture '${relativePath}' uses '--!include'; fixtures in '${AUTO_FIXTURES_FOLDER}/' must be self-contained.`, ); } + // The automatic section always runs inside the migration's transaction. + if (/^--!\s*no-transaction\b/m.test(contents)) { + throw new Error( + `Automatic fixture '${relativePath}' uses '--! no-transaction'; move it out of '${AUTO_FIXTURES_FOLDER}/' and include it manually.`, + ); + } const hash = hashFixture(contents); allFixtures[relativePath] = { fullPath, relativePath, hash, - hasNoTransaction: isNoTransactionDefined(contents), isEmpty: hash === EMPTY_FIXTURE_HASH, }; }); From eb6f49782ee70c2e0eadb5c04abb627239efc8d3 Mon Sep 17 00:00:00 2001 From: Jukka Raimovaara Date: Fri, 16 Oct 2026 11:26:44 +0300 Subject: [PATCH 7/9] Drop the uncommit-only readCurrentMigration option uncommit again checks the compiled current migration, as in #269, so it refuses while a fixture has pending changes; commit or revert the edit first. Nothing is lost either way, and readCurrentMigration keeps its original signature. Co-Authored-By: Claude Opus 5.5 --- __tests__/autoFixtures.test.ts | 16 ++++++++++------ src/commands/uncommit.ts | 3 +-- src/current.ts | 5 +---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/__tests__/autoFixtures.test.ts b/__tests__/autoFixtures.test.ts index be00938..8af88ca 100644 --- a/__tests__/autoFixtures.test.ts +++ b/__tests__/autoFixtures.test.ts @@ -264,14 +264,18 @@ describe("database lifecycle", () => { expect(await fsp.readFile("migrations/current.sql", "utf8")).toBe(content); }); - it("can uncommit while fixtures have pending changes", async () => { + it("refuses to uncommit while a fixture has pending changes", async () => { + const path = "migrations/fixtures/auto/example.sql"; await commit(settings); - await fsp.writeFile("migrations/fixtures/auto/example.sql", "select 2;"); + const original = await fsp.readFile(path, "utf8"); + await fsp.writeFile(path, "select 2;"); + await expect(uncommit(settings)).rejects.toThrow( + "current migration is not blank", + ); + // Once the edit is committed or reverted, uncommit works again. + await fsp.writeFile(path, original); await uncommit(settings); - const { raw, auto, compiled } = await read(); - expect(raw).not.toContain("auto/example.sql"); - expect(auto).toEqual(["auto/example.sql"]); - expect(compiled.match(/select 2;/g)).toHaveLength(1); + expect((await read()).auto).toEqual(["auto/example.sql"]); }); it("restores what the user wrote after a failed commit", async () => { diff --git a/src/commands/uncommit.ts b/src/commands/uncommit.ts index d6a4153..d5aa27c 100644 --- a/src/commands/uncommit.ts +++ b/src/commands/uncommit.ts @@ -26,12 +26,11 @@ export async function _uncommit(parsedSettings: ParsedSettings): Promise { throw new Error("There's no committed migration to uncommit"); } - // Check current.sql is blank; automatic fixtures are derived again afterwards + // Check current.sql is blank const currentLocation = await getCurrentMigrationLocation(parsedSettings); const { compiled: currentBody } = await readCurrentMigration( parsedSettings, currentLocation, - { autoFixtures: false }, ); const minifiedCurrentBody = pgMinify(currentBody); if (minifiedCurrentBody !== "") { diff --git a/src/current.ts b/src/current.ts index dfd8535..8b5b6a0 100644 --- a/src/current.ts +++ b/src/current.ts @@ -57,15 +57,12 @@ export async function getCurrentMigrationLocation( export async function readCurrentMigration( parsedSettings: ParsedSettings, location: CurrentMigrationLocation, - { autoFixtures = true }: { autoFixtures?: boolean } = {}, ): Promise<{ raw: string; compiled: string }> { const { body: raw, paths } = await readCurrentMigrationRaw(location); let compiled = await compileIncludes(parsedSettings, raw, new Set(paths)); // Creating `fixtures/auto/` turns automatic fixtures on; without it, skip // reading the committed history. - const currentFixtures = autoFixtures - ? await getLocalFixtures(parsedSettings) - : {}; + const currentFixtures = await getLocalFixtures(parsedSettings); if (Object.keys(currentFixtures).length > 0) { // Everything the migration already includes, directly or through another // fixture, whatever the spelling of its `--!include`. From ddf3d1cb6b5e18b5c1c1ca977e639f9cd7f6fc25 Mon Sep 17 00:00:00 2001 From: Jukka Raimovaara Date: Fri, 16 Oct 2026 15:54:48 +0300 Subject: [PATCH 8/9] Forbid including automatic fixtures with --!include Fixtures in fixtures/auto now only ever run in the automatic section: compileIncludes takes a forbidAutoIncludes flag, which readCurrentMigration sets, so neither the current migration nor a fixture it includes can --!include one. That removes the already-included detection. AUTO_FIXTURES_FOLDER moves to migration.ts to avoid a circular import. The compile and run commands are unchanged. Co-Authored-By: Claude Opus 5.5 --- README.md | 16 +++++++++------- __tests__/autoFixtures.test.ts | 6 ++---- src/current.ts | 21 ++++++++------------- src/fixtures.ts | 18 +----------------- src/migration.ts | 30 +++++++++++++++++++++++++++++- 5 files changed, 49 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 1d0918f..044d0a0 100644 --- a/README.md +++ b/README.md @@ -828,13 +828,15 @@ fixture does not drop anything; write any cleanup SQL yourself. Automatic fixtures run before the rest of the migration, in path order, so name them to order their dependencies (for example `auto/010-functions/…` before -`auto/020-policies/…`). To run a fixture elsewhere, for example a view that -selects a column this migration adds, include it yourself with -`--!include auto/…`; a fixture that the migration already includes, directly or -through another fixture, is not added again. Some changes cannot be made by -running the new definition: adding a parameter to a function needs a -`drop function` first, and possibly dropping and recreating the policies that -use it. Make those changes in the migration yourself. +`auto/020-policies/…`). They only ever run there: nothing can include a fixture +from `fixtures/auto/` with `--!include`, neither the current migration nor +another fixture. If a fixture change depends on the rest of the migration, such +as a view selecting a column that the migration adds, commit the migration first +and change the fixture in the next one. Some changes cannot be made by running +the new definition: adding a parameter to a function needs a `drop function` +first, and possibly dropping and recreating the policies that use it. Make such +a change by hand in a migration and commit it, then update the fixture to match; +the next migration runs the matching definition again, which changes nothing. A fixture that no committed migration has included counts as new. That includes a fixture moved into `fixtures/auto/`, because its path changes, so moving diff --git a/__tests__/autoFixtures.test.ts b/__tests__/autoFixtures.test.ts index 8af88ca..dd18d67 100644 --- a/__tests__/autoFixtures.test.ts +++ b/__tests__/autoFixtures.test.ts @@ -145,15 +145,13 @@ it.each([ { "migrations/fixtures/wrapper.sql": "--!include auto/example.sql\n" }, ], ])( - "does not add a fixture the migration already includes %s", + "rejects including an automatic fixture %s", async (_, content, files: Record) => { for (const [path, fileContent] of Object.entries(files)) { await fsp.writeFile(path, fileContent); } await fsp.writeFile("migrations/current.sql", content); - const { auto, compiled } = await read(); - expect(auto).toEqual([]); - expect(compiled.match(/select '--literal'/g)).toHaveLength(1); + await expect(read()).rejects.toThrow("only included automatically"); }, ); diff --git a/src/current.ts b/src/current.ts index 8b5b6a0..a1271d0 100644 --- a/src/current.ts +++ b/src/current.ts @@ -1,6 +1,5 @@ import * as assert from "assert"; import { promises as fsp } from "fs"; -import { posix } from "path"; import { getCommittedFixturesState, getLocalFixtures } from "./fixtures"; import { isNoTransactionDefined } from "./header"; @@ -59,21 +58,18 @@ export async function readCurrentMigration( location: CurrentMigrationLocation, ): Promise<{ raw: string; compiled: string }> { const { body: raw, paths } = await readCurrentMigrationRaw(location); - let compiled = await compileIncludes(parsedSettings, raw, new Set(paths)); + // Fixtures in `fixtures/auto/` are only ever included automatically. + let compiled = await compileIncludes( + parsedSettings, + raw, + new Set(paths), + true, + ); // Creating `fixtures/auto/` turns automatic fixtures on; without it, skip // reading the committed history. const currentFixtures = await getLocalFixtures(parsedSettings); if (Object.keys(currentFixtures).length > 0) { - // Everything the migration already includes, directly or through another - // fixture, whatever the spelling of its `--!include`. - const alreadyIncluded = new Set( - [...compiled.matchAll(/^--! Included (.+)$/gm)].map((match) => - posix.normalize(match[1]), - ), - ); - - // Find any fixture that has changed since the last commit that hasn't been - // specifically `--!include`'d + // Find any fixture that has changed since the last commit const committedState = await getCommittedFixturesState(parsedSettings); const changedFixtures: string[] = []; for (const [relativePath, { hash, isEmpty }] of Object.entries( @@ -81,7 +77,6 @@ export async function readCurrentMigration( )) { if ( committedState[relativePath] !== hash && - !alreadyIncluded.has(relativePath) && // A new fixture with nothing to run is not a change. !(isEmpty && committedState[relativePath] == null) ) { diff --git a/src/fixtures.ts b/src/fixtures.ts index bb0ca7d..850b22e 100644 --- a/src/fixtures.ts +++ b/src/fixtures.ts @@ -4,7 +4,7 @@ import { posix, sep } from "path"; import pgMinify from "pg-minify"; import { realpathOrNull, walk } from "./lib"; -import { getAllMigrations } from "./migration"; +import { AUTO_FIXTURES_FOLDER, getAllMigrations } from "./migration"; import type { ParsedSettings } from "./settings"; interface FileFixture { @@ -29,22 +29,6 @@ interface FileFixture { isEmpty: boolean; } -/** - * Fixtures in this folder (inside the fixtures folder) that have changed since - * the last commit are included automatically; creating the folder turns this - * on. They must be self-contained (no `--!include`) and run inside the - * migration's transaction (no `--! no-transaction`). - * - * NOTE: only use it for stateless database entities: functions, RLS policies, - * grants, views, etc - things that can be dropped and recreated almost - * instantly. Keep stateful entities such as indexes, constraints and - * materialized views in other fixtures, as dropping and recreating these takes - * significant time. - * - * @experimental - */ -export const AUTO_FIXTURES_FOLDER = "auto"; - interface FixturesCache { fixtures: { [relativePath: string]: Array<{ diff --git a/src/migration.ts b/src/migration.ts index 38f8a1c..4d20e89 100644 --- a/src/migration.ts +++ b/src/migration.ts @@ -1,5 +1,5 @@ import * as fsp from "fs/promises"; -import { relative } from "path"; +import { relative, sep } from "path"; import { calculateHash } from "./hash"; import { isNoTransactionDefined } from "./header"; @@ -129,10 +129,28 @@ export function compilePlaceholders( )(content); } +/** + * Fixtures in this folder (inside the fixtures folder) that have changed since + * the last commit are included automatically; creating the folder turns this + * on. They must be self-contained (no `--!include`) and run inside the + * migration's transaction (no `--! no-transaction`), and nothing else may + * `--!include` them. + * + * NOTE: only use it for stateless database entities: functions, RLS policies, + * grants, views, etc - things that can be dropped and recreated almost + * instantly. Keep stateful entities such as indexes, constraints and + * materialized views in other fixtures, as dropping and recreating these takes + * significant time. + * + * @experimental + */ +export const AUTO_FIXTURES_FOLDER = "auto"; + export async function compileIncludes( parsedSettings: ParsedSettings, content: string, processedFiles: ReadonlySet, + forbidAutoIncludes = false, ): Promise { if (/--!\s*(End)?Included?/.test(content)) { throw new Error( @@ -187,6 +205,15 @@ export async function compileIncludes( ); } + if ( + forbidAutoIncludes && + relativePath.split(sep)[0] === AUTO_FIXTURES_FOLDER + ) { + throw new Error( + `Forbidden: cannot include '${rawSqlPath}' because fixtures in '${AUTO_FIXTURES_FOLDER}/' are only included automatically`, + ); + } + // Looks good to me sqlPathByRawSqlPath[rawSqlPath] = sqlPath; } @@ -200,6 +227,7 @@ export async function compileIncludes( parsedSettings, fileContents, new Set([...processedFiles, sqlPath]), + forbidAutoIncludes, ); return processed; }), From 4fb10f6b350f905eeef006da73a988399587708f Mon Sep 17 00:00:00 2001 From: Jukka Raimovaara Date: Fri, 16 Oct 2026 16:17:04 +0300 Subject: [PATCH 9/9] Let the migration place automatic fixtures itself The current migration may again --!include a fixture from fixtures/auto, for example after creating the table a policy applies to; that fixture is then not added at the start. Only the top-level file may do so: compileIncludes now always sets forbidAutoIncludes for nested includes, so no fixture can include an automatic fixture. Co-Authored-By: Claude Opus 5.5 --- README.md | 17 +++++------ __tests__/autoFixtures.test.ts | 55 ++++++++++++++++++++++++++-------- src/current.ts | 22 +++++++++----- src/migration.ts | 10 ++++--- 4 files changed, 70 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 044d0a0..4341d35 100644 --- a/README.md +++ b/README.md @@ -828,15 +828,14 @@ fixture does not drop anything; write any cleanup SQL yourself. Automatic fixtures run before the rest of the migration, in path order, so name them to order their dependencies (for example `auto/010-functions/…` before -`auto/020-policies/…`). They only ever run there: nothing can include a fixture -from `fixtures/auto/` with `--!include`, neither the current migration nor -another fixture. If a fixture change depends on the rest of the migration, such -as a view selecting a column that the migration adds, commit the migration first -and change the fixture in the next one. Some changes cannot be made by running -the new definition: adding a parameter to a function needs a `drop function` -first, and possibly dropping and recreating the policies that use it. Make such -a change by hand in a migration and commit it, then update the fixture to match; -the next migration runs the matching definition again, which changes nothing. +`auto/020-policies/…`). To run one later, for example a policy on a table that +the migration creates, include it in the migration yourself with +`--!include auto/…`; it is then not added at the start. Only the migration +itself can include a fixture from `fixtures/auto/`; other fixtures cannot. Some +changes cannot be made by running the new definition: adding a parameter to a +function needs a `drop function` first, and possibly dropping and recreating the +policies that use it. Write those `drop`s in the migration and include the +affected fixtures after them. A fixture that no committed migration has included counts as new. That includes a fixture moved into `fixtures/auto/`, because its path changes, so moving diff --git a/__tests__/autoFixtures.test.ts b/__tests__/autoFixtures.test.ts index dd18d67..195c0fb 100644 --- a/__tests__/autoFixtures.test.ts +++ b/__tests__/autoFixtures.test.ts @@ -137,24 +137,27 @@ it("treats a fixture moved into fixtures/auto as new", async () => { }); it.each([ - ["directly", "--!include auto/example.sql\n", {}], - ["with another spelling", "--! include ./auto/example.sql\n", {}], - [ - "through another fixture", - "--!include wrapper.sql\n", - { "migrations/fixtures/wrapper.sql": "--!include auto/example.sql\n" }, - ], + ["directly", "--!include auto/example.sql\n"], + ["with another spelling", "--! include ./auto/example.sql\n"], ])( - "rejects including an automatic fixture %s", - async (_, content, files: Record) => { - for (const [path, fileContent] of Object.entries(files)) { - await fsp.writeFile(path, fileContent); - } + "does not add a fixture the migration includes itself %s", + async (_, content) => { await fsp.writeFile("migrations/current.sql", content); - await expect(read()).rejects.toThrow("only included automatically"); + const { auto, compiled } = await read(); + expect(auto).toEqual([]); + expect(compiled.match(/select '--literal'/g)).toHaveLength(1); }, ); +it("rejects including an automatic fixture from another fixture", async () => { + await fsp.writeFile( + "migrations/fixtures/wrapper.sql", + "--!include auto/example.sql\n", + ); + await fsp.writeFile("migrations/current.sql", "--!include wrapper.sql\n"); + await expect(read()).rejects.toThrow("only the migration itself"); +}); + it("rejects --!include inside fixtures/auto", async () => { await fsp.writeFile("migrations/fixtures/shared.sql", "select 2;"); await fsp.writeFile( @@ -251,6 +254,32 @@ describe("database lifecycle", () => { }, ); + it("runs a fixture the migration includes after creating its table", async () => { + await fsp.writeFile( + "migrations/fixtures/auto/policy.sql", + "comment on table placed_test is 'placed';\n", + ); + await fsp.writeFile( + "migrations/current.sql", + "create table if not exists placed_test (id integer);\n--!include auto/policy.sql\n", + ); + await current(settings); + const parsed = await parseSettings(settings); + const rows = await withClient( + parsed.connectionString, + parsed, + async (client) => + ( + await client.query( + "select obj_description('placed_test'::regclass) as description", + ) + ).rows, + ); + expect(rows).toEqual([{ description: "placed" }]); + await commit(settings); + expect((await read()).auto).toEqual([]); + }); + it("keeps the commit message through commit and uncommit", async () => { const content = "--! Message: add example\n\nselect 1;\n"; await fsp.writeFile("migrations/current.sql", content); diff --git a/src/current.ts b/src/current.ts index a1271d0..6b152ed 100644 --- a/src/current.ts +++ b/src/current.ts @@ -1,5 +1,6 @@ import * as assert from "assert"; import { promises as fsp } from "fs"; +import { posix } from "path"; import { getCommittedFixturesState, getLocalFixtures } from "./fixtures"; import { isNoTransactionDefined } from "./header"; @@ -58,18 +59,22 @@ export async function readCurrentMigration( location: CurrentMigrationLocation, ): Promise<{ raw: string; compiled: string }> { const { body: raw, paths } = await readCurrentMigrationRaw(location); - // Fixtures in `fixtures/auto/` are only ever included automatically. - let compiled = await compileIncludes( - parsedSettings, - raw, - new Set(paths), - true, - ); + let compiled = await compileIncludes(parsedSettings, raw, new Set(paths)); // Creating `fixtures/auto/` turns automatic fixtures on; without it, skip // reading the committed history. const currentFixtures = await getLocalFixtures(parsedSettings); if (Object.keys(currentFixtures).length > 0) { - // Find any fixture that has changed since the last commit + // Fixtures the migration includes itself, to run them later in the + // migration, whatever the spelling of its `--!include`. Other fixtures + // cannot include them. + const alreadyIncluded = new Set( + [...compiled.matchAll(/^--! Included (.+)$/gm)].map((match) => + posix.normalize(match[1]), + ), + ); + + // Find any fixture that has changed since the last commit that hasn't been + // specifically `--!include`'d const committedState = await getCommittedFixturesState(parsedSettings); const changedFixtures: string[] = []; for (const [relativePath, { hash, isEmpty }] of Object.entries( @@ -77,6 +82,7 @@ export async function readCurrentMigration( )) { if ( committedState[relativePath] !== hash && + !alreadyIncluded.has(relativePath) && // A new fixture with nothing to run is not a change. !(isEmpty && committedState[relativePath] == null) ) { diff --git a/src/migration.ts b/src/migration.ts index 4d20e89..3c33ff1 100644 --- a/src/migration.ts +++ b/src/migration.ts @@ -133,8 +133,9 @@ export function compilePlaceholders( * Fixtures in this folder (inside the fixtures folder) that have changed since * the last commit are included automatically; creating the folder turns this * on. They must be self-contained (no `--!include`) and run inside the - * migration's transaction (no `--! no-transaction`), and nothing else may - * `--!include` them. + * migration's transaction (no `--! no-transaction`). Only the migration itself + * may `--!include` one, to run it later in the migration; other fixtures may + * not. * * NOTE: only use it for stateless database entities: functions, RLS policies, * grants, views, etc - things that can be dropped and recreated almost @@ -210,7 +211,7 @@ export async function compileIncludes( relativePath.split(sep)[0] === AUTO_FIXTURES_FOLDER ) { throw new Error( - `Forbidden: cannot include '${rawSqlPath}' because fixtures in '${AUTO_FIXTURES_FOLDER}/' are only included automatically`, + `Forbidden: cannot include '${rawSqlPath}' from another fixture; only the migration itself can include fixtures from '${AUTO_FIXTURES_FOLDER}/'`, ); } @@ -227,7 +228,8 @@ export async function compileIncludes( parsedSettings, fileContents, new Set([...processedFiles, sqlPath]), - forbidAutoIncludes, + // Only the top-level file may include automatic fixtures. + true, ); return processed; }),