use a migration system

AGENTS.md

@@ -16,6 +16,24 @@
 - Persistence: Postgres with JDBC
 - Maven for dependency management
 
+## Database migrations
+
+All internal database schema and data changes must be made with SQL migrations in
+`src/main/resources/db/migrations`. Migration filenames use:
+
+```text
+yyyyMMddHHmmss_short_description.sql
+```
+
+The 14-digit UTC timestamp is the migration version and must be unique. Migrations
+are run in filename order at application startup, inside one transaction per file.
+Applied versions are recorded in `schema_migrations`, so never edit or rename a
+migration that may have run in another environment; add a new migration instead.
+
+Keep migrations compatible with Postgres. Update `src/main/resources/db/reset.sql`
+when adding a table so `DB_REINITIALIZE=true` can continue to rebuild disposable
+test databases by dropping the application tables and replaying all migrations.
+
 
 
 ## Tests

src/main/kotlin/dev/mduchene/bolts/persistence/Database.kt

@@ -1,12 +1,14 @@
 package dev.mduchene.bolts.persistence
 
-import dev.mduchene.bolts.entity.identifierFromName
 import java.sql.Connection
 import java.sql.DriverManager
 import java.sql.PreparedStatement
 import java.sql.ResultSet
 
-class Database(private val config: DatabaseConfig) {
+class Database(
+    private val config: DatabaseConfig,
+    private val migrationRunner: MigrationRunner = MigrationRunner(),
+) {
     fun getConnection(): Connection =
         DriverManager.getConnection(config.url, config.username, config.password)
 
@@ -38,42 +40,8 @@ class Database(private val config: DatabaseConfig) {
 
     fun initialize(reinitialize: Boolean = false) {
         getConnection().use { connection ->
-            connection.createStatement().use { statement ->
+            if (reinitialize) migrationRunner.reset(connection)
-                if (reinitialize) {
+            migrationRunner.migrate(connection)
-                    statement.execute(DROP_ENTITY_FIELDS_TABLE)
-                    statement.execute(DROP_ENTITY_DEFINITIONS_TABLE)
-                    statement.execute(DROP_SESSIONS_TABLE)
-                    statement.execute(DROP_USERS_TABLE)
-                }
-                statement.execute(CREATE_USERS_TABLE)
-                statement.execute(ADD_USER_ROLE)
-                statement.execute(CREATE_SESSIONS_TABLE)
-                statement.execute(CREATE_ENTITY_DEFINITIONS_TABLE)
-                statement.execute(CREATE_ENTITY_FIELDS_TABLE)
-                statement.execute(ADD_ENTITY_IDENTIFIER)
-                statement.execute(ADD_FIELD_IDENTIFIER)
-                statement.execute(ADD_FIELD_TARGET_ENTITY)
-                statement.execute(ADD_FIELD_RELATIONSHIP_TYPE)
-                statement.execute(ADD_FIELD_TARGET_ENTITY_CONSTRAINT)
-            }
-            backfillIdentifiers(connection)
-        }
-    }
-
-    private fun backfillIdentifiers(connection: Connection) {
-        listOf("entity_definitions", "entity_fields").forEach { table ->
-            connection.prepareStatement("SELECT id, name FROM $table WHERE identifier = ''").use { select ->
-                select.executeQuery().use { rows ->
-                    connection.prepareStatement("UPDATE $table SET identifier = ? WHERE id = ?").use { update ->
-                        while (rows.next()) {
-                            update.setString(1, identifierFromName(rows.getString("name")))
-                            update.setLong(2, rows.getLong("id"))
-                            update.addBatch()
-                        }
-                        update.executeBatch()
-                    }
-                }
-            }
         }
     }
 
@@ -92,91 +60,4 @@ class Database(private val config: DatabaseConfig) {
     ): T = getConnection().use { connection ->
         connection.prepareStatement(sql).use { statement -> statement.action() }
     }
-
-    private companion object {
-        const val DROP_ENTITY_FIELDS_TABLE = "DROP TABLE IF EXISTS entity_fields"
-        const val DROP_ENTITY_DEFINITIONS_TABLE = "DROP TABLE IF EXISTS entity_definitions"
-        const val DROP_SESSIONS_TABLE = "DROP TABLE IF EXISTS sessions"
-        const val DROP_USERS_TABLE = "DROP TABLE IF EXISTS users"
-
-        const val CREATE_USERS_TABLE = """
-            CREATE TABLE IF NOT EXISTS users (
-                id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
-                username VARCHAR(100) NOT NULL UNIQUE,
-                password_hash VARCHAR(255) NOT NULL
-            )
-        """
-
-        const val ADD_USER_ROLE = """
-            ALTER TABLE users
-            ADD COLUMN IF NOT EXISTS role VARCHAR(50) NOT NULL DEFAULT 'user'
-        """
-
-        const val CREATE_SESSIONS_TABLE = """
-            CREATE TABLE IF NOT EXISTS sessions (
-                token VARCHAR(36) PRIMARY KEY,
-                user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
-                created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
-            )
-        """
-
-        const val CREATE_ENTITY_DEFINITIONS_TABLE = """
-            CREATE TABLE IF NOT EXISTS entity_definitions (
-                id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
-                name VARCHAR(100) NOT NULL UNIQUE,
-                identifier VARCHAR(100) NOT NULL,
-                created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
-            )
-        """
-
-        const val CREATE_ENTITY_FIELDS_TABLE = """
-            CREATE TABLE IF NOT EXISTS entity_fields (
-                id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
-                entity_id BIGINT NOT NULL REFERENCES entity_definitions(id) ON DELETE CASCADE,
-                name VARCHAR(100) NOT NULL,
-                identifier VARCHAR(100) NOT NULL,
-                field_type VARCHAR(30) NOT NULL,
-                target_entity_id BIGINT REFERENCES entity_definitions(id) ON DELETE CASCADE,
-                relationship_type VARCHAR(30),
-                created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
-                UNIQUE (entity_id, name)
-            )
-        """
-
-        const val ADD_ENTITY_IDENTIFIER = """
-            ALTER TABLE entity_definitions
-            ADD COLUMN IF NOT EXISTS identifier VARCHAR(100) NOT NULL DEFAULT ''
-        """
-
-        const val ADD_FIELD_IDENTIFIER = """
-            ALTER TABLE entity_fields
-            ADD COLUMN IF NOT EXISTS identifier VARCHAR(100) NOT NULL DEFAULT ''
-        """
-
-        const val ADD_FIELD_TARGET_ENTITY = """
-            ALTER TABLE entity_fields
-            ADD COLUMN IF NOT EXISTS target_entity_id BIGINT
-        """
-
-        const val ADD_FIELD_RELATIONSHIP_TYPE = """
-            ALTER TABLE entity_fields
-            ADD COLUMN IF NOT EXISTS relationship_type VARCHAR(30)
-        """
-
-        const val ADD_FIELD_TARGET_ENTITY_CONSTRAINT = """
-            DO ${'$'}$
-            BEGIN
-                IF NOT EXISTS (
-                    SELECT 1 FROM pg_constraint
-                    WHERE conname = 'entity_fields_target_entity_id_fkey'
-                ) THEN
-                    ALTER TABLE entity_fields
-                    ADD CONSTRAINT entity_fields_target_entity_id_fkey
-                    FOREIGN KEY (target_entity_id)
-                    REFERENCES entity_definitions(id)
-                    ON DELETE CASCADE;
-                END IF;
-            END ${'$'}$
-        """
-    }
 }

src/main/kotlin/dev/mduchene/bolts/persistence/MigrationRunner.kt

@@ -0,0 +1,125 @@
+package dev.mduchene.bolts.persistence
+
+import java.net.JarURLConnection
+import java.nio.file.Files
+import java.nio.file.Path
+import java.sql.Connection
+import java.sql.Timestamp
+import java.time.Instant
+import java.util.jar.JarFile
+
+class MigrationRunner(
+    private val classLoader: ClassLoader = MigrationRunner::class.java.classLoader,
+) {
+    fun migrate(connection: Connection) {
+        createHistoryTable(connection)
+        val applied = appliedVersions(connection)
+
+        migrations().filterNot { it.version in applied }.forEach { migration ->
+            inTransaction(connection) {
+                createStatement().use { it.execute(migration.sql) }
+                prepareStatement(
+                    "INSERT INTO schema_migrations (version, filename, applied_at) VALUES (?, ?, ?)",
+                ).use {
+                    it.setString(1, migration.version)
+                    it.setString(2, migration.filename)
+                    it.setTimestamp(3, Timestamp.from(Instant.now()))
+                    it.executeUpdate()
+                }
+            }
+        }
+    }
+
+    fun reset(connection: Connection) {
+        val sql = checkNotNull(classLoader.getResource(RESET_RESOURCE)) {
+            "Missing database reset resource: $RESET_RESOURCE"
+        }.readText()
+        connection.createStatement().use { it.execute(sql) }
+    }
+
+    internal fun migrations(): List<Migration> {
+        val resources = classLoader.getResources(MIGRATION_DIRECTORY).toList()
+        check(resources.isNotEmpty()) { "No migration directory found: $MIGRATION_DIRECTORY" }
+
+        val names = resources.flatMap { directory ->
+            when (directory.protocol) {
+                "file" -> Files.list(Path.of(directory.toURI())).use { paths ->
+                    paths.filter(Files::isRegularFile)
+                        .map { it.fileName.toString() }
+                        .toList()
+                }
+                "jar" -> (directory.openConnection() as JarURLConnection).jarFile.use(::migrationNames)
+                else -> error("Unsupported migration resource protocol: ${directory.protocol}")
+            }
+        }.distinct().sorted()
+
+        val migrations = names.map { filename ->
+            val match = MIGRATION_FILENAME.matchEntire(filename)
+                ?: error("Invalid migration filename: $filename")
+            val resource = "$MIGRATION_DIRECTORY/$filename"
+            val sql = checkNotNull(classLoader.getResource(resource)) { "Missing migration resource: $resource" }
+                .readText()
+            Migration(match.groupValues[1], filename, sql)
+        }
+        check(migrations.map(Migration::version).distinct().size == migrations.size) {
+            "Migration timestamps must be unique"
+        }
+        return migrations
+    }
+
+    private fun createHistoryTable(connection: Connection) {
+        connection.createStatement().use {
+            it.execute(
+                """
+                    CREATE TABLE IF NOT EXISTS schema_migrations (
+                        version VARCHAR(14) PRIMARY KEY,
+                        filename VARCHAR(255) NOT NULL,
+                        applied_at TIMESTAMP WITH TIME ZONE NOT NULL
+                    )
+                """.trimIndent(),
+            )
+        }
+    }
+
+    private fun appliedVersions(connection: Connection): Set<String> =
+        connection.createStatement().use { statement ->
+            statement.executeQuery("SELECT version FROM schema_migrations").use { rows ->
+                buildSet {
+                    while (rows.next()) add(rows.getString("version"))
+                }
+            }
+        }
+
+    private fun inTransaction(connection: Connection, action: Connection.() -> Unit) {
+        val previousAutoCommit = connection.autoCommit
+        connection.autoCommit = false
+        try {
+            connection.action()
+            connection.commit()
+        } catch (exception: Exception) {
+            connection.rollback()
+            throw exception
+        } finally {
+            connection.autoCommit = previousAutoCommit
+        }
+    }
+
+    private fun migrationNames(jar: JarFile): List<String> =
+        jar.entries().asSequence()
+            .map { it.name }
+            .filter { it.startsWith("$MIGRATION_DIRECTORY/") && !it.endsWith("/") }
+            .map { it.substringAfterLast("/") }
+            .toList()
+
+    internal data class Migration(
+        val version: String,
+        val filename: String,
+        val sql: String,
+    )
+
+    private companion object {
+        const val MIGRATION_DIRECTORY = "db/migrations"
+        const val RESET_RESOURCE = "db/reset.sql"
+        val MIGRATION_FILENAME = Regex("""(\d{14})_[a-z0-9_]+\.sql""")
+    }
+}

src/main/resources/db/migrations/20260101000000_create_users.sql

@@ -0,0 +1,5 @@
+CREATE TABLE IF NOT EXISTS users (
+    id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    username VARCHAR(100) NOT NULL UNIQUE,
+    password_hash VARCHAR(255) NOT NULL
+);

src/main/resources/db/migrations/20260101000100_add_user_role.sql

@@ -0,0 +1,2 @@
+ALTER TABLE users
+ADD COLUMN IF NOT EXISTS role VARCHAR(50) NOT NULL DEFAULT 'user';

src/main/resources/db/migrations/20260101000200_create_sessions.sql

@@ -0,0 +1,5 @@
+CREATE TABLE IF NOT EXISTS sessions (
+    token VARCHAR(36) PRIMARY KEY,
+    user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
+);

src/main/resources/db/migrations/20260101000300_create_entity_definitions.sql

@@ -0,0 +1,5 @@
+CREATE TABLE IF NOT EXISTS entity_definitions (
+    id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    name VARCHAR(100) NOT NULL UNIQUE,
+    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
+);

src/main/resources/db/migrations/20260101000400_create_entity_fields.sql

@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS entity_fields (
+    id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    entity_id BIGINT NOT NULL REFERENCES entity_definitions(id) ON DELETE CASCADE,
+    name VARCHAR(100) NOT NULL,
+    field_type VARCHAR(30) NOT NULL,
+    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    UNIQUE (entity_id, name)
+);

src/main/resources/db/migrations/20260101000500_add_field_relationships.sql

@@ -0,0 +1,19 @@
+ALTER TABLE entity_fields
+ADD COLUMN IF NOT EXISTS target_entity_id BIGINT;
+
+ALTER TABLE entity_fields
+ADD COLUMN IF NOT EXISTS relationship_type VARCHAR(30);
+
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM pg_constraint
+        WHERE conname = 'entity_fields_target_entity_id_fkey'
+    ) THEN
+        ALTER TABLE entity_fields
+        ADD CONSTRAINT entity_fields_target_entity_id_fkey
+        FOREIGN KEY (target_entity_id)
+        REFERENCES entity_definitions(id)
+        ON DELETE CASCADE;
+    END IF;
+END $$;

src/main/resources/db/migrations/20260101000600_add_identifiers.sql

@@ -0,0 +1,53 @@
+ALTER TABLE entity_definitions
+ADD COLUMN IF NOT EXISTS identifier VARCHAR(100) NOT NULL DEFAULT '';
+
+ALTER TABLE entity_fields
+ADD COLUMN IF NOT EXISTS identifier VARCHAR(100) NOT NULL DEFAULT '';
+
+WITH identifiers AS (
+    SELECT
+        id,
+        REGEXP_REPLACE(
+            INITCAP(
+                REGEXP_REPLACE(
+                    REGEXP_REPLACE(
+                        TRANSLATE(name, 'ÀÁÂÃÄÅàáâãäåÇçÈÉÊËèéêëÌÍÎÏìíîïÑñÒÓÔÕÖòóôõöÙÚÛÜùúûüÝŸýÿ',
+                            'AAAAAAaaaaaaCcEEEEeeeeIIIIiiiiNnOOOOOoooooUUUUuuuuYYyy'),
+                        '([a-z0-9])([A-Z])', '\1 \2', 'g'
+                    ),
+                    '[^a-zA-Z0-9]+', ' ', 'g'
+                )
+            ),
+            '\s+', '', 'g'
+        ) AS value
+    FROM entity_definitions
+    WHERE identifier = ''
+)
+UPDATE entity_definitions
+SET identifier = LOWER(LEFT(identifiers.value, 1)) || SUBSTRING(identifiers.value FROM 2)
+FROM identifiers
+WHERE entity_definitions.id = identifiers.id;
+
+WITH identifiers AS (
+    SELECT
+        id,
+        REGEXP_REPLACE(
+            INITCAP(
+                REGEXP_REPLACE(
+                    REGEXP_REPLACE(
+                        TRANSLATE(name, 'ÀÁÂÃÄÅàáâãäåÇçÈÉÊËèéêëÌÍÎÏìíîïÑñÒÓÔÕÖòóôõöÙÚÛÜùúûüÝŸýÿ',
+                            'AAAAAAaaaaaaCcEEEEeeeeIIIIiiiiNnOOOOOoooooUUUUuuuuYYyy'),
+                        '([a-z0-9])([A-Z])', '\1 \2', 'g'
+                    ),
+                    '[^a-zA-Z0-9]+', ' ', 'g'
+                )
+            ),
+            '\s+', '', 'g'
+        ) AS value
+    FROM entity_fields
+    WHERE identifier = ''
+)
+UPDATE entity_fields
+SET identifier = LOWER(LEFT(identifiers.value, 1)) || SUBSTRING(identifiers.value FROM 2)
+FROM identifiers
+WHERE entity_fields.id = identifiers.id;

src/main/resources/db/reset.sql

@@ -0,0 +1,5 @@
+DROP TABLE IF EXISTS entity_fields;
+DROP TABLE IF EXISTS entity_definitions;
+DROP TABLE IF EXISTS sessions;
+DROP TABLE IF EXISTS users;
+DROP TABLE IF EXISTS schema_migrations;

src/test/kotlin/dev/mduchene/bolts/persistence/MigrationRunnerTest.kt

@@ -0,0 +1,54 @@
+package dev.mduchene.bolts.persistence
+
+import java.sql.DriverManager
+import java.util.UUID
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertTrue
+
+class MigrationRunnerTest {
+    @Test
+    fun `discovers migrations in timestamp order`() {
+        val migrations = MigrationRunner().migrations()
+
+        assertTrue(migrations.isNotEmpty())
+        assertEquals(migrations.map { it.version }.sorted(), migrations.map { it.version })
+        assertEquals(migrations.size, migrations.map { it.version }.distinct().size)
+        assertEquals("20260101000000_create_users.sql", migrations.first().filename)
+    }
+
+    @Test
+    fun `runs each migration only once`() {
+        val schema = "migration_test_${UUID.randomUUID().toString().replace("-", "")}"
+        DriverManager.getConnection(
+            environmentOrDefault("TEST_DB_URL", "jdbc:postgresql://localhost:5432/postgres"),
+            environmentOrDefault("TEST_DB_USERNAME", "root"),
+            environmentOrDefault("TEST_DB_PASSWORD", "root"),
+        ).use { connection ->
+            try {
+                connection.createStatement().use {
+                    it.execute("CREATE SCHEMA $schema")
+                    it.execute("SET search_path TO $schema")
+                }
+
+                val runner = MigrationRunner()
+                runner.migrate(connection)
+                runner.migrate(connection)
+
+                connection.createStatement().use { statement ->
+                    statement.executeQuery("SELECT COUNT(*) FROM schema_migrations").use { result ->
+                        result.next()
+                        assertEquals(runner.migrations().size, result.getInt(1))
+                    }
+                }
+            } finally {
+                connection.createStatement().use {
+                    it.execute("DROP SCHEMA IF EXISTS $schema CASCADE")
+                }
+            }
+        }
+    }
+
+    private fun environmentOrDefault(name: String, defaultValue: String): String =
+        System.getenv(name)?.takeIf { it.isNotBlank() } ?: defaultValue
+}