diff --git a/AGENTS.md b/AGENTS.md index 471e96a..79cf73b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,24 @@ - Persistence: Postgres with JDBC - Maven for dependency management +## Database migrations + +All internal database schema and data changes must be made with SQL migrations in +`src/main/resources/db/migrations`. Migration filenames use: + +```text +yyyyMMddHHmmss_short_description.sql +``` + +The 14-digit UTC timestamp is the migration version and must be unique. Migrations +are run in filename order at application startup, inside one transaction per file. +Applied versions are recorded in `schema_migrations`, so never edit or rename a +migration that may have run in another environment; add a new migration instead. + +Keep migrations compatible with Postgres. Update `src/main/resources/db/reset.sql` +when adding a table so `DB_REINITIALIZE=true` can continue to rebuild disposable +test databases by dropping the application tables and replaying all migrations. + ## Tests diff --git a/src/main/kotlin/dev/mduchene/bolts/persistence/Database.kt b/src/main/kotlin/dev/mduchene/bolts/persistence/Database.kt index d4240e8..ec37bff 100644 --- a/src/main/kotlin/dev/mduchene/bolts/persistence/Database.kt +++ b/src/main/kotlin/dev/mduchene/bolts/persistence/Database.kt @@ -1,12 +1,14 @@ package dev.mduchene.bolts.persistence -import dev.mduchene.bolts.entity.identifierFromName import java.sql.Connection import java.sql.DriverManager import java.sql.PreparedStatement import java.sql.ResultSet -class Database(private val config: DatabaseConfig) { +class Database( + private val config: DatabaseConfig, + private val migrationRunner: MigrationRunner = MigrationRunner(), +) { fun getConnection(): Connection = DriverManager.getConnection(config.url, config.username, config.password) @@ -38,42 +40,8 @@ class Database(private val config: DatabaseConfig) { fun initialize(reinitialize: Boolean = false) { getConnection().use { connection -> - connection.createStatement().use { statement -> - if (reinitialize) { - statement.execute(DROP_ENTITY_FIELDS_TABLE) - statement.execute(DROP_ENTITY_DEFINITIONS_TABLE) - statement.execute(DROP_SESSIONS_TABLE) - statement.execute(DROP_USERS_TABLE) - } - statement.execute(CREATE_USERS_TABLE) - statement.execute(ADD_USER_ROLE) - statement.execute(CREATE_SESSIONS_TABLE) - statement.execute(CREATE_ENTITY_DEFINITIONS_TABLE) - statement.execute(CREATE_ENTITY_FIELDS_TABLE) - statement.execute(ADD_ENTITY_IDENTIFIER) - statement.execute(ADD_FIELD_IDENTIFIER) - statement.execute(ADD_FIELD_TARGET_ENTITY) - statement.execute(ADD_FIELD_RELATIONSHIP_TYPE) - statement.execute(ADD_FIELD_TARGET_ENTITY_CONSTRAINT) - } - backfillIdentifiers(connection) - } - } - - private fun backfillIdentifiers(connection: Connection) { - listOf("entity_definitions", "entity_fields").forEach { table -> - connection.prepareStatement("SELECT id, name FROM $table WHERE identifier = ''").use { select -> - select.executeQuery().use { rows -> - connection.prepareStatement("UPDATE $table SET identifier = ? WHERE id = ?").use { update -> - while (rows.next()) { - update.setString(1, identifierFromName(rows.getString("name"))) - update.setLong(2, rows.getLong("id")) - update.addBatch() - } - update.executeBatch() - } - } - } + if (reinitialize) migrationRunner.reset(connection) + migrationRunner.migrate(connection) } } @@ -92,91 +60,4 @@ class Database(private val config: DatabaseConfig) { ): T = getConnection().use { connection -> connection.prepareStatement(sql).use { statement -> statement.action() } } - - private companion object { - const val DROP_ENTITY_FIELDS_TABLE = "DROP TABLE IF EXISTS entity_fields" - const val DROP_ENTITY_DEFINITIONS_TABLE = "DROP TABLE IF EXISTS entity_definitions" - const val DROP_SESSIONS_TABLE = "DROP TABLE IF EXISTS sessions" - const val DROP_USERS_TABLE = "DROP TABLE IF EXISTS users" - - const val CREATE_USERS_TABLE = """ - CREATE TABLE IF NOT EXISTS users ( - id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - username VARCHAR(100) NOT NULL UNIQUE, - password_hash VARCHAR(255) NOT NULL - ) - """ - - const val ADD_USER_ROLE = """ - ALTER TABLE users - ADD COLUMN IF NOT EXISTS role VARCHAR(50) NOT NULL DEFAULT 'user' - """ - - const val CREATE_SESSIONS_TABLE = """ - CREATE TABLE IF NOT EXISTS sessions ( - token VARCHAR(36) PRIMARY KEY, - user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - """ - - const val CREATE_ENTITY_DEFINITIONS_TABLE = """ - CREATE TABLE IF NOT EXISTS entity_definitions ( - id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - name VARCHAR(100) NOT NULL UNIQUE, - identifier VARCHAR(100) NOT NULL, - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - """ - - const val CREATE_ENTITY_FIELDS_TABLE = """ - CREATE TABLE IF NOT EXISTS entity_fields ( - id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - entity_id BIGINT NOT NULL REFERENCES entity_definitions(id) ON DELETE CASCADE, - name VARCHAR(100) NOT NULL, - identifier VARCHAR(100) NOT NULL, - field_type VARCHAR(30) NOT NULL, - target_entity_id BIGINT REFERENCES entity_definitions(id) ON DELETE CASCADE, - relationship_type VARCHAR(30), - created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - UNIQUE (entity_id, name) - ) - """ - - const val ADD_ENTITY_IDENTIFIER = """ - ALTER TABLE entity_definitions - ADD COLUMN IF NOT EXISTS identifier VARCHAR(100) NOT NULL DEFAULT '' - """ - - const val ADD_FIELD_IDENTIFIER = """ - ALTER TABLE entity_fields - ADD COLUMN IF NOT EXISTS identifier VARCHAR(100) NOT NULL DEFAULT '' - """ - - const val ADD_FIELD_TARGET_ENTITY = """ - ALTER TABLE entity_fields - ADD COLUMN IF NOT EXISTS target_entity_id BIGINT - """ - - const val ADD_FIELD_RELATIONSHIP_TYPE = """ - ALTER TABLE entity_fields - ADD COLUMN IF NOT EXISTS relationship_type VARCHAR(30) - """ - - const val ADD_FIELD_TARGET_ENTITY_CONSTRAINT = """ - DO ${'$'}$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_constraint - WHERE conname = 'entity_fields_target_entity_id_fkey' - ) THEN - ALTER TABLE entity_fields - ADD CONSTRAINT entity_fields_target_entity_id_fkey - FOREIGN KEY (target_entity_id) - REFERENCES entity_definitions(id) - ON DELETE CASCADE; - END IF; - END ${'$'}$ - """ - } } diff --git a/src/main/kotlin/dev/mduchene/bolts/persistence/MigrationRunner.kt b/src/main/kotlin/dev/mduchene/bolts/persistence/MigrationRunner.kt new file mode 100644 index 0000000..29a49b9 --- /dev/null +++ b/src/main/kotlin/dev/mduchene/bolts/persistence/MigrationRunner.kt @@ -0,0 +1,125 @@ +package dev.mduchene.bolts.persistence + +import java.net.JarURLConnection +import java.nio.file.Files +import java.nio.file.Path +import java.sql.Connection +import java.sql.Timestamp +import java.time.Instant +import java.util.jar.JarFile + +class MigrationRunner( + private val classLoader: ClassLoader = MigrationRunner::class.java.classLoader, +) { + fun migrate(connection: Connection) { + createHistoryTable(connection) + val applied = appliedVersions(connection) + + migrations().filterNot { it.version in applied }.forEach { migration -> + inTransaction(connection) { + createStatement().use { it.execute(migration.sql) } + prepareStatement( + "INSERT INTO schema_migrations (version, filename, applied_at) VALUES (?, ?, ?)", + ).use { + it.setString(1, migration.version) + it.setString(2, migration.filename) + it.setTimestamp(3, Timestamp.from(Instant.now())) + it.executeUpdate() + } + } + } + } + + fun reset(connection: Connection) { + val sql = checkNotNull(classLoader.getResource(RESET_RESOURCE)) { + "Missing database reset resource: $RESET_RESOURCE" + }.readText() + connection.createStatement().use { it.execute(sql) } + } + + internal fun migrations(): List { + val resources = classLoader.getResources(MIGRATION_DIRECTORY).toList() + check(resources.isNotEmpty()) { "No migration directory found: $MIGRATION_DIRECTORY" } + + val names = resources.flatMap { directory -> + when (directory.protocol) { + "file" -> Files.list(Path.of(directory.toURI())).use { paths -> + paths.filter(Files::isRegularFile) + .map { it.fileName.toString() } + .toList() + } + "jar" -> (directory.openConnection() as JarURLConnection).jarFile.use(::migrationNames) + else -> error("Unsupported migration resource protocol: ${directory.protocol}") + } + }.distinct().sorted() + + val migrations = names.map { filename -> + val match = MIGRATION_FILENAME.matchEntire(filename) + ?: error("Invalid migration filename: $filename") + val resource = "$MIGRATION_DIRECTORY/$filename" + val sql = checkNotNull(classLoader.getResource(resource)) { "Missing migration resource: $resource" } + .readText() + Migration(match.groupValues[1], filename, sql) + } + check(migrations.map(Migration::version).distinct().size == migrations.size) { + "Migration timestamps must be unique" + } + return migrations + } + + private fun createHistoryTable(connection: Connection) { + connection.createStatement().use { + it.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version VARCHAR(14) PRIMARY KEY, + filename VARCHAR(255) NOT NULL, + applied_at TIMESTAMP WITH TIME ZONE NOT NULL + ) + """.trimIndent(), + ) + } + } + + private fun appliedVersions(connection: Connection): Set = + connection.createStatement().use { statement -> + statement.executeQuery("SELECT version FROM schema_migrations").use { rows -> + buildSet { + while (rows.next()) add(rows.getString("version")) + } + } + } + + private fun inTransaction(connection: Connection, action: Connection.() -> Unit) { + val previousAutoCommit = connection.autoCommit + connection.autoCommit = false + try { + connection.action() + connection.commit() + } catch (exception: Exception) { + connection.rollback() + throw exception + } finally { + connection.autoCommit = previousAutoCommit + } + } + + private fun migrationNames(jar: JarFile): List = + jar.entries().asSequence() + .map { it.name } + .filter { it.startsWith("$MIGRATION_DIRECTORY/") && !it.endsWith("/") } + .map { it.substringAfterLast("/") } + .toList() + + internal data class Migration( + val version: String, + val filename: String, + val sql: String, + ) + + private companion object { + const val MIGRATION_DIRECTORY = "db/migrations" + const val RESET_RESOURCE = "db/reset.sql" + val MIGRATION_FILENAME = Regex("""(\d{14})_[a-z0-9_]+\.sql""") + } +} diff --git a/src/main/resources/db/migrations/20260101000000_create_users.sql b/src/main/resources/db/migrations/20260101000000_create_users.sql new file mode 100644 index 0000000..4fa59ff --- /dev/null +++ b/src/main/resources/db/migrations/20260101000000_create_users.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS users ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + username VARCHAR(100) NOT NULL UNIQUE, + password_hash VARCHAR(255) NOT NULL +); diff --git a/src/main/resources/db/migrations/20260101000100_add_user_role.sql b/src/main/resources/db/migrations/20260101000100_add_user_role.sql new file mode 100644 index 0000000..cb48d10 --- /dev/null +++ b/src/main/resources/db/migrations/20260101000100_add_user_role.sql @@ -0,0 +1,2 @@ +ALTER TABLE users +ADD COLUMN IF NOT EXISTS role VARCHAR(50) NOT NULL DEFAULT 'user'; diff --git a/src/main/resources/db/migrations/20260101000200_create_sessions.sql b/src/main/resources/db/migrations/20260101000200_create_sessions.sql new file mode 100644 index 0000000..f47d190 --- /dev/null +++ b/src/main/resources/db/migrations/20260101000200_create_sessions.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS sessions ( + token VARCHAR(36) PRIMARY KEY, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP +); diff --git a/src/main/resources/db/migrations/20260101000300_create_entity_definitions.sql b/src/main/resources/db/migrations/20260101000300_create_entity_definitions.sql new file mode 100644 index 0000000..1a64f0e --- /dev/null +++ b/src/main/resources/db/migrations/20260101000300_create_entity_definitions.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS entity_definitions ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name VARCHAR(100) NOT NULL UNIQUE, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP +); diff --git a/src/main/resources/db/migrations/20260101000400_create_entity_fields.sql b/src/main/resources/db/migrations/20260101000400_create_entity_fields.sql new file mode 100644 index 0000000..802cb42 --- /dev/null +++ b/src/main/resources/db/migrations/20260101000400_create_entity_fields.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS entity_fields ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + entity_id BIGINT NOT NULL REFERENCES entity_definitions(id) ON DELETE CASCADE, + name VARCHAR(100) NOT NULL, + field_type VARCHAR(30) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE (entity_id, name) +); diff --git a/src/main/resources/db/migrations/20260101000500_add_field_relationships.sql b/src/main/resources/db/migrations/20260101000500_add_field_relationships.sql new file mode 100644 index 0000000..967376e --- /dev/null +++ b/src/main/resources/db/migrations/20260101000500_add_field_relationships.sql @@ -0,0 +1,19 @@ +ALTER TABLE entity_fields +ADD COLUMN IF NOT EXISTS target_entity_id BIGINT; + +ALTER TABLE entity_fields +ADD COLUMN IF NOT EXISTS relationship_type VARCHAR(30); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'entity_fields_target_entity_id_fkey' + ) THEN + ALTER TABLE entity_fields + ADD CONSTRAINT entity_fields_target_entity_id_fkey + FOREIGN KEY (target_entity_id) + REFERENCES entity_definitions(id) + ON DELETE CASCADE; + END IF; +END $$; diff --git a/src/main/resources/db/migrations/20260101000600_add_identifiers.sql b/src/main/resources/db/migrations/20260101000600_add_identifiers.sql new file mode 100644 index 0000000..5fc74c5 --- /dev/null +++ b/src/main/resources/db/migrations/20260101000600_add_identifiers.sql @@ -0,0 +1,53 @@ +ALTER TABLE entity_definitions +ADD COLUMN IF NOT EXISTS identifier VARCHAR(100) NOT NULL DEFAULT ''; + +ALTER TABLE entity_fields +ADD COLUMN IF NOT EXISTS identifier VARCHAR(100) NOT NULL DEFAULT ''; + +WITH identifiers AS ( + SELECT + id, + REGEXP_REPLACE( + INITCAP( + REGEXP_REPLACE( + REGEXP_REPLACE( + TRANSLATE(name, 'ÀÁÂÃÄÅàáâãäåÇçÈÉÊËèéêëÌÍÎÏìíîïÑñÒÓÔÕÖòóôõöÙÚÛÜùúûüÝŸýÿ', + 'AAAAAAaaaaaaCcEEEEeeeeIIIIiiiiNnOOOOOoooooUUUUuuuuYYyy'), + '([a-z0-9])([A-Z])', '\1 \2', 'g' + ), + '[^a-zA-Z0-9]+', ' ', 'g' + ) + ), + '\s+', '', 'g' + ) AS value + FROM entity_definitions + WHERE identifier = '' +) +UPDATE entity_definitions +SET identifier = LOWER(LEFT(identifiers.value, 1)) || SUBSTRING(identifiers.value FROM 2) +FROM identifiers +WHERE entity_definitions.id = identifiers.id; + +WITH identifiers AS ( + SELECT + id, + REGEXP_REPLACE( + INITCAP( + REGEXP_REPLACE( + REGEXP_REPLACE( + TRANSLATE(name, 'ÀÁÂÃÄÅàáâãäåÇçÈÉÊËèéêëÌÍÎÏìíîïÑñÒÓÔÕÖòóôõöÙÚÛÜùúûüÝŸýÿ', + 'AAAAAAaaaaaaCcEEEEeeeeIIIIiiiiNnOOOOOoooooUUUUuuuuYYyy'), + '([a-z0-9])([A-Z])', '\1 \2', 'g' + ), + '[^a-zA-Z0-9]+', ' ', 'g' + ) + ), + '\s+', '', 'g' + ) AS value + FROM entity_fields + WHERE identifier = '' +) +UPDATE entity_fields +SET identifier = LOWER(LEFT(identifiers.value, 1)) || SUBSTRING(identifiers.value FROM 2) +FROM identifiers +WHERE entity_fields.id = identifiers.id; diff --git a/src/main/resources/db/reset.sql b/src/main/resources/db/reset.sql new file mode 100644 index 0000000..b6ca9cd --- /dev/null +++ b/src/main/resources/db/reset.sql @@ -0,0 +1,5 @@ +DROP TABLE IF EXISTS entity_fields; +DROP TABLE IF EXISTS entity_definitions; +DROP TABLE IF EXISTS sessions; +DROP TABLE IF EXISTS users; +DROP TABLE IF EXISTS schema_migrations; diff --git a/src/test/kotlin/dev/mduchene/bolts/persistence/MigrationRunnerTest.kt b/src/test/kotlin/dev/mduchene/bolts/persistence/MigrationRunnerTest.kt new file mode 100644 index 0000000..81326de --- /dev/null +++ b/src/test/kotlin/dev/mduchene/bolts/persistence/MigrationRunnerTest.kt @@ -0,0 +1,54 @@ +package dev.mduchene.bolts.persistence + +import java.sql.DriverManager +import java.util.UUID +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class MigrationRunnerTest { + @Test + fun `discovers migrations in timestamp order`() { + val migrations = MigrationRunner().migrations() + + assertTrue(migrations.isNotEmpty()) + assertEquals(migrations.map { it.version }.sorted(), migrations.map { it.version }) + assertEquals(migrations.size, migrations.map { it.version }.distinct().size) + assertEquals("20260101000000_create_users.sql", migrations.first().filename) + } + + @Test + fun `runs each migration only once`() { + val schema = "migration_test_${UUID.randomUUID().toString().replace("-", "")}" + DriverManager.getConnection( + environmentOrDefault("TEST_DB_URL", "jdbc:postgresql://localhost:5432/postgres"), + environmentOrDefault("TEST_DB_USERNAME", "root"), + environmentOrDefault("TEST_DB_PASSWORD", "root"), + ).use { connection -> + try { + connection.createStatement().use { + it.execute("CREATE SCHEMA $schema") + it.execute("SET search_path TO $schema") + } + + val runner = MigrationRunner() + runner.migrate(connection) + runner.migrate(connection) + + connection.createStatement().use { statement -> + statement.executeQuery("SELECT COUNT(*) FROM schema_migrations").use { result -> + result.next() + assertEquals(runner.migrations().size, result.getInt(1)) + } + } + } finally { + connection.createStatement().use { + it.execute("DROP SCHEMA IF EXISTS $schema CASCADE") + } + } + } + } + + private fun environmentOrDefault(name: String, defaultValue: String): String = + System.getenv(name)?.takeIf { it.isNotBlank() } ?: defaultValue +}