mduchene/bolts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use a migration system

Browse Source
This commit is contained in:
Maxime Duchêne-Savard 2026-07-29 16:38:25 -04:00
parent ccfc6dcfa4
commit 4fb57ef61b
12 changed files with 305 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
AGENTS.md
View File

@ -16,6 +16,24 @@
- Persistence: Postgres with JDBC
- Maven for dependency management
## Database migrations
All internal database schema and data changes must be made with SQL migrations in
`src/main/resources/db/migrations`. Migration filenames use:
```text
yyyyMMddHHmmss_short_description.sql
```
The 14-digit UTC timestamp is the migration version and must be unique. Migrations
are run in filename order at application startup, inside one transaction per file.
Applied versions are recorded in `schema_migrations`, so never edit or rename a
migration that may have run in another environment; add a new migration instead.
Keep migrations compatible with Postgres. Update `src/main/resources/db/reset.sql`
when adding a table so `DB_REINITIALIZE=true` can continue to rebuild disposable
test databases by dropping the application tables and replaying all migrations.
## Tests

131
src/main/kotlin/dev/mduchene/bolts/persistence/Database.kt
View File

@ -1,12 +1,14 @@
package dev.mduchene.bolts.persistence
import dev.mduchene.bolts.entity.identifierFromName
import java.sql.Connection
import java.sql.DriverManager
import java.sql.PreparedStatement
import java.sql.ResultSet
class Database(private val config: DatabaseConfig) {
class Database(
private val config: DatabaseConfig,
private val migrationRunner: MigrationRunner = MigrationRunner(),
) {
fun getConnection(): Connection =
DriverManager.getConnection(config.url, config.username, config.password)
@ -38,42 +40,8 @@ class Database(private val config: DatabaseConfig) {
fun initialize(reinitialize: Boolean = false) {
getConnection().use { connection ->
connection.createStatement().use { statement ->
if (reinitialize) {
statement.execute(DROP_ENTITY_FIELDS_TABLE)
statement.execute(DROP_ENTITY_DEFINITIONS_TABLE)
statement.execute(DROP_SESSIONS_TABLE)
statement.execute(DROP_USERS_TABLE)
}
statement.execute(CREATE_USERS_TABLE)
statement.execute(ADD_USER_ROLE)
statement.execute(CREATE_SESSIONS_TABLE)
statement.execute(CREATE_ENTITY_DEFINITIONS_TABLE)
statement.execute(CREATE_ENTITY_FIELDS_TABLE)
statement.execute(ADD_ENTITY_IDENTIFIER)
statement.execute(ADD_FIELD_IDENTIFIER)
statement.execute(ADD_FIELD_TARGET_ENTITY)
statement.execute(ADD_FIELD_RELATIONSHIP_TYPE)
statement.execute(ADD_FIELD_TARGET_ENTITY_CONSTRAINT)
}
backfillIdentifiers(connection)
}
}
private fun backfillIdentifiers(connection: Connection) {
listOf("entity_definitions", "entity_fields").forEach { table ->
connection.prepareStatement("SELECT id, name FROM $table WHERE identifier = ''").use { select ->
select.executeQuery().use { rows ->
connection.prepareStatement("UPDATE $table SET identifier = ? WHERE id = ?").use { update ->
while (rows.next()) {
update.setString(1, identifierFromName(rows.getString("name")))
update.setLong(2, rows.getLong("id"))
update.addBatch()
}
update.executeBatch()
}
}
}
if (reinitialize) migrationRunner.reset(connection)
migrationRunner.migrate(connection)
}
}
@ -92,91 +60,4 @@ class Database(private val config: DatabaseConfig) {
): T = getConnection().use { connection ->
connection.prepareStatement(sql).use { statement -> statement.action() }
}
private companion object {
const val DROP_ENTITY_FIELDS_TABLE = "DROP TABLE IF EXISTS entity_fields"
const val DROP_ENTITY_DEFINITIONS_TABLE = "DROP TABLE IF EXISTS entity_definitions"
const val DROP_SESSIONS_TABLE = "DROP TABLE IF EXISTS sessions"
const val DROP_USERS_TABLE = "DROP TABLE IF EXISTS users"
const val CREATE_USERS_TABLE = """
CREATE TABLE IF NOT EXISTS users (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
username VARCHAR(100) NOT NULL UNIQUE,
password_hash VARCHAR(255) NOT NULL
)
"""
const val ADD_USER_ROLE = """
ALTER TABLE users
ADD COLUMN IF NOT EXISTS role VARCHAR(50) NOT NULL DEFAULT 'user'
"""
const val CREATE_SESSIONS_TABLE = """
CREATE TABLE IF NOT EXISTS sessions (
token VARCHAR(36) PRIMARY KEY,
user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
)
"""
const val CREATE_ENTITY_DEFINITIONS_TABLE = """
CREATE TABLE IF NOT EXISTS entity_definitions (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
name VARCHAR(100) NOT NULL UNIQUE,
identifier VARCHAR(100) NOT NULL,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
)
"""
const val CREATE_ENTITY_FIELDS_TABLE = """
CREATE TABLE IF NOT EXISTS entity_fields (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
entity_id BIGINT NOT NULL REFERENCES entity_definitions(id) ON DELETE CASCADE,
name VARCHAR(100) NOT NULL,
identifier VARCHAR(100) NOT NULL,
field_type VARCHAR(30) NOT NULL,
target_entity_id BIGINT REFERENCES entity_definitions(id) ON DELETE CASCADE,
relationship_type VARCHAR(30),
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE (entity_id, name)
)
"""
const val ADD_ENTITY_IDENTIFIER = """
ALTER TABLE entity_definitions
ADD COLUMN IF NOT EXISTS identifier VARCHAR(100) NOT NULL DEFAULT ''
"""
const val ADD_FIELD_IDENTIFIER = """
ALTER TABLE entity_fields
ADD COLUMN IF NOT EXISTS identifier VARCHAR(100) NOT NULL DEFAULT ''
"""
const val ADD_FIELD_TARGET_ENTITY = """
ALTER TABLE entity_fields
ADD COLUMN IF NOT EXISTS target_entity_id BIGINT
"""
const val ADD_FIELD_RELATIONSHIP_TYPE = """
ALTER TABLE entity_fields
ADD COLUMN IF NOT EXISTS relationship_type VARCHAR(30)
"""
const val ADD_FIELD_TARGET_ENTITY_CONSTRAINT = """
DO ${'$'}$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_constraint
WHERE conname = 'entity_fields_target_entity_id_fkey'
) THEN
ALTER TABLE entity_fields
ADD CONSTRAINT entity_fields_target_entity_id_fkey
FOREIGN KEY (target_entity_id)
REFERENCES entity_definitions(id)
ON DELETE CASCADE;
END IF;
END ${'$'}$
"""
}
}

125
src/main/kotlin/dev/mduchene/bolts/persistence/MigrationRunner.kt Normal file
View File

@ -0,0 +1,125 @@
package dev.mduchene.bolts.persistence
import java.net.JarURLConnection
import java.nio.file.Files
import java.nio.file.Path
import java.sql.Connection
import java.sql.Timestamp
import java.time.Instant
import java.util.jar.JarFile
class MigrationRunner(
private val classLoader: ClassLoader = MigrationRunner::class.java.classLoader,
) {
fun migrate(connection: Connection) {
createHistoryTable(connection)
val applied = appliedVersions(connection)
migrations().filterNot { it.version in applied }.forEach { migration ->
inTransaction(connection) {
createStatement().use { it.execute(migration.sql) }
prepareStatement(
"INSERT INTO schema_migrations (version, filename, applied_at) VALUES (?, ?, ?)",
).use {
it.setString(1, migration.version)
it.setString(2, migration.filename)
it.setTimestamp(3, Timestamp.from(Instant.now()))
it.executeUpdate()
}
}
}
}
fun reset(connection: Connection) {
val sql = checkNotNull(classLoader.getResource(RESET_RESOURCE)) {
"Missing database reset resource: $RESET_RESOURCE"
}.readText()
connection.createStatement().use { it.execute(sql) }
}
internal fun migrations(): List<Migration> {
val resources = classLoader.getResources(MIGRATION_DIRECTORY).toList()
check(resources.isNotEmpty()) { "No migration directory found: $MIGRATION_DIRECTORY" }
val names = resources.flatMap { directory ->
when (directory.protocol) {
"file" -> Files.list(Path.of(directory.toURI())).use { paths ->
paths.filter(Files::isRegularFile)
.map { it.fileName.toString() }
.toList()
}
"jar" -> (directory.openConnection() as JarURLConnection).jarFile.use(::migrationNames)
else -> error("Unsupported migration resource protocol: ${directory.protocol}")
}
}.distinct().sorted()
val migrations = names.map { filename ->
val match = MIGRATION_FILENAME.matchEntire(filename)
?: error("Invalid migration filename: $filename")
val resource = "$MIGRATION_DIRECTORY/$filename"
val sql = checkNotNull(classLoader.getResource(resource)) { "Missing migration resource: $resource" }
.readText()
Migration(match.groupValues[1], filename, sql)
}
check(migrations.map(Migration::version).distinct().size == migrations.size) {
"Migration timestamps must be unique"
}
return migrations
}
private fun createHistoryTable(connection: Connection) {
connection.createStatement().use {
it.execute(
"""
CREATE TABLE IF NOT EXISTS schema_migrations (
version VARCHAR(14) PRIMARY KEY,
filename VARCHAR(255) NOT NULL,
applied_at TIMESTAMP WITH TIME ZONE NOT NULL
)
""".trimIndent(),
)
}
}
private fun appliedVersions(connection: Connection): Set<String> =
connection.createStatement().use { statement ->
statement.executeQuery("SELECT version FROM schema_migrations").use { rows ->
buildSet {
while (rows.next()) add(rows.getString("version"))
}
}
}
private fun inTransaction(connection: Connection, action: Connection.() -> Unit) {
val previousAutoCommit = connection.autoCommit
connection.autoCommit = false
try {
connection.action()
connection.commit()
} catch (exception: Exception) {
connection.rollback()
throw exception
} finally {
connection.autoCommit = previousAutoCommit
}
}
private fun migrationNames(jar: JarFile): List<String> =
jar.entries().asSequence()
.map { it.name }
.filter { it.startsWith("$MIGRATION_DIRECTORY/") && !it.endsWith("/") }
.map { it.substringAfterLast("/") }
.toList()
internal data class Migration(
val version: String,
val filename: String,
val sql: String,
)
private companion object {
const val MIGRATION_DIRECTORY = "db/migrations"
const val RESET_RESOURCE = "db/reset.sql"
val MIGRATION_FILENAME = Regex("""(\d{14})_[a-z0-9_]+\.sql""")
}
}

5
src/main/resources/db/migrations/20260101000000_create_users.sql Normal file
View File

@ -0,0 +1,5 @@
CREATE TABLE IF NOT EXISTS users (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
username VARCHAR(100) NOT NULL UNIQUE,
password_hash VARCHAR(255) NOT NULL
);

2
src/main/resources/db/migrations/20260101000100_add_user_role.sql Normal file
View File

@ -0,0 +1,2 @@
ALTER TABLE users
ADD COLUMN IF NOT EXISTS role VARCHAR(50) NOT NULL DEFAULT 'user';

5
src/main/resources/db/migrations/20260101000200_create_sessions.sql Normal file
View File

@ -0,0 +1,5 @@
CREATE TABLE IF NOT EXISTS sessions (
token VARCHAR(36) PRIMARY KEY,
user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
);

5
src/main/resources/db/migrations/20260101000300_create_entity_definitions.sql Normal file
View File

@ -0,0 +1,5 @@
CREATE TABLE IF NOT EXISTS entity_definitions (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
name VARCHAR(100) NOT NULL UNIQUE,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
);

8
src/main/resources/db/migrations/20260101000400_create_entity_fields.sql Normal file
View File

@ -0,0 +1,8 @@
CREATE TABLE IF NOT EXISTS entity_fields (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
entity_id BIGINT NOT NULL REFERENCES entity_definitions(id) ON DELETE CASCADE,
name VARCHAR(100) NOT NULL,
field_type VARCHAR(30) NOT NULL,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE (entity_id, name)
);

19
src/main/resources/db/migrations/20260101000500_add_field_relationships.sql Normal file
View File

@ -0,0 +1,19 @@
ALTER TABLE entity_fields
ADD COLUMN IF NOT EXISTS target_entity_id BIGINT;
ALTER TABLE entity_fields
ADD COLUMN IF NOT EXISTS relationship_type VARCHAR(30);
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_constraint
WHERE conname = 'entity_fields_target_entity_id_fkey'
) THEN
ALTER TABLE entity_fields
ADD CONSTRAINT entity_fields_target_entity_id_fkey
FOREIGN KEY (target_entity_id)
REFERENCES entity_definitions(id)
ON DELETE CASCADE;
END IF;
END $$;

53
src/main/resources/db/migrations/20260101000600_add_identifiers.sql Normal file
View File

@ -0,0 +1,53 @@
ALTER TABLE entity_definitions
ADD COLUMN IF NOT EXISTS identifier VARCHAR(100) NOT NULL DEFAULT '';
ALTER TABLE entity_fields
ADD COLUMN IF NOT EXISTS identifier VARCHAR(100) NOT NULL DEFAULT '';
WITH identifiers AS (
SELECT
id,
REGEXP_REPLACE(
INITCAP(
REGEXP_REPLACE(
REGEXP_REPLACE(
TRANSLATE(name, 'ÀÁÂÃÄÅàáâãäåÇçÈÉÊËèéêëÌÍÎÏìíîïÑñÒÓÔÕÖòóôõöÙÚÛÜùúûüÝŸýÿ',
'AAAAAAaaaaaaCcEEEEeeeeIIIIiiiiNnOOOOOoooooUUUUuuuuYYyy'),
'([a-z0-9])([A-Z])', '\1 \2', 'g'
),
'[^a-zA-Z0-9]+', ' ', 'g'
)
),
'\s+', '', 'g'
) AS value
FROM entity_definitions
WHERE identifier = ''
)
UPDATE entity_definitions
SET identifier = LOWER(LEFT(identifiers.value, 1)) || SUBSTRING(identifiers.value FROM 2)
FROM identifiers
WHERE entity_definitions.id = identifiers.id;
WITH identifiers AS (
SELECT
id,
REGEXP_REPLACE(
INITCAP(
REGEXP_REPLACE(
REGEXP_REPLACE(
TRANSLATE(name, 'ÀÁÂÃÄÅàáâãäåÇçÈÉÊËèéêëÌÍÎÏìíîïÑñÒÓÔÕÖòóôõöÙÚÛÜùúûüÝŸýÿ',
'AAAAAAaaaaaaCcEEEEeeeeIIIIiiiiNnOOOOOoooooUUUUuuuuYYyy'),
'([a-z0-9])([A-Z])', '\1 \2', 'g'
),
'[^a-zA-Z0-9]+', ' ', 'g'
)
),
'\s+', '', 'g'
) AS value
FROM entity_fields
WHERE identifier = ''
)
UPDATE entity_fields
SET identifier = LOWER(LEFT(identifiers.value, 1)) || SUBSTRING(identifiers.value FROM 2)
FROM identifiers
WHERE entity_fields.id = identifiers.id;

5
src/main/resources/db/reset.sql Normal file
View File

@ -0,0 +1,5 @@
DROP TABLE IF EXISTS entity_fields;
DROP TABLE IF EXISTS entity_definitions;
DROP TABLE IF EXISTS sessions;
DROP TABLE IF EXISTS users;
DROP TABLE IF EXISTS schema_migrations;

54
src/test/kotlin/dev/mduchene/bolts/persistence/MigrationRunnerTest.kt Normal file
View File

@ -0,0 +1,54 @@
package dev.mduchene.bolts.persistence
import java.sql.DriverManager
import java.util.UUID
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class MigrationRunnerTest {
@Test
fun `discovers migrations in timestamp order`() {
val migrations = MigrationRunner().migrations()
assertTrue(migrations.isNotEmpty())
assertEquals(migrations.map { it.version }.sorted(), migrations.map { it.version })
assertEquals(migrations.size, migrations.map { it.version }.distinct().size)
assertEquals("20260101000000_create_users.sql", migrations.first().filename)
}
@Test
fun `runs each migration only once`() {
val schema = "migration_test_${UUID.randomUUID().toString().replace("-", "")}"
DriverManager.getConnection(
environmentOrDefault("TEST_DB_URL", "jdbc:postgresql://localhost:5432/postgres"),
environmentOrDefault("TEST_DB_USERNAME", "root"),
environmentOrDefault("TEST_DB_PASSWORD", "root"),
).use { connection ->
try {
connection.createStatement().use {
it.execute("CREATE SCHEMA $schema")
it.execute("SET search_path TO $schema")
}
val runner = MigrationRunner()
runner.migrate(connection)
runner.migrate(connection)
connection.createStatement().use { statement ->
statement.executeQuery("SELECT COUNT(*) FROM schema_migrations").use { result ->
result.next()
assertEquals(runner.migrations().size, result.getInt(1))
}
}
} finally {
connection.createStatement().use {
it.execute("DROP SCHEMA IF EXISTS $schema CASCADE")
}
}
}
}
private fun environmentOrDefault(name: String, defaultValue: String): String =
System.getenv(name)?.takeIf { it.isNotBlank() } ?: defaultValue
}