diff --git a/frontend/src/pages/entity-configuration.vue b/frontend/src/pages/entity-configuration.vue index e14a4d8..2704b67 100644 --- a/frontend/src/pages/entity-configuration.vue +++ b/frontend/src/pages/entity-configuration.vue @@ -31,6 +31,20 @@ fields: EntityField[] } + interface MigrationValidation { + valid: boolean + errors: string[] + warnings: string[] + actions: string[] + } + + interface MigrationResult { + success: boolean + message: string + actions: string[] + errors: string[] + } + const entities = ref([]) const loading = ref(true) const saving = ref(false) @@ -47,6 +61,10 @@ const editingFieldType = ref('TEXT') const editingTargetEntityId = ref() const editingRelationshipType = ref('ONE_TO_ONE') + const validatingMigration = ref(false) + const runningMigration = ref(false) + const migrationValidation = ref() + const migrationResult = ref() function request (url: string, method = 'GET', values?: Record) { return fetch(url, { @@ -223,6 +241,8 @@ errorMessage.value = '' try { await action() + migrationValidation.value = undefined + migrationResult.value = undefined } catch { errorMessage.value = 'The change could not be saved. Please try again.' } finally { @@ -230,6 +250,45 @@ } } + async function validateMigration () { + validatingMigration.value = true + migrationResult.value = undefined + errorMessage.value = '' + try { + const response = await request('/api/entity-definitions/migration/validate', 'POST') + if (!response.ok) throw new Error('Migration validation failed') + migrationValidation.value = await response.json() as MigrationValidation + } catch { + errorMessage.value = 'Unable to validate the entity migration.' + } finally { + validatingMigration.value = false + } + } + + async function runMigration () { + if (!migrationValidation.value?.valid) return + runningMigration.value = true + migrationResult.value = undefined + errorMessage.value = '' + try { + const response = await request('/api/entity-definitions/migration/run', 'POST') + const result = await response.json() as MigrationResult + migrationResult.value = result + if (!response.ok && result.errors.length === 0) { + result.errors = ['The migration could not be completed.'] + } + } catch { + migrationResult.value = { + success: false, + message: 'Entity migration failed. No database changes were kept.', + actions: [], + errors: ['The server did not return migration details.'], + } + } finally { + runningMigration.value = false + } + } + function typeLabel (type: FieldType) { if (type === 'RELATIONSHIP') return 'Relationship' if (type === 'USER') return 'User reference' @@ -254,7 +313,7 @@

Administration

Entity Configuration

-

Define the entities and fields your CRM will use. This configuration does not create database tables yet.

+

Define the entities and fields your CRM will use, then validate and apply the database changes.

@@ -276,6 +335,74 @@
+ +
+
+

Database migration

+

Validate the configuration to preview the tables, fields, and relationships that will be created.

+
+ +
+ + + +
+ +
+ {{ migrationValidation.valid ? 'Validation passed' : 'Validation failed' }} + +
    +
  • {{ item }}
    • +
+ +
    +
  • {{ item }}
    • +
+ + + +

The database is already up to date.

+
+ + +
+

Loading configuration…

No entities yet. Add your first entity above.

@@ -437,6 +564,17 @@ .add-field { padding: 1rem 1.25rem 1.25rem; border-top: 1px solid var(--v0-divider); background: color-mix(in srgb, var(--v0-surface-variant) 35%, transparent); } .field-inputs input { flex: 1; } .error { color: var(--v0-error); } + .migration-panel { margin-bottom: 2rem; padding: 1.25rem; border: 1px solid var(--v0-divider); border-radius: 1rem; background: var(--v0-surface); } + .migration-panel h2 { margin: 0; font-size: 1.15rem; } + .migration-panel > div > p { margin: 0.35rem 0 0; color: var(--v0-on-surface-variant); } + .migration-actions { display: flex; gap: 0.65rem; margin-top: 1rem; } + .migration-report { margin-top: 1rem; padding: 1rem; border-radius: 0.7rem; line-height: 1.5; } + .migration-report.valid { border: 1px solid #2e7d32; background: color-mix(in srgb, #2e7d32 10%, transparent); } + .migration-report.invalid { border: 1px solid var(--v0-error); background: color-mix(in srgb, var(--v0-error) 9%, transparent); } + .migration-report ul, .migration-report ol { margin: 0.55rem 0 0; padding-left: 1.3rem; } + .migration-report > p { margin: 0.65rem 0 0; font-weight: 600; } + .error-list { color: var(--v0-error); } + .warning-list { color: var(--v0-on-surface-variant); } .status { padding: 1.5rem; border: 1px dashed var(--v0-divider); border-radius: 0.8rem; color: var(--v0-on-surface-variant); } .sr-only { position: absolute; width: 1px; height: 1px; padding: 0; overflow: hidden; clip: rect(0, 0, 0, 0); white-space: nowrap; border: 0; } @media (max-width: 700px) { @@ -444,5 +582,6 @@ .form-row, .field-inputs, .edit-row, .edit-field { align-items: stretch; flex-direction: column; } .entity-heading, .field-row { padding-block: 1rem; } input, select, .primary-button, .secondary-button { width: 100%; } + .migration-actions { align-items: stretch; flex-direction: column; } } diff --git a/frontend/tests/entity-configuration.spec.ts b/frontend/tests/entity-configuration.spec.ts index e1ffb4f..c791d7c 100644 --- a/frontend/tests/entity-configuration.spec.ts +++ b/frontend/tests/entity-configuration.spec.ts @@ -168,3 +168,45 @@ test('admin can add, edit, and remove an entity and its fields', async ({ page } await contact.getByRole('button', { name: 'Remove entity' }).click() await expect(page.getByText('No entities yet.')).toBeVisible() }) + +test('admin validates before running a migration and sees migration failures', async ({ page }) => { + await useAdminSession(page) + await page.route('**/api/entity-definitions', route => route.fulfill({ + json: [{ + id: 1, + name: 'Company', + identifier: 'company', + fields: [{ id: 1, name: 'Name', identifier: 'name', type: 'TEXT' }], + }], + })) + await page.route('**/api/entity-definitions/migration/validate', route => route.fulfill({ + json: { + valid: true, + errors: [], + warnings: ['Table company already exists; missing fields will be added with ALTER TABLE.'], + actions: ['Add company.name (TEXT).'], + }, + })) + await page.route('**/api/entity-definitions/migration/run', route => route.fulfill({ + status: 400, + json: { + success: false, + message: 'Entity migration failed and was rolled back.', + actions: [], + errors: ['permission denied for table company'], + }, + })) + + await page.goto('/admin/entities') + const runButton = page.getByRole('button', { name: 'Run migration' }) + await expect(runButton).toBeDisabled() + + await page.getByRole('button', { name: 'Validate migration' }).click() + await expect(page.getByText('Validation passed')).toBeVisible() + await expect(page.getByText('Add company.name (TEXT).')).toBeVisible() + await expect(runButton).toBeEnabled() + + await runButton.click() + await expect(page.getByText('Entity migration failed and was rolled back.')).toBeVisible() + await expect(page.getByText('permission denied for table company')).toBeVisible() +}) diff --git a/src/main/kotlin/Main.kt b/src/main/kotlin/Main.kt index 9a71a90..ad6103a 100644 --- a/src/main/kotlin/Main.kt +++ b/src/main/kotlin/Main.kt @@ -1,4 +1,5 @@ import dev.mduchene.bolts.entity.EntityDefinitionRepository +import dev.mduchene.bolts.entity.EntityMigrationService import dev.mduchene.bolts.persistence.Database import dev.mduchene.bolts.persistence.DatabaseConfig import dev.mduchene.bolts.user.LoginService @@ -19,11 +20,12 @@ fun main() { val loginService = LoginService(users, SessionRepository(database)) loginService.initializeAdmin() + val entities = EntityDefinitionRepository(database) val controllers = listOf( HomeController(), AuthController(loginService), UserController(users, loginService), - EntityDefinitionController(EntityDefinitionRepository(database), loginService), + EntityDefinitionController(entities, loginService, EntityMigrationService(database, entities)), ) Javalin.create { config -> diff --git a/src/main/kotlin/dev/mduchene/bolts/entity/EntityMigrationService.kt b/src/main/kotlin/dev/mduchene/bolts/entity/EntityMigrationService.kt new file mode 100644 index 0000000..7c218ac --- /dev/null +++ b/src/main/kotlin/dev/mduchene/bolts/entity/EntityMigrationService.kt @@ -0,0 +1,293 @@ +package dev.mduchene.bolts.entity + +import dev.mduchene.bolts.persistence.Database +import java.sql.Connection + +data class EntityMigrationValidation( + val valid: Boolean, + val errors: List, + val warnings: List, + val actions: List, +) + +data class EntityMigrationResult( + val success: Boolean, + val message: String, + val actions: List, + val errors: List = emptyList(), +) + +class EntityMigrationService( + private val database: Database, + private val entities: EntityDefinitionRepository, +) { + fun validate(): EntityMigrationValidation = + database.getConnection().use { connection -> plan(connection, entities.findAll()).validation } + + fun migrate(): EntityMigrationResult = + try { + database.transaction { connection -> + val plan = plan(connection, loadEntities(connection)) + if (!plan.validation.valid) { + return@transaction EntityMigrationResult( + success = false, + message = "Validation failed; no database changes were made.", + actions = plan.validation.actions, + errors = plan.validation.errors, + ) + } + + connection.createStatement().use { statement -> + plan.statements.forEach { statement.execute(it.sql) } + } + EntityMigrationResult( + success = true, + message = "Entity migration completed successfully.", + actions = plan.validation.actions, + ) + } + } catch (exception: Exception) { + EntityMigrationResult( + success = false, + message = "Entity migration failed and was rolled back.", + actions = emptyList(), + errors = listOf(exception.message ?: "Database migration failed"), + ) + } + + private fun plan(connection: Connection, definitions: List): MigrationPlan { + val errors = mutableListOf() + val warnings = mutableListOf() + val statements = mutableListOf() + val identifierPattern = Regex("[A-Za-z_][A-Za-z0-9_]*") + val reservedTables = setOf("entity_definitions", "entity_fields", "schema_migrations", "sessions", "users") + val duplicateEntities = definitions.groupBy { it.identifier.lowercase() }.filterValues { it.size > 1 } + + if (definitions.isEmpty()) warnings += "No entity definitions are configured." + duplicateEntities.values.forEach { + errors += "Entities ${it.joinToString { entity -> "“${entity.name}”" }} use the same identifier." + } + + definitions.forEach { entity -> + if (!identifierPattern.matches(entity.identifier)) { + errors += "Entity “${entity.name}” has an invalid SQL identifier: ${entity.identifier}." + } + if (entity.identifier.lowercase() in reservedTables) { + errors += "Entity “${entity.name}” uses the reserved application table ${entity.identifier}." + } + entity.fields.groupBy { it.identifier.lowercase() }.filterValues { it.size > 1 }.values.forEach { + errors += "Entity “${entity.name}” has duplicate field identifier ${it.first().identifier}." + } + entity.fields.forEach { field -> + if (!identifierPattern.matches(field.identifier)) { + errors += "Field “${entity.name}.${field.name}” has an invalid SQL identifier: ${field.identifier}." + } + if (field.identifier.equals("id", ignoreCase = true)) { + errors += "Field “${entity.name}.${field.name}” cannot use the reserved identifier id." + } + if (field.type == FieldType.RELATIONSHIP) { + if (field.targetEntityId == null || definitions.none { it.id == field.targetEntityId }) { + errors += "Relationship “${entity.name}.${field.name}” has no valid target entity." + } + if (field.relationshipType == null) { + errors += "Relationship “${entity.name}.${field.name}” has no relationship type." + } + } + } + } + + if (errors.isNotEmpty()) return migrationPlan(errors, warnings, statements) + + val existingTables = tableNames(connection) + val ordered = dependencyOrder(definitions) + + ordered.forEach { entity -> + if (entity.identifier.lowercase() !in existingTables) { + statements += MigrationStatement( + """CREATE TABLE ${quote(entity.identifier)} ("id" BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY)""", + "Create table ${entity.identifier}.", + ) + } else { + warnings += "Table ${entity.identifier} already exists; missing fields will be added with ALTER TABLE." + val idType = columnTypes(connection, entity.identifier)["id"] + if (idType == null) errors += "Existing table ${entity.identifier} has no id column." + else if (idType !in setOf("bigint", "integer", "smallint")) { + errors += "Existing table ${entity.identifier} has an incompatible id column ($idType)." + } + } + } + + definitions.forEach { entity -> + val columns = if (entity.identifier.lowercase() in existingTables) { + columnTypes(connection, entity.identifier) + } else { + emptyMap() + } + entity.fields.forEach { field -> + val expected = sqlType(field.type) + val actual = columns[field.identifier.lowercase()] + if (actual == null) { + val table = existingTables[entity.identifier.lowercase()] ?: entity.identifier + statements += MigrationStatement( + "ALTER TABLE ${quote(table)} ADD COLUMN ${quote(field.identifier)} $expected", + "Add ${entity.identifier}.${field.identifier} ($expected).", + ) + } else if (!compatible(field.type, actual)) { + errors += "Existing column ${entity.identifier}.${field.identifier} is $actual, but ${field.type} requires $expected." + } else { + warnings += "Column ${entity.identifier}.${field.identifier} already exists and will be left unchanged." + } + } + } + + definitions.forEach { entity -> + entity.fields.filter { it.type == FieldType.RELATIONSHIP || it.type == FieldType.USER }.forEach { field -> + val target = if (field.type == FieldType.USER) "users" else { + definitions.first { it.id == field.targetEntityId }.identifier.let { + existingTables[it.lowercase()] ?: it + } + } + val table = existingTables[entity.identifier.lowercase()] ?: entity.identifier + val constraint = "em_fk_${entity.id}_${field.id}" + if (!constraintExists(connection, constraint)) { + statements += MigrationStatement( + "ALTER TABLE ${quote(table)} ADD CONSTRAINT ${quote(constraint)} " + + "FOREIGN KEY (${quote(field.identifier)}) REFERENCES ${quote(target)} (\"id\")", + "Add reference ${entity.identifier}.${field.identifier} → $target.id.", + ) + } + if (field.relationshipType == RelationshipType.ONE_TO_ONE) { + val unique = "em_uq_${entity.id}_${field.id}" + if (!constraintExists(connection, unique)) { + statements += MigrationStatement( + "ALTER TABLE ${quote(table)} ADD CONSTRAINT ${quote(unique)} " + + "UNIQUE (${quote(field.identifier)})", + "Make ${entity.identifier}.${field.identifier} one-to-one.", + ) + } + } + } + } + + return migrationPlan(errors, warnings, statements) + } + + private fun loadEntities(connection: Connection): List { + val fields = mutableMapOf>() + connection.prepareStatement( + """ + SELECT id, entity_id, name, identifier, field_type, target_entity_id, relationship_type + FROM entity_fields ORDER BY id + """.trimIndent(), + ).use { statement -> + statement.executeQuery().use { result -> + while (result.next()) { + val entityId = result.getLong("entity_id") + fields.getOrPut(entityId, ::mutableListOf) += EntityField( + id = result.getLong("id"), + name = result.getString("name"), + identifier = result.getString("identifier"), + type = FieldType.valueOf(result.getString("field_type")), + targetEntityId = result.getLong("target_entity_id").takeUnless { result.wasNull() }, + relationshipType = result.getString("relationship_type")?.let(RelationshipType::valueOf), + ) + } + } + } + return connection.prepareStatement("SELECT id, name, identifier FROM entity_definitions ORDER BY id").use { statement -> + statement.executeQuery().use { result -> + buildList { + while (result.next()) { + val id = result.getLong("id") + add(EntityDefinition(id, result.getString("name"), result.getString("identifier"), fields[id].orEmpty())) + } + } + } + } + } + + private fun dependencyOrder(definitions: List): List { + val byId = definitions.associateBy(EntityDefinition::id) + val visited = mutableSetOf() + val visiting = mutableSetOf() + val result = mutableListOf() + fun visit(entity: EntityDefinition) { + if (entity.id in visited || !visiting.add(entity.id)) return + entity.fields.mapNotNull { it.targetEntityId }.mapNotNull(byId::get).forEach(::visit) + visiting.remove(entity.id) + visited += entity.id + result += entity + } + definitions.forEach(::visit) + return result + } + + private fun tableNames(connection: Connection): Map = + connection.prepareStatement( + "SELECT table_name FROM information_schema.tables WHERE table_schema = current_schema()", + ).use { statement -> + statement.executeQuery().use { result -> + buildMap { + while (result.next()) { + val name = result.getString(1) + put(name.lowercase(), name) + } + } + } + } + + private fun columnTypes(connection: Connection, table: String): Map = + connection.prepareStatement( + """ + SELECT column_name, data_type FROM information_schema.columns + WHERE table_schema = current_schema() AND lower(table_name) = lower(?) + """.trimIndent(), + ).use { statement -> + statement.setString(1, table) + statement.executeQuery().use { result -> + buildMap { while (result.next()) put(result.getString(1).lowercase(), result.getString(2).lowercase()) } + } + } + + private fun constraintExists(connection: Connection, name: String): Boolean = + connection.prepareStatement( + "SELECT 1 FROM information_schema.table_constraints WHERE constraint_schema = current_schema() AND constraint_name = ?", + ).use { statement -> + statement.setString(1, name) + statement.executeQuery().use { it.next() } + } + + private fun sqlType(type: FieldType): String = when (type) { + FieldType.TEXT -> "TEXT" + FieldType.NUMBER -> "NUMERIC" + FieldType.BOOLEAN -> "BOOLEAN" + FieldType.DATE -> "DATE" + FieldType.EMAIL, FieldType.PHONE -> "VARCHAR(255)" + FieldType.RELATIONSHIP, FieldType.USER -> "BIGINT" + } + + private fun compatible(type: FieldType, actual: String): Boolean = when (type) { + FieldType.TEXT, FieldType.EMAIL, FieldType.PHONE -> actual in setOf("text", "character varying", "character") + FieldType.NUMBER -> actual in setOf("numeric", "decimal", "real", "double precision", "bigint", "integer", "smallint") + FieldType.BOOLEAN -> actual == "boolean" + FieldType.DATE -> actual == "date" + FieldType.RELATIONSHIP, FieldType.USER -> actual in setOf("bigint", "integer", "smallint") + } + + private fun quote(identifier: String) = "\"${identifier.replace("\"", "\"\"")}\"" + + private fun migrationPlan( + errors: List, + warnings: List, + statements: List, + ) = MigrationPlan( + EntityMigrationValidation(errors.isEmpty(), errors, warnings, statements.map(MigrationStatement::description)), + statements, + ) + + private data class MigrationStatement(val sql: String, val description: String) + private data class MigrationPlan( + val validation: EntityMigrationValidation, + val statements: List, + ) +} diff --git a/src/main/kotlin/dev/mduchene/bolts/persistence/Database.kt b/src/main/kotlin/dev/mduchene/bolts/persistence/Database.kt index ec37bff..31dc1ff 100644 --- a/src/main/kotlin/dev/mduchene/bolts/persistence/Database.kt +++ b/src/main/kotlin/dev/mduchene/bolts/persistence/Database.kt @@ -45,6 +45,17 @@ class Database( } } + fun transaction(action: (Connection) -> T): T = + getConnection().use { connection -> + connection.autoCommit = false + try { + action(connection).also { connection.commit() } + } catch (exception: Exception) { + connection.rollback() + throw exception + } + } + private fun query( sql: String, bind: PreparedStatement.() -> Unit, diff --git a/src/main/kotlin/dev/mduchene/bolts/web/EntityDefinitionController.kt b/src/main/kotlin/dev/mduchene/bolts/web/EntityDefinitionController.kt index 518ecfc..042c3d2 100644 --- a/src/main/kotlin/dev/mduchene/bolts/web/EntityDefinitionController.kt +++ b/src/main/kotlin/dev/mduchene/bolts/web/EntityDefinitionController.kt @@ -1,6 +1,7 @@ package dev.mduchene.bolts.web import dev.mduchene.bolts.entity.EntityDefinitionRepository +import dev.mduchene.bolts.entity.EntityMigrationService import dev.mduchene.bolts.entity.FieldType import dev.mduchene.bolts.entity.RelationshipType import dev.mduchene.bolts.user.LoginService @@ -11,12 +12,23 @@ import io.javalin.router.JavalinDefaultRoutingApi class EntityDefinitionController( private val entities: EntityDefinitionRepository, private val loginService: LoginService, + private val migrations: EntityMigrationService, ) : Controller { override fun register(routes: JavalinDefaultRoutingApi) { routes.get("/api/entity-definitions") { ctx -> if (!ctx.requireAdmin(loginService)) return@get ctx.json(entities.findAll()) } + routes.post("/api/entity-definitions/migration/validate") { ctx -> + if (!ctx.requireAdmin(loginService)) return@post + ctx.json(migrations.validate()) + } + routes.post("/api/entity-definitions/migration/run") { ctx -> + if (!ctx.requireAdmin(loginService)) return@post + val result = migrations.migrate() + if (!result.success) ctx.status(HttpStatus.BAD_REQUEST) + ctx.json(result) + } routes.post("/api/entity-definitions") { ctx -> if (!ctx.requireAdmin(loginService)) return@post val name = ctx.requiredFormParam("name") ?: return@post