From a9ed9ade1f768850c947a037a9d63ca5eb96fe53 Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Sat, 19 Sep 2026 01:17:10 +0200 Subject: [PATCH] fix(db): tolerate MySQL HeatWave NOT SECONDARY columns in migration (#3452) Atlas aborts schema inspection on the HeatWave-specific EXTRA attribute "NOT SECONDARY". Before Schema.Create, replay each flagged column's own definition from SHOW CREATE TABLE minus the attribute via ALTER TABLE MODIFY COLUMN, clearing it without touching the logical schema. Covers both startup migration paths and the v3-to-v4 migrator; a no-op on non-MySQL backends. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- application/migrator/migrator.go | 1 + inventory/client.go | 1 + inventory/heatwave_test.go | 43 +++++++++++ inventory/migration.go | 123 +++++++++++++++++++++++++++++++ 4 files changed, 168 insertions(+) create mode 100644 inventory/heatwave_test.go diff --git a/application/migrator/migrator.go b/application/migrator/migrator.go index 5bb53520..3e9806f9 100644 --- a/application/migrator/migrator.go +++ b/application/migrator/migrator.go @@ -181,6 +181,7 @@ func (m *Migrator) Migrate() error { // Continue from the current step if m.state.Step <= StepSchema { m.l.Info("Creating basic v4 table schema...") + inventory.RepairHeatWaveNotSecondary(m.l, m.v4client, context.Background()) if err := m.v4client.Schema.Create(context.Background()); err != nil { return fmt.Errorf("failed creating schema resources: %w", err) } diff --git a/inventory/client.go b/inventory/client.go index af65a6fb..f339d94a 100644 --- a/inventory/client.go +++ b/inventory/client.go @@ -39,6 +39,7 @@ func InitializeDBClient(l logging.Logger, } else { // Version is current, but additive schema changes (new columns) // still need to reach existing databases between releases. + RepairHeatWaveNotSecondary(l, client, ctx) if err := client.Schema.Create(ctx); err != nil { return nil, fmt.Errorf("failed to apply additive schema changes: %w", err) } diff --git a/inventory/heatwave_test.go b/inventory/heatwave_test.go new file mode 100644 index 00000000..780afe1a --- /dev/null +++ b/inventory/heatwave_test.go @@ -0,0 +1,43 @@ +package inventory + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +const heatWaveDDL = "CREATE TABLE `groups` (\n" + + " `id` bigint NOT NULL AUTO_INCREMENT,\n" + + " `created_at` datetime(3) NULL,\n" + + " `name` varchar(255) NOT NULL,\n" + + " `permissions` blob NOT NULL COMMENT 'json blob' NOT SECONDARY,\n" + + " `note` varchar(255) NULL DEFAULT 'not secondary',\n" + + " PRIMARY KEY (`id`),\n" + + " UNIQUE KEY `groups_name` (`name`),\n" + + " KEY `groups_created_at` (`created_at`)\n" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4" + +func TestExtractColumnDef(t *testing.T) { + t.Run("strips NOT SECONDARY keeping the rest", func(t *testing.T) { + def, err := extractColumnDef(heatWaveDDL, "permissions") + require.NoError(t, err) + require.Equal(t, "`permissions` blob NOT NULL COMMENT 'json blob'", def) + }) + + t.Run("leaves unrelated columns intact", func(t *testing.T) { + def, err := extractColumnDef(heatWaveDDL, "note") + require.NoError(t, err) + // The string 'not secondary' inside a DEFAULT literal must survive. + require.Equal(t, "`note` varchar(255) NULL DEFAULT 'not secondary'", def) + }) + + t.Run("index lines are not matched", func(t *testing.T) { + _, err := extractColumnDef(heatWaveDDL, "groups_name") + require.Error(t, err) + }) + + t.Run("missing column errors", func(t *testing.T) { + _, err := extractColumnDef(heatWaveDDL, "nope") + require.Error(t, err) + }) +} diff --git a/inventory/migration.go b/inventory/migration.go index 838f9315..a00b5883 100644 --- a/inventory/migration.go +++ b/inventory/migration.go @@ -42,6 +42,7 @@ func migrate(l logging.Logger, client *ent.Client, ctx context.Context, kv cache l.Info("Start initializing database schema...") l.Info("Creating basic table schema...") repairLegacyUserGroupColumn(l, client, ctx) + RepairHeatWaveNotSecondary(l, client, ctx) if err := client.Schema.Create(ctx); err != nil { return fmt.Errorf("Failed creating schema resources: %w", err) } @@ -94,6 +95,128 @@ func repairLegacyUserGroupColumn(l logging.Logger, client *ent.Client, ctx conte } } +// RepairHeatWaveNotSecondary strips the MySQL HeatWave `NOT SECONDARY` column +// attribute before ent's schema migration. Atlas treats it as an unknown EXTRA +// value and aborts schema inspection (upstream #3452). For every flagged column +// we replay its own definition from SHOW CREATE TABLE minus the attribute via +// ALTER TABLE ... MODIFY COLUMN, which clears it without touching the logical +// schema. On non-MySQL backends the INFORMATION_SCHEMA query fails and the whole +// function is a no-op. +func RepairHeatWaveNotSecondary(l logging.Logger, client *ent.Client, ctx context.Context) { + rows, err := client.QueryContext(ctx, + `SELECT TABLE_NAME, COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND UPPER(EXTRA) LIKE '%NOT SECONDARY%'`) + if err != nil { + l.Debug("Skip HeatWave NOT SECONDARY repair: %s", err) + return + } + + var flagged [][2]string + for rows.Next() { + var table, column string + if err := rows.Scan(&table, &column); err != nil { + l.Debug("Skip HeatWave NOT SECONDARY repair scan: %s", err) + rows.Close() + return + } + flagged = append(flagged, [2]string{table, column}) + } + rows.Close() + + for _, tc := range flagged { + def, err := heatWaveColumnDef(client, ctx, tc[0], tc[1]) + if err != nil { + l.Warning("Skip NOT SECONDARY repair for %s.%s: %s", tc[0], tc[1], err) + continue + } + + stm := fmt.Sprintf("ALTER TABLE `%s` MODIFY COLUMN %s", + strings.ReplaceAll(tc[0], "`", "``"), def) + if _, err := client.ExecContext(ctx, stm); err != nil { + l.Warning("Failed to strip NOT SECONDARY on %s.%s: %s", tc[0], tc[1], err) + continue + } + l.Info("Stripped HeatWave NOT SECONDARY attribute on %s.%s", tc[0], tc[1]) + } +} + +// heatWaveColumnDef extracts a column's full definition from SHOW CREATE TABLE +// and returns it without the `NOT SECONDARY` attribute, ready for use after +// ALTER TABLE ... MODIFY COLUMN. +func heatWaveColumnDef(client *ent.Client, ctx context.Context, table, column string) (string, error) { + rows, err := client.QueryContext(ctx, + fmt.Sprintf("SHOW CREATE TABLE `%s`", strings.ReplaceAll(table, "`", "``"))) + if err != nil { + return "", err + } + defer rows.Close() + + var name, ddl string + if !rows.Next() { + return "", fmt.Errorf("table %s not found", table) + } + if err := rows.Scan(&name, &ddl); err != nil { + return "", err + } + + return extractColumnDef(ddl, column) +} + +// extractColumnDef finds `column`'s definition line in SHOW CREATE TABLE output +// and returns it without the `NOT SECONDARY` attribute. +func extractColumnDef(ddl, column string) (string, error) { + quoted := "`" + strings.ReplaceAll(column, "`", "``") + "`" + for _, line := range strings.Split(ddl, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, quoted+" ") { + continue + } + def := stripNotSecondary(strings.TrimSuffix(line, ",")) + if findNotSecondary(def) >= 0 { + return "", fmt.Errorf("failed to strip NOT SECONDARY from column %s", column) + } + return def, nil + } + return "", fmt.Errorf("column %s not found in table definition", column) +} + +func stripNotSecondary(def string) string { + // Remove the `NOT SECONDARY` attribute clause, but not the same words + // inside a quoted literal (e.g. DEFAULT 'not secondary'). + out := def + for { + idx := findNotSecondary(out) + if idx < 0 { + return strings.TrimSpace(out) + } + out = out[:idx] + out[idx+len("NOT SECONDARY"):] + } +} + +// findNotSecondary returns the index of the first `NOT SECONDARY` occurrence +// outside single-quoted string literals, or -1. +func findNotSecondary(def string) int { + const marker = "NOT SECONDARY" + upper := strings.ToUpper(def) + inQuote := false + for i := 0; i+len(marker) <= len(upper); i++ { + if upper[i] == '\'' { + if inQuote && i+1 < len(upper) && upper[i+1] == '\'' { + i++ // escaped '' inside a literal + continue + } + inQuote = !inQuote + } + if !inQuote && strings.HasPrefix(upper[i:], marker) { + end := i + len(marker) + if (i == 0 || upper[i-1] == ' ') && (end == len(upper) || upper[end] == ' ' || upper[end] == ',') { + return i + } + } + } + return -1 +} + func migrateDefaultSettings(l logging.Logger, client *ent.Client, ctx context.Context, kv cache.Driver) { // clean kv cache if err := kv.DeleteAll(); err != nil {