fix(db): tolerate MySQL HeatWave NOT SECONDARY columns in migration (#3452)

Atlas aborts schema inspection on the HeatWave-specific EXTRA attribute
"NOT SECONDARY". Before Schema.Create, replay each flagged column's own
definition from SHOW CREATE TABLE minus the attribute via ALTER TABLE
MODIFY COLUMN, clearing it without touching the logical schema. Covers
both startup migration paths and the v3-to-v4 migrator; a no-op on
non-MySQL backends.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

application/migrator/migrator.go

@@ -181,6 +181,7 @@ func (m *Migrator) Migrate() error {
 	// Continue from the current step
 	if m.state.Step <= StepSchema {
 		m.l.Info("Creating basic v4 table schema...")
+		inventory.RepairHeatWaveNotSecondary(m.l, m.v4client, context.Background())
 		if err := m.v4client.Schema.Create(context.Background()); err != nil {
 			return fmt.Errorf("failed creating schema resources: %w", err)
 		}

inventory/client.go

@@ -39,6 +39,7 @@ func InitializeDBClient(l logging.Logger,
 	} else {
 		// Version is current, but additive schema changes (new columns)
 		// still need to reach existing databases between releases.
+		RepairHeatWaveNotSecondary(l, client, ctx)
 		if err := client.Schema.Create(ctx); err != nil {
 			return nil, fmt.Errorf("failed to apply additive schema changes: %w", err)
 		}

inventory/heatwave_test.go

@@ -0,0 +1,43 @@
+package inventory
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+const heatWaveDDL = "CREATE TABLE `groups` (\n" +
+	"  `id` bigint NOT NULL AUTO_INCREMENT,\n" +
+	"  `created_at` datetime(3) NULL,\n" +
+	"  `name` varchar(255) NOT NULL,\n" +
+	"  `permissions` blob NOT NULL COMMENT 'json blob' NOT SECONDARY,\n" +
+	"  `note` varchar(255) NULL DEFAULT 'not secondary',\n" +
+	"  PRIMARY KEY (`id`),\n" +
+	"  UNIQUE KEY `groups_name` (`name`),\n" +
+	"  KEY `groups_created_at` (`created_at`)\n" +
+	") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4"
+
+func TestExtractColumnDef(t *testing.T) {
+	t.Run("strips NOT SECONDARY keeping the rest", func(t *testing.T) {
+		def, err := extractColumnDef(heatWaveDDL, "permissions")
+		require.NoError(t, err)
+		require.Equal(t, "`permissions` blob NOT NULL COMMENT 'json blob'", def)
+	})
+
+	t.Run("leaves unrelated columns intact", func(t *testing.T) {
+		def, err := extractColumnDef(heatWaveDDL, "note")
+		require.NoError(t, err)
+		// The string 'not secondary' inside a DEFAULT literal must survive.
+		require.Equal(t, "`note` varchar(255) NULL DEFAULT 'not secondary'", def)
+	})
+
+	t.Run("index lines are not matched", func(t *testing.T) {
+		_, err := extractColumnDef(heatWaveDDL, "groups_name")
+		require.Error(t, err)
+	})
+
+	t.Run("missing column errors", func(t *testing.T) {
+		_, err := extractColumnDef(heatWaveDDL, "nope")
+		require.Error(t, err)
+	})
+}

inventory/migration.go

@@ -42,6 +42,7 @@ func migrate(l logging.Logger, client *ent.Client, ctx context.Context, kv cache
 	l.Info("Start initializing database schema...")
 	l.Info("Creating basic table schema...")
 	repairLegacyUserGroupColumn(l, client, ctx)
+	RepairHeatWaveNotSecondary(l, client, ctx)
 	if err := client.Schema.Create(ctx); err != nil {
 		return fmt.Errorf("Failed creating schema resources: %w", err)
 	}
@@ -94,6 +95,128 @@ func repairLegacyUserGroupColumn(l logging.Logger, client *ent.Client, ctx conte
 	}
 }
 
+// RepairHeatWaveNotSecondary strips the MySQL HeatWave `NOT SECONDARY` column
+// attribute before ent's schema migration. Atlas treats it as an unknown EXTRA
+// value and aborts schema inspection (upstream #3452). For every flagged column
+// we replay its own definition from SHOW CREATE TABLE minus the attribute via
+// ALTER TABLE ... MODIFY COLUMN, which clears it without touching the logical
+// schema. On non-MySQL backends the INFORMATION_SCHEMA query fails and the whole
+// function is a no-op.
+func RepairHeatWaveNotSecondary(l logging.Logger, client *ent.Client, ctx context.Context) {
+	rows, err := client.QueryContext(ctx,
+		`SELECT TABLE_NAME, COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS
+		 WHERE TABLE_SCHEMA = DATABASE() AND UPPER(EXTRA) LIKE '%NOT SECONDARY%'`)
+	if err != nil {
+		l.Debug("Skip HeatWave NOT SECONDARY repair: %s", err)
+		return
+	}
+
+	var flagged [][2]string
+	for rows.Next() {
+		var table, column string
+		if err := rows.Scan(&table, &column); err != nil {
+			l.Debug("Skip HeatWave NOT SECONDARY repair scan: %s", err)
+			rows.Close()
+			return
+		}
+		flagged = append(flagged, [2]string{table, column})
+	}
+	rows.Close()
+
+	for _, tc := range flagged {
+		def, err := heatWaveColumnDef(client, ctx, tc[0], tc[1])
+		if err != nil {
+			l.Warning("Skip NOT SECONDARY repair for %s.%s: %s", tc[0], tc[1], err)
+			continue
+		}
+
+		stm := fmt.Sprintf("ALTER TABLE `%s` MODIFY COLUMN %s",
+			strings.ReplaceAll(tc[0], "`", "``"), def)
+		if _, err := client.ExecContext(ctx, stm); err != nil {
+			l.Warning("Failed to strip NOT SECONDARY on %s.%s: %s", tc[0], tc[1], err)
+			continue
+		}
+		l.Info("Stripped HeatWave NOT SECONDARY attribute on %s.%s", tc[0], tc[1])
+	}
+}
+
+// heatWaveColumnDef extracts a column's full definition from SHOW CREATE TABLE
+// and returns it without the `NOT SECONDARY` attribute, ready for use after
+// ALTER TABLE ... MODIFY COLUMN.
+func heatWaveColumnDef(client *ent.Client, ctx context.Context, table, column string) (string, error) {
+	rows, err := client.QueryContext(ctx,
+		fmt.Sprintf("SHOW CREATE TABLE `%s`", strings.ReplaceAll(table, "`", "``")))
+	if err != nil {
+		return "", err
+	}
+	defer rows.Close()
+
+	var name, ddl string
+	if !rows.Next() {
+		return "", fmt.Errorf("table %s not found", table)
+	}
+	if err := rows.Scan(&name, &ddl); err != nil {
+		return "", err
+	}
+
+	return extractColumnDef(ddl, column)
+}
+
+// extractColumnDef finds `column`'s definition line in SHOW CREATE TABLE output
+// and returns it without the `NOT SECONDARY` attribute.
+func extractColumnDef(ddl, column string) (string, error) {
+	quoted := "`" + strings.ReplaceAll(column, "`", "``") + "`"
+	for _, line := range strings.Split(ddl, "\n") {
+		line = strings.TrimSpace(line)
+		if !strings.HasPrefix(line, quoted+" ") {
+			continue
+		}
+		def := stripNotSecondary(strings.TrimSuffix(line, ","))
+		if findNotSecondary(def) >= 0 {
+			return "", fmt.Errorf("failed to strip NOT SECONDARY from column %s", column)
+		}
+		return def, nil
+	}
+	return "", fmt.Errorf("column %s not found in table definition", column)
+}
+
+func stripNotSecondary(def string) string {
+	// Remove the `NOT SECONDARY` attribute clause, but not the same words
+	// inside a quoted literal (e.g. DEFAULT 'not secondary').
+	out := def
+	for {
+		idx := findNotSecondary(out)
+		if idx < 0 {
+			return strings.TrimSpace(out)
+		}
+		out = out[:idx] + out[idx+len("NOT SECONDARY"):]
+	}
+}
+
+// findNotSecondary returns the index of the first `NOT SECONDARY` occurrence
+// outside single-quoted string literals, or -1.
+func findNotSecondary(def string) int {
+	const marker = "NOT SECONDARY"
+	upper := strings.ToUpper(def)
+	inQuote := false
+	for i := 0; i+len(marker) <= len(upper); i++ {
+		if upper[i] == '\'' {
+			if inQuote && i+1 < len(upper) && upper[i+1] == '\'' {
+				i++ // escaped '' inside a literal
+				continue
+			}
+			inQuote = !inQuote
+		}
+		if !inQuote && strings.HasPrefix(upper[i:], marker) {
+			end := i + len(marker)
+			if (i == 0 || upper[i-1] == ' ') && (end == len(upper) || upper[end] == ' ' || upper[end] == ',') {
+				return i
+			}
+		}
+	}
+	return -1
+}
+
 func migrateDefaultSettings(l logging.Logger, client *ent.Client, ctx context.Context, kv cache.Driver) {
 	// clean kv cache
 	if err := kv.DeleteAll(); err != nil {