From 6c2e10fa8e598d08320469a13568694371fd8d53 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Wed, 12 Feb 2025 12:44:04 +0100 Subject: [PATCH 01/15] First pass at raise_on_warnings --- dev.yml | 23 +++++++++++++++++++++++ go/base/context.go | 1 + go/logic/applier.go | 18 ++++++++++++++++++ go/logic/migrator.go | 8 ++++++++ 4 files changed, 50 insertions(+) create mode 100644 dev.yml diff --git a/dev.yml b/dev.yml new file mode 100644 index 000000000..0d17ed29a --- /dev/null +++ b/dev.yml @@ -0,0 +1,23 @@ +name: gh-ost + +env: + TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: /var/run/docker.sock + TESTCONTAINERS_RYUK_DISABLED: "true" + +up: + - go: + version: "1.22.12" + - podman + - custom: + name: Go Dependencies + met?: go mod download + meet: echo 'go mod failed to download dependencies'; false + - custom: + name: testcontainers-go podman setup + met?: export DOCKER_HOST=unix://$(podman machine inspect --format '{{.ConnectionInfo.PodmanSocket.Path}}') + meet: ":" + +commands: + test: + desc: Run all the tests. + run: script/test \ No newline at end of file diff --git a/go/base/context.go b/go/base/context.go index 8a09c43a8..535ed3a99 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -231,6 +231,7 @@ type MigrationContext struct { ColumnRenameMap map[string]string DroppedColumnsMap map[string]bool MappedSharedColumns *sql.ColumnList + MigrationLastInsertSQLWarnings []string MigrationRangeMinValues *sql.ColumnValues MigrationRangeMaxValues *sql.ColumnValues Iteration int64 diff --git a/go/logic/applier.go b/go/logic/applier.go index 6c8ac71a8..f95f2aa9f 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -753,6 +753,24 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected if err != nil { return nil, err } + + rows, err := tx.Query("SHOW WARNINGS") + if err != nil { + return nil, err + } + + var sqlWarnings []string + for rows.Next() { + var level, message string + var code int + if err := rows.Scan(&level, &code, &message); err != nil { + this.migrationContext.Log.Warningf("Failed to read SHOW WARNIGNS row") + continue + } + sqlWarnings = append(sqlWarnings, fmt.Sprintf("%s: %s (%d)", level, message, code)) + } + this.migrationContext.MigrationLastInsertSQLWarnings = sqlWarnings + if err := tx.Commit(); err != nil { return nil, err } diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 5e4e4eccf..963e5871d 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1265,6 +1265,14 @@ func (this *Migrator) iterateChunks() error { if err != nil { return err // wrapping call will retry } + + // TODO: option that checks for warnings (terminate-on-warnings?) + // TODO: decide if need to check row count discrepancy (are we dropping rows?) + if len(this.migrationContext.MigrationLastInsertSQLWarnings) > 0 { + joinedWarnings := strings.Join(this.migrationContext.MigrationLastInsertSQLWarnings, "; ") + terminateRowIteration(fmt.Errorf("last SQL insert had warnings: %s", joinedWarnings)) + } + atomic.AddInt64(&this.migrationContext.TotalRowsCopied, rowsAffected) atomic.AddInt64(&this.migrationContext.Iteration, 1) return nil From 4af1cb0725dfa4744f81dcc2e1adf17b25464823 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Fri, 21 Feb 2025 15:27:32 +0100 Subject: [PATCH 02/15] Update dev.yml --- dev.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dev.yml b/dev.yml index 0d17ed29a..4c28ff349 100644 --- a/dev.yml +++ b/dev.yml @@ -12,12 +12,10 @@ up: name: Go Dependencies met?: go mod download meet: echo 'go mod failed to download dependencies'; false - - custom: - name: testcontainers-go podman setup - met?: export DOCKER_HOST=unix://$(podman machine inspect --format '{{.ConnectionInfo.PodmanSocket.Path}}') - meet: ":" commands: test: desc: Run all the tests. - run: script/test \ No newline at end of file + run: | + export DOCKER_HOST=unix://$(podman machine inspect --format '{{.ConnectionInfo.PodmanSocket.Path}}') + script/test From 7a4fc8f872a3723cbfd14773d15a77acd8879793 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Wed, 26 Feb 2025 10:16:29 +0100 Subject: [PATCH 03/15] add CLI option, ignore expected warnings, check for row count discrepancy --- go/base/context.go | 1 + go/cmd/gh-ost/main.go | 1 + go/logic/applier.go | 32 +++++++++++++++++++------------- go/logic/migrator.go | 16 +++++++++++----- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 535ed3a99..ac077076f 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -151,6 +151,7 @@ type MigrationContext struct { HooksHintOwner string HooksHintToken string HooksStatusIntervalSec int64 + PanicOnWarnings bool DropServeSocket bool ServeSocketFile string diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 225676364..a1670cdd4 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -108,6 +108,7 @@ func main() { chunkSize := flag.Int64("chunk-size", 1000, "amount of rows to handle in each iteration (allowed range: 10-100,000)") dmlBatchSize := flag.Int64("dml-batch-size", 10, "batch size for DML events to apply in a single transaction (range 1-100)") defaultRetries := flag.Int64("default-retries", 60, "Default number of retries for various operations before panicking") + flag.BoolVar(&migrationContext.PanicOnWarnings, "panic-on-warnings", false, "Panic when SQL warnings are encountered when copying a batch indicating data loss") cutOverLockTimeoutSeconds := flag.Int64("cut-over-lock-timeout-seconds", 3, "Max number of seconds to hold locks on tables while attempting to cut-over (retry attempted when lock exceeds timeout) or attempting instant DDL") niceRatio := flag.Float64("nice-ratio", 0, "force being 'nice', imply sleep time per chunk time; range: [0.0..100.0]. Example values: 0 is aggressive. 1: for every 1ms spent copying rows, sleep additional 1ms (effectively doubling runtime); 0.7: for every 10ms spend in a rowcopy chunk, spend 7ms sleeping immediately after") diff --git a/go/logic/applier.go b/go/logic/applier.go index f95f2aa9f..67a08e8e5 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -754,22 +754,28 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected return nil, err } - rows, err := tx.Query("SHOW WARNINGS") - if err != nil { - return nil, err - } + if this.migrationContext.PanicOnWarnings { + rows, err := tx.Query("SHOW WARNINGS") + if err != nil { + return nil, err + } - var sqlWarnings []string - for rows.Next() { - var level, message string - var code int - if err := rows.Scan(&level, &code, &message); err != nil { - this.migrationContext.Log.Warningf("Failed to read SHOW WARNIGNS row") - continue + var sqlWarnings []string + for rows.Next() { + var level, message string + var code int + if err := rows.Scan(&level, &code, &message); err != nil { + this.migrationContext.Log.Warningf("Failed to read SHOW WARNINGS row") + continue + } + pkDuplicateSuffix := fmt.Sprintf("for key '%s.PRIMARY'", this.migrationContext.GetGhostTableName()) + if strings.HasPrefix(message, "Duplicate entry") && strings.HasSuffix(message, pkDuplicateSuffix) { + continue + } + sqlWarnings = append(sqlWarnings, fmt.Sprintf("%s: %s (%d)", level, message, code)) } - sqlWarnings = append(sqlWarnings, fmt.Sprintf("%s: %s (%d)", level, message, code)) + this.migrationContext.MigrationLastInsertSQLWarnings = sqlWarnings } - this.migrationContext.MigrationLastInsertSQLWarnings = sqlWarnings if err := tx.Commit(); err != nil { return nil, err diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 963e5871d..d89ca922f 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1266,11 +1266,17 @@ func (this *Migrator) iterateChunks() error { return err // wrapping call will retry } - // TODO: option that checks for warnings (terminate-on-warnings?) - // TODO: decide if need to check row count discrepancy (are we dropping rows?) - if len(this.migrationContext.MigrationLastInsertSQLWarnings) > 0 { - joinedWarnings := strings.Join(this.migrationContext.MigrationLastInsertSQLWarnings, "; ") - terminateRowIteration(fmt.Errorf("last SQL insert had warnings: %s", joinedWarnings)) + if this.migrationContext.PanicOnWarnings { + chunkSize := atomic.LoadInt64(&this.migrationContext.ChunkSize) + if len(this.migrationContext.MigrationLastInsertSQLWarnings) > 0 { + for warning := range this.migrationContext.MigrationLastInsertSQLWarnings { + this.migrationContext.Log.Infof("last SQL insert warning: %s", warning) + } + if chunkSize != rowsAffected { + joinedWarnings := strings.Join(this.migrationContext.MigrationLastInsertSQLWarnings, "; ") + terminateRowIteration(fmt.Errorf("last SQL insert had warnings: %s", joinedWarnings)) + } + } } atomic.AddInt64(&this.migrationContext.TotalRowsCopied, rowsAffected) From c6c8f24f3ed5db2f0f32ccaa840eae0997a3f6eb Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Wed, 26 Feb 2025 17:57:40 +0100 Subject: [PATCH 04/15] Count rows in each insert range prepared query - log insert warnings always - terminate if row count doesn't match and the PanicOnWarnings flag is set --- go/logic/applier.go | 22 +++++++++++++--------- go/logic/migrator.go | 12 ++++++------ go/sql/builder.go | 41 ++++++++++++++++++++++++++++------------- 3 files changed, 47 insertions(+), 28 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index 67a08e8e5..15f46f9c2 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -662,7 +662,7 @@ func (this *Applier) ReadMigrationRangeValues() error { // which will be used for copying the next chunk of rows. Ir returns "false" if there is // no further chunk to work through, i.e. we're past the last chunk and are done with // iterating the range (and this done with copying row chunks) -func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange bool, err error) { +func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange bool, expectedRowCount int64, err error) { this.migrationContext.MigrationIterationRangeMinValues = this.migrationContext.MigrationIterationRangeMaxValues if this.migrationContext.MigrationIterationRangeMinValues == nil { this.migrationContext.MigrationIterationRangeMinValues = this.migrationContext.MigrationRangeMinValues @@ -683,32 +683,36 @@ func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange boo fmt.Sprintf("iteration:%d", this.migrationContext.GetIteration()), ) if err != nil { - return hasFurtherRange, err + return hasFurtherRange, expectedRowCount, err } rows, err := this.db.Query(query, explodedArgs...) if err != nil { - return hasFurtherRange, err + return hasFurtherRange, expectedRowCount, err } defer rows.Close() - iterationRangeMaxValues := sql.NewColumnValues(this.migrationContext.UniqueKey.Len()) + iterationRangeMaxValues := sql.NewColumnValues(this.migrationContext.UniqueKey.Len() + 1) for rows.Next() { if err = rows.Scan(iterationRangeMaxValues.ValuesPointers...); err != nil { - return hasFurtherRange, err + return hasFurtherRange, expectedRowCount, err } - hasFurtherRange = true + + expectedRowCount = (*iterationRangeMaxValues.ValuesPointers[len(iterationRangeMaxValues.ValuesPointers)-1].(*interface{})).(int64) + iterationRangeMaxValues = sql.ToColumnValues(iterationRangeMaxValues.AbstractValues()[:len(iterationRangeMaxValues.AbstractValues())-1]) + + hasFurtherRange = expectedRowCount > 0 } if err = rows.Err(); err != nil { - return hasFurtherRange, err + return hasFurtherRange, expectedRowCount, err } if hasFurtherRange { this.migrationContext.MigrationIterationRangeMaxValues = iterationRangeMaxValues - return hasFurtherRange, nil + return hasFurtherRange, expectedRowCount, nil } } this.migrationContext.Log.Debugf("Iteration complete: no further range to iterate") - return hasFurtherRange, nil + return hasFurtherRange, expectedRowCount, nil } // ApplyIterationInsertQuery issues a chunk-INSERT query on the ghost table. It is where diff --git a/go/logic/migrator.go b/go/logic/migrator.go index d89ca922f..6095d4d45 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1238,8 +1238,9 @@ func (this *Migrator) iterateChunks() error { // When hasFurtherRange is false, original table might be write locked and CalculateNextIterationRangeEndValues would hangs forever hasFurtherRange := false + expectedRangeSize := int64(0) if err := this.retryOperation(func() (e error) { - hasFurtherRange, e = this.applier.CalculateNextIterationRangeEndValues() + hasFurtherRange, expectedRangeSize, e = this.applier.CalculateNextIterationRangeEndValues() return e }); err != nil { return terminateRowIteration(err) @@ -1267,14 +1268,13 @@ func (this *Migrator) iterateChunks() error { } if this.migrationContext.PanicOnWarnings { - chunkSize := atomic.LoadInt64(&this.migrationContext.ChunkSize) if len(this.migrationContext.MigrationLastInsertSQLWarnings) > 0 { - for warning := range this.migrationContext.MigrationLastInsertSQLWarnings { - this.migrationContext.Log.Infof("last SQL insert warning: %s", warning) + for _, warning := range this.migrationContext.MigrationLastInsertSQLWarnings { + this.migrationContext.Log.Infof("ApplyIterationInsertQuery has SQL warnings! %s", warning) } - if chunkSize != rowsAffected { + if expectedRangeSize != rowsAffected { joinedWarnings := strings.Join(this.migrationContext.MigrationLastInsertSQLWarnings, "; ") - terminateRowIteration(fmt.Errorf("last SQL insert had warnings: %s", joinedWarnings)) + terminateRowIteration(fmt.Errorf("ApplyIterationInsertQuery failed because of SQL warnings: [%s]", joinedWarnings)) } } } diff --git a/go/sql/builder.go b/go/sql/builder.go index 332aef100..aea4d932e 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -275,7 +275,7 @@ func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string uniqueKeyColumnNames := duplicateNames(uniqueKeyColumns.Names()) uniqueKeyColumnAscending := make([]string, len(uniqueKeyColumnNames)) - uniqueKeyColumnDescending := make([]string, len(uniqueKeyColumnNames)) + uniqueKeyColumnDescending := make([]string, len(uniqueKeyColumnNames)) // TODO unused variable for i, column := range uniqueKeyColumns.Columns() { uniqueKeyColumnNames[i] = EscapeName(uniqueKeyColumnNames[i]) if column.Type == EnumColumnType { @@ -286,21 +286,29 @@ func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string uniqueKeyColumnDescending[i] = fmt.Sprintf("%s desc", uniqueKeyColumnNames[i]) } } + joinedColumnNames := strings.Join(uniqueKeyColumnNames, ", ") result = fmt.Sprintf(` - select /* gh-ost %s.%s %s */ - %s - from - %s.%s - where - %s and %s + with /* gh-ost %s.%s %s */ key_range as ( + select + %s + from + %s.%s + where + %s and %s + ) + select + %s, + (select count(*) from key_range) + from key_range order by %s limit 1 offset %d`, databaseName, tableName, hint, - strings.Join(uniqueKeyColumnNames, ", "), + joinedColumnNames, databaseName, tableName, rangeStartComparison, rangeEndComparison, + joinedColumnNames, strings.Join(uniqueKeyColumnAscending, ", "), (chunkSize - 1), ) @@ -342,9 +350,10 @@ func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName str uniqueKeyColumnDescending[i] = fmt.Sprintf("%s desc", uniqueKeyColumnNames[i]) } } + + joinedColumnNames := strings.Join(uniqueKeyColumnNames, ", ") result = fmt.Sprintf(` - select /* gh-ost %s.%s %s */ %s - from ( + with /* gh-ost %s.%s %s */ key_range as ( select %s from @@ -353,14 +362,20 @@ func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName str %s and %s order by %s - limit %d) select_osc_chunk + limit %d + ) + select + %s, + (select count(*) from key_range) + from key_range order by %s limit 1`, - databaseName, tableName, hint, strings.Join(uniqueKeyColumnNames, ", "), - strings.Join(uniqueKeyColumnNames, ", "), databaseName, tableName, + databaseName, tableName, hint, joinedColumnNames, + databaseName, tableName, rangeStartComparison, rangeEndComparison, strings.Join(uniqueKeyColumnAscending, ", "), chunkSize, + joinedColumnNames, strings.Join(uniqueKeyColumnDescending, ", "), ) return result, explodedArgs, nil From 9395272d3892ad2ec0b98fd30d82078a50f8eb8e Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Thu, 27 Feb 2025 09:59:45 +0100 Subject: [PATCH 05/15] Lint comments - ensure errors are handled in show warnings --- go/logic/applier.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/go/logic/applier.go b/go/logic/applier.go index 15f46f9c2..678806bde 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -759,10 +759,15 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected } if this.migrationContext.PanicOnWarnings { + //nolint:execinquery rows, err := tx.Query("SHOW WARNINGS") if err != nil { return nil, err } + defer rows.Close() + if err = rows.Err(); err != nil { + return nil, err + } var sqlWarnings []string for rows.Next() { From 4d7e8567b68cce2b5dfde06b38d7037705ee85f4 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Thu, 27 Feb 2025 10:07:02 +0100 Subject: [PATCH 06/15] update TestBuildUniqueKeyRangeEndPreparedQuery --- go/sql/builder_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/go/sql/builder_test.go b/go/sql/builder_test.go index d43f65056..fc94f32c2 100644 --- a/go/sql/builder_test.go +++ b/go/sql/builder_test.go @@ -337,8 +337,7 @@ func TestBuildUniqueKeyRangeEndPreparedQuery(t *testing.T) { query, explodedArgs, err := BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, originalTableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test") require.NoError(t, err) expected := ` - select /* gh-ost mydb.tbl test */ name, position - from ( + with /* gh-ost mydb.tbl test */ key_range as ( select name, position from @@ -346,7 +345,12 @@ func TestBuildUniqueKeyRangeEndPreparedQuery(t *testing.T) { where ((name > ?) or (((name = ?)) AND (position > ?))) and ((name < ?) or (((name = ?)) AND (position < ?)) or ((name = ?) and (position = ?))) order by name asc, position asc - limit 500) select_osc_chunk + limit 500 + ) + select + name, position, + (select count(*) from key_range) + from key_range order by name desc, position desc limit 1` From 30d78bd8df5a4cb512bfeb8ebf8729e49a4a32a6 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Thu, 27 Feb 2025 12:00:47 +0100 Subject: [PATCH 07/15] Localtests for PanicOnWarnings with data loss --- .../create.sql | 25 +++++++++++++++++++ .../expect_failure | 2 ++ .../extra_args | 1 + .../create.sql | 23 +++++++++++++++++ .../expect_failure | 1 + .../extra_args | 1 + 6 files changed, 53 insertions(+) create mode 100644 localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/create.sql create mode 100644 localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/expect_failure create mode 100644 localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/extra_args create mode 100644 localtests/panic-on-warnings-duplicate-values-for-unique-index/create.sql create mode 100644 localtests/panic-on-warnings-duplicate-values-for-unique-index/expect_failure create mode 100644 localtests/panic-on-warnings-duplicate-values-for-unique-index/extra_args diff --git a/localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/create.sql b/localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/create.sql new file mode 100644 index 000000000..0bdce5fd7 --- /dev/null +++ b/localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/create.sql @@ -0,0 +1,25 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + name varchar(255) not null, + primary key (id) +) auto_increment=1; + +create unique index name_index on gh_ost_test (name); + +insert into gh_ost_test (`name`) values ('John Smith'); +insert into gh_ost_test (`name`) values ('John Travolta'); + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert ignore into gh_ost_test values ('John ' || last_insert_id()); + insert ignore into gh_ost_test values ('Adam ' || last_insert_id()); +end ;; diff --git a/localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/expect_failure b/localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/expect_failure new file mode 100644 index 000000000..2c81a143c --- /dev/null +++ b/localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/expect_failure @@ -0,0 +1,2 @@ +Warning: Duplicate entry 'John' for key +Warning: Duplicate entry 'John' for key \ No newline at end of file diff --git a/localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/extra_args b/localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/extra_args new file mode 100644 index 000000000..09bf2ac77 --- /dev/null +++ b/localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/extra_args @@ -0,0 +1 @@ +--panic-on-warnings --alter "change column name name varchar(4) not null" \ No newline at end of file diff --git a/localtests/panic-on-warnings-duplicate-values-for-unique-index/create.sql b/localtests/panic-on-warnings-duplicate-values-for-unique-index/create.sql new file mode 100644 index 000000000..218f57314 --- /dev/null +++ b/localtests/panic-on-warnings-duplicate-values-for-unique-index/create.sql @@ -0,0 +1,23 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + name varchar(255) not null, + primary key (id) +) auto_increment=1; + +insert into gh_ost_test (`name`) values ('John'); +insert into gh_ost_test (`name`) values ('John'); + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert ignore into gh_ost_test values ('John ' || last_insert_id()); + insert ignore into gh_ost_test values ('Adam ' || last_insert_id()); +end ;; diff --git a/localtests/panic-on-warnings-duplicate-values-for-unique-index/expect_failure b/localtests/panic-on-warnings-duplicate-values-for-unique-index/expect_failure new file mode 100644 index 000000000..0f683a9b8 --- /dev/null +++ b/localtests/panic-on-warnings-duplicate-values-for-unique-index/expect_failure @@ -0,0 +1 @@ +Warning: Duplicate entry 'John' \ No newline at end of file diff --git a/localtests/panic-on-warnings-duplicate-values-for-unique-index/extra_args b/localtests/panic-on-warnings-duplicate-values-for-unique-index/extra_args new file mode 100644 index 000000000..2a9d95a0d --- /dev/null +++ b/localtests/panic-on-warnings-duplicate-values-for-unique-index/extra_args @@ -0,0 +1 @@ +--panic-on-warnings --alter "add unique index name_index(name)" \ No newline at end of file From d56fca964daf12e88f6340428d4e4db22b86eb57 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Fri, 28 Feb 2025 10:03:30 +0100 Subject: [PATCH 08/15] Unwrap CTE (mysql 5) --- go/sql/builder.go | 55 +++++++++++++++++++++++++++++------------- go/sql/builder_test.go | 22 +++++++++++------ 2 files changed, 53 insertions(+), 24 deletions(-) diff --git a/go/sql/builder.go b/go/sql/builder.go index aea4d932e..497c32bf2 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -288,31 +288,40 @@ func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string } joinedColumnNames := strings.Join(uniqueKeyColumnNames, ", ") result = fmt.Sprintf(` - with /* gh-ost %s.%s %s */ key_range as ( + select /* gh-ost %s.%s %s */ + %s, + (select count(*) from ( + select + %s + from + %s.%s + where + %s and %s + ) select_osc_chunk) + from ( select %s from %s.%s where %s and %s - ) - select - %s, - (select count(*) from key_range) - from key_range + ) select_osc_chunk order by %s limit 1 offset %d`, databaseName, tableName, hint, - joinedColumnNames, + joinedColumnNames, joinedColumnNames, databaseName, tableName, rangeStartComparison, rangeEndComparison, joinedColumnNames, + databaseName, tableName, + rangeStartComparison, rangeEndComparison, strings.Join(uniqueKeyColumnAscending, ", "), (chunkSize - 1), ) - return result, explodedArgs, nil + // 2x the explodedArgs for the subquery (CTE would be possible but not supported by MySQL 5) + return result, append(explodedArgs, explodedArgs...), nil } func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) { @@ -353,7 +362,20 @@ func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName str joinedColumnNames := strings.Join(uniqueKeyColumnNames, ", ") result = fmt.Sprintf(` - with /* gh-ost %s.%s %s */ key_range as ( + select /* gh-ost %s.%s %s */ + %s, + (select count(*) from ( + select + %s + from + %s.%s + where + %s and %s + order by + %s + limit %d + ) select_osc_chunk) + from ( select %s from @@ -363,22 +385,21 @@ func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName str order by %s limit %d - ) - select - %s, - (select count(*) from key_range) - from key_range + ) select_osc_chunk order by %s limit 1`, databaseName, tableName, hint, joinedColumnNames, - databaseName, tableName, + joinedColumnNames, databaseName, tableName, + rangeStartComparison, rangeEndComparison, + strings.Join(uniqueKeyColumnAscending, ", "), chunkSize, + joinedColumnNames, databaseName, tableName, rangeStartComparison, rangeEndComparison, strings.Join(uniqueKeyColumnAscending, ", "), chunkSize, - joinedColumnNames, strings.Join(uniqueKeyColumnDescending, ", "), ) - return result, explodedArgs, nil + // 2x the explodedArgs for the subquery (CTE would be possible but not supported by MySQL 5) + return result, append(explodedArgs, explodedArgs...), nil } func BuildUniqueKeyMinValuesPreparedQuery(databaseName, tableName string, uniqueKey *UniqueKey) (string, error) { diff --git a/go/sql/builder_test.go b/go/sql/builder_test.go index fc94f32c2..1ef94da9f 100644 --- a/go/sql/builder_test.go +++ b/go/sql/builder_test.go @@ -337,7 +337,19 @@ func TestBuildUniqueKeyRangeEndPreparedQuery(t *testing.T) { query, explodedArgs, err := BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, originalTableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test") require.NoError(t, err) expected := ` - with /* gh-ost mydb.tbl test */ key_range as ( + select /* gh-ost mydb.tbl test */ + name, position, + (select count(*) from ( + select + name, position + from + mydb.tbl + where ((name > ?) or (((name = ?)) AND (position > ?))) and ((name < ?) or (((name = ?)) AND (position < ?)) or ((name = ?) and (position = ?))) + order by + name asc, position asc + limit 500 + ) select_osc_chunk) + from ( select name, position from @@ -346,16 +358,12 @@ func TestBuildUniqueKeyRangeEndPreparedQuery(t *testing.T) { order by name asc, position asc limit 500 - ) - select - name, position, - (select count(*) from key_range) - from key_range + ) select_osc_chunk order by name desc, position desc limit 1` require.Equal(t, normalizeQuery(expected), normalizeQuery(query)) - require.Equal(t, []interface{}{3, 3, 17, 103, 103, 117, 103, 117}, explodedArgs) + require.Equal(t, []interface{}{3, 3, 17, 103, 103, 117, 103, 117, 3, 3, 17, 103, 103, 117, 103, 117}, explodedArgs) } } From 3c9d224109d10c4725dc84c52f2ffd08bdaae162 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Fri, 28 Feb 2025 17:52:07 +0100 Subject: [PATCH 09/15] Update localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/extra_args Co-authored-by: Bastian Bartmann --- .../extra_args | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/extra_args b/localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/extra_args index 09bf2ac77..eb98d683a 100644 --- a/localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/extra_args +++ b/localtests/panic-on-warnings-duplicate-unique-values-on-column-type-change/extra_args @@ -1 +1 @@ ---panic-on-warnings --alter "change column name name varchar(4) not null" \ No newline at end of file +--panic-on-warnings --alter "modify column name varchar(4) not null" \ No newline at end of file From d9cba90f9adb182c60c2d6ef9467426a4a908cf2 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Tue, 18 Mar 2025 17:10:34 +0100 Subject: [PATCH 10/15] limit BuildUniqueKeyRangeEndPreparedQueryViaOffset subquery properly --- go/sql/builder.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/go/sql/builder.go b/go/sql/builder.go index 497c32bf2..d6cf505ae 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -297,6 +297,8 @@ func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string %s.%s where %s and %s + limit + %d ) select_osc_chunk) from ( select @@ -305,6 +307,8 @@ func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string %s.%s where %s and %s + limit + %d ) select_osc_chunk order by %s @@ -313,10 +317,10 @@ func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string databaseName, tableName, hint, joinedColumnNames, joinedColumnNames, databaseName, tableName, - rangeStartComparison, rangeEndComparison, + rangeStartComparison, rangeEndComparison, chunkSize, joinedColumnNames, databaseName, tableName, - rangeStartComparison, rangeEndComparison, + rangeStartComparison, rangeEndComparison, chunkSize, strings.Join(uniqueKeyColumnAscending, ", "), (chunkSize - 1), ) From 7f614e05442dd332228aa0aabaf07bad0de64f06 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Thu, 20 Mar 2025 12:46:53 +0100 Subject: [PATCH 11/15] Update Applier to support all unique indices with PanicOnWarnings. Add test coverage. --- go/logic/applier.go | 4 +- go/logic/applier_test.go | 92 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index 678806bde..e50df7dd2 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -777,8 +777,8 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected this.migrationContext.Log.Warningf("Failed to read SHOW WARNINGS row") continue } - pkDuplicateSuffix := fmt.Sprintf("for key '%s.PRIMARY'", this.migrationContext.GetGhostTableName()) - if strings.HasPrefix(message, "Duplicate entry") && strings.HasSuffix(message, pkDuplicateSuffix) { + migrationUniqueKeySuffix := fmt.Sprintf("for key '%s.%s'", this.migrationContext.GetGhostTableName(), this.migrationContext.UniqueKey.Name) + if strings.HasPrefix(message, "Duplicate entry") && strings.HasSuffix(message, migrationUniqueKeySuffix) { continue } sqlWarnings = append(sqlWarnings, fmt.Sprintf("%s: %s (%d)", level, message, code)) diff --git a/go/logic/applier_test.go b/go/logic/applier_test.go index f53e65ffb..8214499d5 100644 --- a/go/logic/applier_test.go +++ b/go/logic/applier_test.go @@ -504,6 +504,98 @@ func (suite *ApplierTestSuite) TestCreateGhostTable() { suite.Require().Equal("CREATE TABLE `_testing_gho` (\n `id` int DEFAULT NULL,\n `item_id` int DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci", createDDL) } +func (suite *ApplierTestSuite) TestPanicOnWarningsInApplyIterationInsertQuerySucceedsWithUniqueKeyWarningInsertedByDMLEvent() { + ctx := context.Background() + + var err error + + _, err = suite.db.ExecContext(ctx, "CREATE TABLE test.testing (id INT, item_id INT, UNIQUE KEY (item_id));") + suite.Require().NoError(err) + + _, err = suite.db.ExecContext(ctx, "CREATE TABLE test._testing_gho (id INT, item_id INT, UNIQUE KEY (item_id));") + suite.Require().NoError(err) + + connectionConfig, err := GetConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := base.NewMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.DatabaseName = "test" + migrationContext.SkipPortValidation = true + migrationContext.OriginalTableName = "testing" + migrationContext.SetConnectionConfig("innodb") + + migrationContext.PanicOnWarnings = true + + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "item_id"}) + migrationContext.SharedColumns = sql.NewColumnList([]string{"id", "item_id"}) + migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id", "item_id"}) + migrationContext.UniqueKey = &sql.UniqueKey{ + Name: "item_id", + Columns: *sql.NewColumnList([]string{"item_id"}), + } + + applier := NewApplier(migrationContext) + suite.Require().NoError(applier.prepareQueries()) + defer applier.Teardown() + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + _, err = suite.db.ExecContext(ctx, "INSERT INTO test.testing (id, item_id) VALUES (123456, 42);") + suite.Require().NoError(err) + + dmlEvents := []*binlog.BinlogDMLEvent{ + { + DatabaseName: "test", + TableName: "testing", + DML: binlog.InsertDML, + NewColumnValues: sql.ToColumnValues([]interface{}{123456, 42}), + }, + } + err = applier.ApplyDMLEventQueries(dmlEvents) + suite.Require().NoError(err) + + err = applier.CreateChangelogTable() + suite.Require().NoError(err) + err = applier.ReadMigrationRangeValues() + suite.Require().NoError(err) + + hasFurtherRange, expectedRangeSize, err := applier.CalculateNextIterationRangeEndValues() + suite.Require().NoError(err) + suite.Require().True(hasFurtherRange) + suite.Require().Equal(int64(1), expectedRangeSize) + + _, rowsAffected, _, err := applier.ApplyIterationInsertQuery() + suite.Require().NoError(err) + suite.Require().Equal(int64(0), rowsAffected) + + // Ensure Duplicate entry '42' for key '_testing_gho.item_id' is ignored correctly + suite.Require().Empty(applier.migrationContext.MigrationLastInsertSQLWarnings) + + // Check that the row was inserted + rows, err := suite.db.Query("SELECT * FROM test._testing_gho") + suite.Require().NoError(err) + defer rows.Close() + + var count, id, item_id int + for rows.Next() { + err = rows.Scan(&id, &item_id) + suite.Require().NoError(err) + count += 1 + } + suite.Require().NoError(rows.Err()) + + suite.Require().Equal(1, count) + suite.Require().Equal(123456, id) + suite.Require().Equal(42, item_id) + + suite.Require(). + Equal(int64(1), migrationContext.TotalDMLEventsApplied) + suite.Require(). + Equal(int64(0), migrationContext.RowsDeltaEstimate) +} + func TestApplier(t *testing.T) { suite.Run(t, new(ApplierTestSuite)) } From 74f60a97c6d3afb588a80cc0a58c2e2fed37f6b4 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Mon, 24 Mar 2025 16:51:12 +0100 Subject: [PATCH 12/15] Impl code review feedback for PanicOnWarnings - documentation - remove dev.yml - remove unused variable --- dev.yml | 21 --------------------- doc/command-line-flags.md | 6 ++++++ go/sql/builder.go | 3 --- 3 files changed, 6 insertions(+), 24 deletions(-) delete mode 100644 dev.yml diff --git a/dev.yml b/dev.yml deleted file mode 100644 index 4c28ff349..000000000 --- a/dev.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: gh-ost - -env: - TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: /var/run/docker.sock - TESTCONTAINERS_RYUK_DISABLED: "true" - -up: - - go: - version: "1.22.12" - - podman - - custom: - name: Go Dependencies - met?: go mod download - meet: echo 'go mod failed to download dependencies'; false - -commands: - test: - desc: Run all the tests. - run: | - export DOCKER_HOST=unix://$(podman machine inspect --format '{{.ConnectionInfo.PodmanSocket.Path}}') - script/test diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index cf2b2ca95..d64cf3913 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -202,6 +202,12 @@ List of metrics and threshold values; topping the threshold of any will cause th Typically `gh-ost` is used to migrate tables on a master. If you wish to only perform the migration in full on a replica, connect `gh-ost` to said replica and pass `--migrate-on-replica`. `gh-ost` will briefly connect to the master but otherwise will make no changes on the master. Migration will be fully executed on the replica, while making sure to maintain a small replication lag. +### panic-on-warnings + +When this flag is set, `gh-ost` will panic when SQL warnings indicating data loss are encountered when copying data. This flag helps prevent data loss scenarios with migrations touching unique keys, column collation and types, as well as `NOT NULL` constraints, where `MySQL` will silently drop inserted rows that no longer satisfy the updated constraint (also dependent on the configured `sql_mode`). + +While `panic-on-warnings` is currently disabled by defaults, it will default to `true` in a future version of `gh-ost`. + ### postpone-cut-over-flag-file Indicate a file name, such that the final [cut-over](cut-over.md) step does not take place as long as the file exists. diff --git a/go/sql/builder.go b/go/sql/builder.go index d6cf505ae..c9ac87044 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -275,15 +275,12 @@ func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string uniqueKeyColumnNames := duplicateNames(uniqueKeyColumns.Names()) uniqueKeyColumnAscending := make([]string, len(uniqueKeyColumnNames)) - uniqueKeyColumnDescending := make([]string, len(uniqueKeyColumnNames)) // TODO unused variable for i, column := range uniqueKeyColumns.Columns() { uniqueKeyColumnNames[i] = EscapeName(uniqueKeyColumnNames[i]) if column.Type == EnumColumnType { uniqueKeyColumnAscending[i] = fmt.Sprintf("concat(%s) asc", uniqueKeyColumnNames[i]) - uniqueKeyColumnDescending[i] = fmt.Sprintf("concat(%s) desc", uniqueKeyColumnNames[i]) } else { uniqueKeyColumnAscending[i] = fmt.Sprintf("%s asc", uniqueKeyColumnNames[i]) - uniqueKeyColumnDescending[i] = fmt.Sprintf("%s desc", uniqueKeyColumnNames[i]) } } joinedColumnNames := strings.Join(uniqueKeyColumnNames, ", ") From 1487b5c84887e3753257eed3a897a0167513fb3e Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Mon, 24 Mar 2025 17:01:12 +0100 Subject: [PATCH 13/15] bump golangci-lint for local dev --- script/ensure-golangci-lint-installed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/ensure-golangci-lint-installed b/script/ensure-golangci-lint-installed index 6f7c0c42b..edcd57891 100755 --- a/script/ensure-golangci-lint-installed +++ b/script/ensure-golangci-lint-installed @@ -1,7 +1,7 @@ #!/bin/bash # See https://github.com/golangci/golangci-lint/releases -GOLANGCI_RELEASE=v1.54.2 +GOLANGCI_RELEASE=v1.61.0 GOLANGCI_INSTALL_SCRIPT=https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh if [ -z "$GOPATH" ]; then From 1ca8ffcb5fd26b3a0c4b63bbd2233e8cc15c16be Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Mon, 24 Mar 2025 18:49:21 +0100 Subject: [PATCH 14/15] Support altering index names with PanicOnWarnings --- go/logic/applier.go | 2 +- go/logic/applier_test.go | 5 +++-- go/logic/inspect.go | 3 +++ go/sql/types.go | 9 +++++---- localtests/swap-uk-uk/extra_args | 2 +- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index e50df7dd2..45e740deb 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -777,7 +777,7 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected this.migrationContext.Log.Warningf("Failed to read SHOW WARNINGS row") continue } - migrationUniqueKeySuffix := fmt.Sprintf("for key '%s.%s'", this.migrationContext.GetGhostTableName(), this.migrationContext.UniqueKey.Name) + migrationUniqueKeySuffix := fmt.Sprintf("for key '%s.%s'", this.migrationContext.GetGhostTableName(), this.migrationContext.UniqueKey.NameInGhostTable) if strings.HasPrefix(message, "Duplicate entry") && strings.HasSuffix(message, migrationUniqueKeySuffix) { continue } diff --git a/go/logic/applier_test.go b/go/logic/applier_test.go index 8214499d5..d5fa06949 100644 --- a/go/logic/applier_test.go +++ b/go/logic/applier_test.go @@ -531,8 +531,9 @@ func (suite *ApplierTestSuite) TestPanicOnWarningsInApplyIterationInsertQuerySuc migrationContext.SharedColumns = sql.NewColumnList([]string{"id", "item_id"}) migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id", "item_id"}) migrationContext.UniqueKey = &sql.UniqueKey{ - Name: "item_id", - Columns: *sql.NewColumnList([]string{"item_id"}), + Name: "item_id", + NameInGhostTable: "item_id", + Columns: *sql.NewColumnList([]string{"item_id"}), } applier := NewApplier(migrationContext) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index aec95cdc6..89ade8806 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -846,6 +846,9 @@ func (this *Inspector) getSharedUniqueKeys(originalUniqueKeys, ghostUniqueKeys [ for _, originalUniqueKey := range originalUniqueKeys { for _, ghostUniqueKey := range ghostUniqueKeys { if originalUniqueKey.Columns.IsSubsetOf(&ghostUniqueKey.Columns) { + // In case the unique key gets renamed in -alter, PanicOnWarnings needs to rely on the new name + // to check SQL warnings on the ghost table, so return new name here. + originalUniqueKey.NameInGhostTable = ghostUniqueKey.Name uniqueKeys = append(uniqueKeys, originalUniqueKey) break } diff --git a/go/sql/types.go b/go/sql/types.go index f7aac5f5f..673a60c03 100644 --- a/go/sql/types.go +++ b/go/sql/types.go @@ -265,10 +265,11 @@ func (this *ColumnList) SetCharsetConversion(columnName string, fromCharset stri // UniqueKey is the combination of a key's name and columns type UniqueKey struct { - Name string - Columns ColumnList - HasNullable bool - IsAutoIncrement bool + Name string + NameInGhostTable string // Name of the corresponding key in the Ghost table in case it is being renamed + Columns ColumnList + HasNullable bool + IsAutoIncrement bool } // IsPrimary checks if this unique key is primary diff --git a/localtests/swap-uk-uk/extra_args b/localtests/swap-uk-uk/extra_args index 84161a54e..403757970 100644 --- a/localtests/swap-uk-uk/extra_args +++ b/localtests/swap-uk-uk/extra_args @@ -1 +1 @@ ---alter="drop key id_uidx, drop key its_uidx, add unique key its2_uidx(i, ts), add unique key id2_uidx(id)" +--panic-on-warnings --alter="drop key id_uidx, drop key its_uidx, add unique key its2_uidx(i, ts), add unique key id2_uidx(id)" From 71a63c4a1e6c4ac4a847a606d4c8999dd5ef8c21 Mon Sep 17 00:00:00 2001 From: Jan Grodowski Date: Tue, 25 Mar 2025 09:30:58 +0100 Subject: [PATCH 15/15] Fix string matching for PanicOnWarnings to correctly suppress warnings when renaming unique keys Error message formats are different across mysql distributions and versions --- go/logic/applier.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index 45e740deb..0491aae8d 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -8,6 +8,7 @@ package logic import ( gosql "database/sql" "fmt" + "regexp" "strings" "sync/atomic" "time" @@ -777,8 +778,10 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected this.migrationContext.Log.Warningf("Failed to read SHOW WARNINGS row") continue } - migrationUniqueKeySuffix := fmt.Sprintf("for key '%s.%s'", this.migrationContext.GetGhostTableName(), this.migrationContext.UniqueKey.NameInGhostTable) - if strings.HasPrefix(message, "Duplicate entry") && strings.HasSuffix(message, migrationUniqueKeySuffix) { + // Duplicate warnings are formatted differently across mysql versions, hence the optional table name prefix + migrationUniqueKeyExpression := fmt.Sprintf("for key '(%s\\.)?%s'", this.migrationContext.GetGhostTableName(), this.migrationContext.UniqueKey.NameInGhostTable) + matched, _ := regexp.MatchString(migrationUniqueKeyExpression, message) + if strings.Contains(message, "Duplicate entry") && matched { continue } sqlWarnings = append(sqlWarnings, fmt.Sprintf("%s: %s (%d)", level, message, code))