From 846b20edd11b1e8a92870259673fd7c1ab1e1090 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 28 Dec 2022 23:08:44 +0100 Subject: [PATCH 1/4] WIP --- go/logic/applier.go | 6 ++++-- go/sql/builder.go | 26 +++++++++++++------------- go/sql/builder_test.go | 7 +++++-- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index ad6368e61..d2925079f 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -472,7 +472,8 @@ func (this *Applier) ExecuteThrottleQuery() (int64, error) { // readMigrationMinValues returns the minimum values to be iterated on rowcopy func (this *Applier) readMigrationMinValues(tx *gosql.Tx, uniqueKey *sql.UniqueKey) error { this.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) - query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &uniqueKey.Columns) + query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, + this.migrationContext.UniqueKey.Name, &uniqueKey.Columns) if err != nil { return err } @@ -497,7 +498,8 @@ func (this *Applier) readMigrationMinValues(tx *gosql.Tx, uniqueKey *sql.UniqueK // readMigrationMaxValues returns the maximum values to be iterated on rowcopy func (this *Applier) readMigrationMaxValues(tx *gosql.Tx, uniqueKey *sql.UniqueKey) error { this.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) - query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &uniqueKey.Columns) + query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, + this.migrationContext.UniqueKey.Name, &uniqueKey.Columns) if err != nil { return err } diff --git a/go/sql/builder.go b/go/sql/builder.go index 0169390c2..c11478af1 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -352,15 +352,15 @@ func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName str return result, explodedArgs, nil } -func BuildUniqueKeyMinValuesPreparedQuery(databaseName, tableName string, uniqueKeyColumns *ColumnList) (string, error) { - return buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName, uniqueKeyColumns, "asc") +func BuildUniqueKeyMinValuesPreparedQuery(databaseName, tableName, indexName string, uniqueKeyColumns *ColumnList) (string, error) { + return buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName, indexName, uniqueKeyColumns, "asc") } -func BuildUniqueKeyMaxValuesPreparedQuery(databaseName, tableName string, uniqueKeyColumns *ColumnList) (string, error) { - return buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName, uniqueKeyColumns, "desc") +func BuildUniqueKeyMaxValuesPreparedQuery(databaseName, tableName, indexName string, uniqueKeyColumns *ColumnList) (string, error) { + return buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName, indexName, uniqueKeyColumns, "desc") } -func buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName string, uniqueKeyColumns *ColumnList, order string) (string, error) { +func buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName, indexName string, uniqueKeyColumns *ColumnList, order string) (string, error) { if uniqueKeyColumns.Len() == 0 { return "", fmt.Errorf("Got 0 columns in BuildUniqueKeyMinMaxValuesPreparedQuery") } @@ -378,14 +378,14 @@ func buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName string, uni } } query := fmt.Sprintf(` - select /* gh-ost %s.%s */ %s - from - %s.%s - order by - %s - limit 1 - `, databaseName, tableName, strings.Join(uniqueKeyColumnNames, ", "), - databaseName, tableName, + select /* gh-ost %s.%s */ %s + from + %s.%s + force index (%s) + order by %s limit 1 + `, + databaseName, tableName, strings.Join(uniqueKeyColumnNames, ", "), + databaseName, tableName, indexName, strings.Join(uniqueKeyColumnOrder, ", "), ) return query, nil diff --git a/go/sql/builder_test.go b/go/sql/builder_test.go index 299824269..b4186965b 100644 --- a/go/sql/builder_test.go +++ b/go/sql/builder_test.go @@ -309,14 +309,16 @@ func TestBuildUniqueKeyRangeEndPreparedQuery(t *testing.T) { func TestBuildUniqueKeyMinValuesPreparedQuery(t *testing.T) { databaseName := "mydb" originalTableName := "tbl" + indexName := "PRIMARY" uniqueKeyColumns := NewColumnList([]string{"name", "position"}) { - query, err := BuildUniqueKeyMinValuesPreparedQuery(databaseName, originalTableName, uniqueKeyColumns) + query, err := BuildUniqueKeyMinValuesPreparedQuery(databaseName, originalTableName, indexName, uniqueKeyColumns) test.S(t).ExpectNil(err) expected := ` select /* gh-ost mydb.tbl */ name, position from mydb.tbl + force index (PRIMARY) order by name asc, position asc limit 1 @@ -324,12 +326,13 @@ func TestBuildUniqueKeyMinValuesPreparedQuery(t *testing.T) { test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) } { - query, err := BuildUniqueKeyMaxValuesPreparedQuery(databaseName, originalTableName, uniqueKeyColumns) + query, err := BuildUniqueKeyMaxValuesPreparedQuery(databaseName, originalTableName, indexName, uniqueKeyColumns) test.S(t).ExpectNil(err) expected := ` select /* gh-ost mydb.tbl */ name, position from mydb.tbl + force index (PRIMARY) order by name desc, position desc limit 1 From 1bc9c5104bd6156e74fa5115cfc4ec2d8e96c0a4 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 28 Dec 2022 23:53:20 +0100 Subject: [PATCH 2/4] Pass entire sql.UniqueKey --- go/logic/applier.go | 6 ++---- go/sql/builder.go | 18 +++++++++--------- go/sql/builder_test.go | 10 +++++----- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index d2925079f..42c5da930 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -472,8 +472,7 @@ func (this *Applier) ExecuteThrottleQuery() (int64, error) { // readMigrationMinValues returns the minimum values to be iterated on rowcopy func (this *Applier) readMigrationMinValues(tx *gosql.Tx, uniqueKey *sql.UniqueKey) error { this.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) - query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, - this.migrationContext.UniqueKey.Name, &uniqueKey.Columns) + query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, uniqueKey) if err != nil { return err } @@ -498,8 +497,7 @@ func (this *Applier) readMigrationMinValues(tx *gosql.Tx, uniqueKey *sql.UniqueK // readMigrationMaxValues returns the maximum values to be iterated on rowcopy func (this *Applier) readMigrationMaxValues(tx *gosql.Tx, uniqueKey *sql.UniqueKey) error { this.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) - query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, - this.migrationContext.UniqueKey.Name, &uniqueKey.Columns) + query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, uniqueKey) if err != nil { return err } diff --git a/go/sql/builder.go b/go/sql/builder.go index c11478af1..46d34fa8b 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -352,24 +352,24 @@ func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName str return result, explodedArgs, nil } -func BuildUniqueKeyMinValuesPreparedQuery(databaseName, tableName, indexName string, uniqueKeyColumns *ColumnList) (string, error) { - return buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName, indexName, uniqueKeyColumns, "asc") +func BuildUniqueKeyMinValuesPreparedQuery(databaseName, tableName string, uniqueKey *UniqueKey) (string, error) { + return buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName, uniqueKey, "asc") } -func BuildUniqueKeyMaxValuesPreparedQuery(databaseName, tableName, indexName string, uniqueKeyColumns *ColumnList) (string, error) { - return buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName, indexName, uniqueKeyColumns, "desc") +func BuildUniqueKeyMaxValuesPreparedQuery(databaseName, tableName string, uniqueKey *UniqueKey) (string, error) { + return buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName, uniqueKey, "desc") } -func buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName, indexName string, uniqueKeyColumns *ColumnList, order string) (string, error) { - if uniqueKeyColumns.Len() == 0 { +func buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName string, uniqueKey *UniqueKey, order string) (string, error) { + if uniqueKey.Columns.Len() == 0 { return "", fmt.Errorf("Got 0 columns in BuildUniqueKeyMinMaxValuesPreparedQuery") } databaseName = EscapeName(databaseName) tableName = EscapeName(tableName) - uniqueKeyColumnNames := duplicateNames(uniqueKeyColumns.Names()) + uniqueKeyColumnNames := duplicateNames(uniqueKey.Columns.Names()) uniqueKeyColumnOrder := make([]string, len(uniqueKeyColumnNames)) - for i, column := range uniqueKeyColumns.Columns() { + for i, column := range uniqueKey.Columns.Columns() { uniqueKeyColumnNames[i] = EscapeName(uniqueKeyColumnNames[i]) if column.Type == EnumColumnType { uniqueKeyColumnOrder[i] = fmt.Sprintf("concat(%s) %s", uniqueKeyColumnNames[i], order) @@ -385,7 +385,7 @@ func buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName, indexName order by %s limit 1 `, databaseName, tableName, strings.Join(uniqueKeyColumnNames, ", "), - databaseName, tableName, indexName, + databaseName, tableName, uniqueKey.Name, strings.Join(uniqueKeyColumnOrder, ", "), ) return query, nil diff --git a/go/sql/builder_test.go b/go/sql/builder_test.go index b4186965b..245663936 100644 --- a/go/sql/builder_test.go +++ b/go/sql/builder_test.go @@ -1,5 +1,5 @@ /* - Copyright 2016 GitHub Inc. + Copyright 2022 GitHub Inc. See https://github.com/github/gh-ost/blob/master/LICENSE */ @@ -309,10 +309,10 @@ func TestBuildUniqueKeyRangeEndPreparedQuery(t *testing.T) { func TestBuildUniqueKeyMinValuesPreparedQuery(t *testing.T) { databaseName := "mydb" originalTableName := "tbl" - indexName := "PRIMARY" - uniqueKeyColumns := NewColumnList([]string{"name", "position"}) + columnList := NewColumnList([]string{"name", "position"}) + uniqueKey := &UniqueKey{Name: "PRIMARY", Columns: *columnList} { - query, err := BuildUniqueKeyMinValuesPreparedQuery(databaseName, originalTableName, indexName, uniqueKeyColumns) + query, err := BuildUniqueKeyMinValuesPreparedQuery(databaseName, originalTableName, uniqueKey) test.S(t).ExpectNil(err) expected := ` select /* gh-ost mydb.tbl */ name, position @@ -326,7 +326,7 @@ func TestBuildUniqueKeyMinValuesPreparedQuery(t *testing.T) { test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) } { - query, err := BuildUniqueKeyMaxValuesPreparedQuery(databaseName, originalTableName, indexName, uniqueKeyColumns) + query, err := BuildUniqueKeyMaxValuesPreparedQuery(databaseName, originalTableName, uniqueKey) test.S(t).ExpectNil(err) expected := ` select /* gh-ost mydb.tbl */ name, position From ae43e3f3565843cf66acf8c8f704fefe63ab6086 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 29 Dec 2022 00:35:54 +0100 Subject: [PATCH 3/4] newline for limit --- go/sql/builder.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/sql/builder.go b/go/sql/builder.go index 46d34fa8b..286c87fc0 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -382,7 +382,8 @@ func buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName string, uni from %s.%s force index (%s) - order by %s limit 1 + order by %s + limit 1 `, databaseName, tableName, strings.Join(uniqueKeyColumnNames, ", "), databaseName, tableName, uniqueKey.Name, From 91b7bfe0137f00626544cab2a912d592b2fccf99 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 29 Dec 2022 00:41:25 +0100 Subject: [PATCH 4/4] Rename var --- go/sql/builder_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/sql/builder_test.go b/go/sql/builder_test.go index 245663936..e708d19f4 100644 --- a/go/sql/builder_test.go +++ b/go/sql/builder_test.go @@ -309,8 +309,8 @@ func TestBuildUniqueKeyRangeEndPreparedQuery(t *testing.T) { func TestBuildUniqueKeyMinValuesPreparedQuery(t *testing.T) { databaseName := "mydb" originalTableName := "tbl" - columnList := NewColumnList([]string{"name", "position"}) - uniqueKey := &UniqueKey{Name: "PRIMARY", Columns: *columnList} + uniqueKeyColumns := NewColumnList([]string{"name", "position"}) + uniqueKey := &UniqueKey{Name: "PRIMARY", Columns: *uniqueKeyColumns} { query, err := BuildUniqueKeyMinValuesPreparedQuery(databaseName, originalTableName, uniqueKey) test.S(t).ExpectNil(err)