From 77f65ad89862499916e40e55a419ed4436f74ed9 Mon Sep 17 00:00:00 2001 From: lukelewang Date: Mon, 9 Jan 2023 15:27:15 +0800 Subject: [PATCH 1/2] force index in iteration range end value queries --- go/logic/applier.go | 2 +- go/sql/builder.go | 32 ++++++++++++++++++-------------- go/sql/builder_test.go | 4 +++- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index ad6368e61..52bf01743 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -574,7 +574,7 @@ func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange boo query, explodedArgs, err := buildFunc( this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, - &this.migrationContext.UniqueKey.Columns, + this.migrationContext.UniqueKey, this.migrationContext.MigrationIterationRangeMinValues.AbstractValues(), this.migrationContext.MigrationRangeMaxValues.AbstractValues(), atomic.LoadInt64(&this.migrationContext.ChunkSize), diff --git a/go/sql/builder.go b/go/sql/builder.go index 0169390c2..88272244c 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -238,32 +238,33 @@ func BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableNa return BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, mappedSharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, includeRangeStartValues, transactionalTable) } -func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) { - if uniqueKeyColumns.Len() == 0 { +func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string, uniqueKey *UniqueKey, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) { + if uniqueKey.Columns.Len() == 0 { return "", explodedArgs, fmt.Errorf("Got 0 columns in BuildUniqueKeyRangeEndPreparedQuery") } databaseName = EscapeName(databaseName) tableName = EscapeName(tableName) + indexName := EscapeName(uniqueKey.Name) var startRangeComparisonSign ValueComparisonSign = GreaterThanComparisonSign if includeRangeStartValues { startRangeComparisonSign = GreaterThanOrEqualsComparisonSign } - rangeStartComparison, rangeExplodedArgs, err := BuildRangePreparedComparison(uniqueKeyColumns, rangeStartArgs, startRangeComparisonSign) + rangeStartComparison, rangeExplodedArgs, err := BuildRangePreparedComparison(&uniqueKey.Columns, rangeStartArgs, startRangeComparisonSign) if err != nil { return "", explodedArgs, err } explodedArgs = append(explodedArgs, rangeExplodedArgs...) - rangeEndComparison, rangeExplodedArgs, err := BuildRangePreparedComparison(uniqueKeyColumns, rangeEndArgs, LessThanOrEqualsComparisonSign) + rangeEndComparison, rangeExplodedArgs, err := BuildRangePreparedComparison(&uniqueKey.Columns, rangeEndArgs, LessThanOrEqualsComparisonSign) if err != nil { return "", explodedArgs, err } explodedArgs = append(explodedArgs, rangeExplodedArgs...) - uniqueKeyColumnNames := duplicateNames(uniqueKeyColumns.Names()) + uniqueKeyColumnNames := duplicateNames(uniqueKey.Columns.Names()) uniqueKeyColumnAscending := make([]string, len(uniqueKeyColumnNames)) uniqueKeyColumnDescending := make([]string, len(uniqueKeyColumnNames)) - for i, column := range uniqueKeyColumns.Columns() { + for i, column := range uniqueKey.Columns.Columns() { uniqueKeyColumnNames[i] = EscapeName(uniqueKeyColumnNames[i]) if column.Type == EnumColumnType { uniqueKeyColumnAscending[i] = fmt.Sprintf("concat(%s) asc", uniqueKeyColumnNames[i]) @@ -278,6 +279,7 @@ func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string %s from %s.%s + force index (%s) where %s and %s order by %s @@ -285,7 +287,7 @@ func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string offset %d `, databaseName, tableName, hint, strings.Join(uniqueKeyColumnNames, ", "), - databaseName, tableName, + databaseName, tableName, indexName, rangeStartComparison, rangeEndComparison, strings.Join(uniqueKeyColumnAscending, ", "), (chunkSize - 1), @@ -293,32 +295,33 @@ func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string return result, explodedArgs, nil } -func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) { - if uniqueKeyColumns.Len() == 0 { +func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName string, uniqueKey *UniqueKey, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) { + if uniqueKey.Columns.Len() == 0 { return "", explodedArgs, fmt.Errorf("Got 0 columns in BuildUniqueKeyRangeEndPreparedQuery") } databaseName = EscapeName(databaseName) tableName = EscapeName(tableName) + indexName := EscapeName(uniqueKey.Name) var startRangeComparisonSign ValueComparisonSign = GreaterThanComparisonSign if includeRangeStartValues { startRangeComparisonSign = GreaterThanOrEqualsComparisonSign } - rangeStartComparison, rangeExplodedArgs, err := BuildRangePreparedComparison(uniqueKeyColumns, rangeStartArgs, startRangeComparisonSign) + rangeStartComparison, rangeExplodedArgs, err := BuildRangePreparedComparison(&uniqueKey.Columns, rangeStartArgs, startRangeComparisonSign) if err != nil { return "", explodedArgs, err } explodedArgs = append(explodedArgs, rangeExplodedArgs...) - rangeEndComparison, rangeExplodedArgs, err := BuildRangePreparedComparison(uniqueKeyColumns, rangeEndArgs, LessThanOrEqualsComparisonSign) + rangeEndComparison, rangeExplodedArgs, err := BuildRangePreparedComparison(&uniqueKey.Columns, rangeEndArgs, LessThanOrEqualsComparisonSign) if err != nil { return "", explodedArgs, err } explodedArgs = append(explodedArgs, rangeExplodedArgs...) - uniqueKeyColumnNames := duplicateNames(uniqueKeyColumns.Names()) + uniqueKeyColumnNames := duplicateNames(uniqueKey.Columns.Names()) uniqueKeyColumnAscending := make([]string, len(uniqueKeyColumnNames)) uniqueKeyColumnDescending := make([]string, len(uniqueKeyColumnNames)) - for i, column := range uniqueKeyColumns.Columns() { + for i, column := range uniqueKey.Columns.Columns() { uniqueKeyColumnNames[i] = EscapeName(uniqueKeyColumnNames[i]) if column.Type == EnumColumnType { uniqueKeyColumnAscending[i] = fmt.Sprintf("concat(%s) asc", uniqueKeyColumnNames[i]) @@ -335,6 +338,7 @@ func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName str %s from %s.%s + force index (%s) where %s and %s order by %s @@ -344,7 +348,7 @@ func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName str %s limit 1 `, databaseName, tableName, hint, strings.Join(uniqueKeyColumnNames, ", "), - strings.Join(uniqueKeyColumnNames, ", "), databaseName, tableName, + strings.Join(uniqueKeyColumnNames, ", "), databaseName, tableName, indexName, rangeStartComparison, rangeEndComparison, strings.Join(uniqueKeyColumnAscending, ", "), chunkSize, strings.Join(uniqueKeyColumnDescending, ", "), diff --git a/go/sql/builder_test.go b/go/sql/builder_test.go index 299824269..c08160feb 100644 --- a/go/sql/builder_test.go +++ b/go/sql/builder_test.go @@ -280,10 +280,11 @@ func TestBuildUniqueKeyRangeEndPreparedQuery(t *testing.T) { var chunkSize int64 = 500 { uniqueKeyColumns := NewColumnList([]string{"name", "position"}) + uniqueKey := UniqueKey{"PRIMARY", *uniqueKeyColumns, false, false} rangeStartArgs := []interface{}{3, 17} rangeEndArgs := []interface{}{103, 117} - query, explodedArgs, err := BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, originalTableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test") + query, explodedArgs, err := BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, originalTableName, &uniqueKey, rangeStartArgs, rangeEndArgs, chunkSize, false, "test") test.S(t).ExpectNil(err) expected := ` select /* gh-ost mydb.tbl test */ name, position @@ -292,6 +293,7 @@ func TestBuildUniqueKeyRangeEndPreparedQuery(t *testing.T) { name, position from mydb.tbl + force index (PRIMARY) where ((name > ?) or (((name = ?)) AND (position > ?))) and ((name < ?) or (((name = ?)) AND (position < ?)) or ((name = ?) and (position = ?))) order by name asc, position asc From 2a852df2d94582897006a01767ef5c52068da1d6 Mon Sep 17 00:00:00 2001 From: lukelewang Date: Tue, 10 Jan 2023 10:50:01 +0800 Subject: [PATCH 2/2] format SQL --- go/sql/builder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/sql/builder.go b/go/sql/builder.go index 88272244c..91251cc27 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -279,7 +279,7 @@ func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string %s from %s.%s - force index (%s) + force index (%s) where %s and %s order by %s @@ -338,7 +338,7 @@ func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName str %s from %s.%s - force index (%s) + force index (%s) where %s and %s order by %s