Skip to content

tests: fix next gen integration TiDB startup flakiness #4278

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wlwilliamx wants to merge 1 commit into
base: master
Choose a base branch
from
Open

tests: fix next gen integration TiDB startup flakiness #4278

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 27 additions & 21 deletions tests/integration_tests/_utils/start_tidb_cluster_nextgen
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,24 @@ TEST_DATA_DIR="$CASE_DIR/$TEST_OUT_DATA_DIR"

echo "TEST_DATA_DIR: $TEST_DATA_DIR"

# Random generate the sockets config.
# Make sure we dont use the same sock.
# Random generate the `socket` config for TiDB.
# TiDB parses `socket = ...` as a root-level key. Our generated config ends with
# `[instance]`, so appending `socket = ...` at the end would become `instance.socket`,
# which is rejected by newer TiDB versions (and makes TiDB fail to start).
randomGenSocketsConf() {
config_file="$1"
random_str=$(date '+%s%N')
if [ "$(uname)" == "Darwin" ]; then
random_str=$(cat /dev/random | LC_ALL=C tr -dc "a-zA-Z0-9" | head -c 10)
fi

echo "socket = \"/tmp/tidb-$random_str.sock\"" >>"$config_file"
# Prepend the root-level `socket` and drop any existing `socket = ...` lines
# (old scripts appended it under `[instance]`, which becomes `instance.socket`).
{
echo "socket = \"/tmp/tidb-$random_str.sock\""
sed '/^[[:space:]]*socket[[:space:]]*=/d' "$config_file"
} >"$config_file.tmp"
mv "$config_file.tmp" "$config_file"
Comment on lines +32 to +33
Copy link

@gemini-code-assist gemini-code-assist bot Feb 26, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Using a fixed temporary filename like .tmp can be problematic if multiple instances of this script run concurrently. To make the temporary filename unique and avoid potential race conditions, consider appending the process ID ($$). A more robust solution would be to use mktemp.

	} >"$config_file.tmp.$$"
	mv "$config_file.tmp.$$" "$config_file"

}

check_bin() {
Expand Down Expand Up @@ -343,6 +351,10 @@ check_port_available "$UP_TIKV_WORKER_HOST" "$UP_TIKV_WORKER_PORT" "Upstream TiK
check_port_available "$DOWN_TIKV_WORKER_HOST" "$DOWN_TIKV_WORKER_PORT" "Downstream TiKV-Worker is not ready yet"

####################start TiDB####################
# NOTE: TiDB config keys are section-sensitive. Keep `socket` and
# `max-server-connections` at root level, and keep tuning knobs like
# `run-auto-analyze` / `server-memory-quota` under `[performance]`. Otherwise
# TiDB will report "invalid configuration options" and silently ignore them.
touch "$OUT_DIR/upstream-tidb-system.toml"
if [ -f "$tidb_config" ]; then
cat "$tidb_config" >>"$OUT_DIR/upstream-tidb-system.toml"
Expand All @@ -352,9 +364,7 @@ cat >>"$OUT_DIR/upstream-tidb-system.toml" <<EOF
keyspace-name = "SYSTEM"
enable-telemetry = false
mem-quota-query = 671088640
run-auto-analyze = false
server-memory-quota = 2684354560
analyze-always-skip-wide-columns = true
max-server-connections = 0
tikv-worker-url = "http://$UP_TIKV_WORKER_HOST:$UP_TIKV_WORKER_PORT"
# temp-dir = "$TEST_DATA_DIR/upstream/tidb-system"

Expand All @@ -363,13 +373,14 @@ max-backups = 100

[performance]
tcp-keep-alive = true
run-auto-analyze = false
server-memory-quota = 2684354560

[security]
enable-sem = false

[instance]
tidb_service_scope = 'dxf_service'
max-server-connections = 0
EOF

touch "$OUT_DIR/upstream-tidb-$KEYSPACE_NAME.toml"
Expand All @@ -381,22 +392,20 @@ cat >>"$OUT_DIR/upstream-tidb-$KEYSPACE_NAME.toml" <<EOF
keyspace-name = "$KEYSPACE_NAME"
enable-telemetry = false
mem-quota-query = 671088640
run-auto-analyze = false
server-memory-quota = 2684354560
analyze-always-skip-wide-columns = true
max-server-connections = 0
# temp-dir = "$TEST_DATA_DIR/upstream/tidb-$KEYSPACE_NAME"

[log.file]
max-backups = 100

[performance]
tcp-keep-alive = true
run-auto-analyze = false
server-memory-quota = 2684354560

[security]
enable-sem = false

[instance]
max-server-connections = 0
EOF

cp "$OUT_DIR/upstream-tidb-$KEYSPACE_NAME.toml" "$OUT_DIR/upstream-tidb-$KEYSPACE_NAME-other.toml"
Expand All @@ -410,9 +419,7 @@ cat >>"$OUT_DIR/downstream-tidb-system.toml" <<EOF
keyspace-name = "SYSTEM"
enable-telemetry = false
mem-quota-query = 671088640
run-auto-analyze = false
server-memory-quota = 2684354560
analyze-always-skip-wide-columns = true
max-server-connections = 0
tikv-worker-url = "http://$DOWN_TIKV_WORKER_HOST:$DOWN_TIKV_WORKER_PORT"
# temp-dir = "$TEST_DATA_DIR/downstream/tidb-system"

Expand All @@ -421,13 +428,14 @@ max-backups = 100

[performance]
tcp-keep-alive = true
run-auto-analyze = false
server-memory-quota = 2684354560

[security]
enable-sem = false

[instance]
tidb_service_scope = 'dxf_service'
max-server-connections = 0
EOF

touch "$OUT_DIR/downstream-tidb-$KEYSPACE_NAME.toml"
Expand All @@ -439,22 +447,20 @@ cat >>"$OUT_DIR/downstream-tidb-$KEYSPACE_NAME.toml" <<EOF
keyspace-name = "$KEYSPACE_NAME"
enable-telemetry = false
mem-quota-query = 671088640
run-auto-analyze = false
server-memory-quota = 2684354560
analyze-always-skip-wide-columns = true
max-server-connections = 0
# temp-dir = "$TEST_DATA_DIR/downstream/tidb-$KEYSPACE_NAME"

[log.file]
max-backups = 100

[performance]
tcp-keep-alive = true
run-auto-analyze = false
server-memory-quota = 2684354560

[security]
enable-sem = false

[instance]
max-server-connections = 0
EOF

echo "Starting upstream system TiDB"
Expand Down
20 changes: 10 additions & 10 deletions tests/integration_tests/_utils/start_tls_tidb_cluster_impl_nextgen
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,26 +115,29 @@ key-path = "$TLS_DIR/server-key.pem"
EOF

# system tidb server config file
# NOTE: TiDB config keys are section-sensitive. Keep `socket` and
# `max-server-connections` at root level, and keep tuning knobs like
# `run-auto-analyze` / `server-memory-quota` under `[performance]`. Otherwise
# TiDB will report "invalid configuration options" and silently ignore them.
cat - >"$OUT_DIR/tidb-system-config-tls.toml" <<EOF
socket = "/tmp/tidb-system-tls.sock"
keyspace-name = "SYSTEM"
enable-telemetry = false
mem-quota-query = 671088640
new_collations_enabled_on_first_bootstrap = true
run-auto-analyze = false
server-memory-quota = 2684354560
analyze-always-skip-wide-columns = true
max-server-connections = 0
tikv-worker-url = "http://$UP_TIKV_WORKER_HOST:$UP_TIKV_WORKER_PORT"

[log.file]
max-backups = 100

[performance]
tcp-keep-alive = true
run-auto-analyze = false
server-memory-quota = 2684354560

[instance]
tidb_service_scope = 'dxf_service'
max-server-connections = 0

[security]
enable-sem = false
Expand All @@ -152,18 +155,15 @@ keyspace-name = "$KEYSPACE_NAME"
enable-telemetry = false
mem-quota-query = 671088640
new_collations_enabled_on_first_bootstrap = true
run-auto-analyze = false
server-memory-quota = 2684354560
analyze-always-skip-wide-columns = true
max-server-connections = 0

[log.file]
max-backups = 100

[performance]
tcp-keep-alive = true

[instance]
max-server-connections = 0
run-auto-analyze = false
server-memory-quota = 2684354560

[security]
enable-sem = false
Expand Down
4 changes: 4 additions & 0 deletions tests/integration_tests/_utils/stop_tidb_cluster
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

# cdc server is ran by binary cdc.test, kill cdc server first to avoid too much
# noise in cdc logs.
#
# Also kill sink consumer binaries started by integration tests to avoid leaking
# background processes across cases/groups.

PKILL="killall -q -w -s 9 "
if [ "$(uname)" == "Darwin" ]; then
Expand All @@ -11,6 +14,7 @@ fi
${PKILL} cdc.test || true
${PKILL} cdc || true
${PKILL} cdc_kafka_consumer || true
${PKILL} cdc_pulsar_consumer || true
${PKILL} cdc_storage_consumer || true
${PKILL} tidb-server || true
${PKILL} tikv-server || true
Expand Down