diff --git a/sp_PerfCheck/sp_PerfCheck.sql b/sp_PerfCheck/sp_PerfCheck.sql index 02b06787..2da0e013 100644 --- a/sp_PerfCheck/sp_PerfCheck.sql +++ b/sp_PerfCheck/sp_PerfCheck.sql @@ -50,6 +50,13 @@ ALTER PROCEDURE dbo.sp_PerfCheck ( @database_name sysname = NULL, /* Database to check, NULL for all user databases */ + @slow_read_ms decimal(10, 2) = 20.0, /* Flag data-file reads slower than this (ms); High at 5x */ + @slow_write_ms decimal(10, 2) = 20.0, /* Flag data-file writes slower than this (ms); High at 5x */ + @significant_wait_threshold_pct decimal(5, 2) = 10.0, /* Minimum % of uptime for a wait to be reported */ + @wait_high_pct decimal(5, 2) = 50.0, /* Resource wait at/above this % of uptime is High */ + @wait_medium_pct decimal(5, 2) = 20.0, /* Resource wait at/above this % of uptime is Medium */ + @memory_grant_warning integer = 100, /* Forced grants at/above this count are Medium */ + @memory_grant_critical integer = 10000, /* Forced grants at/above this count are High */ @help bit = 0, /*For helpfulness*/ @debug bit = 0, /* Print diagnostic messages */ @version varchar(30) = NULL OUTPUT, /* Returns version */ @@ -103,6 +110,13 @@ BEGIN WHEN N'@debug' THEN 'prints debug information during execution' WHEN N'@version' THEN 'returns the version number of the procedure' WHEN N'@version_date' THEN 'returns the date this version was released' + WHEN N'@slow_read_ms' THEN 'flag data-file reads slower than this many ms (High at 5x)' + WHEN N'@slow_write_ms' THEN 'flag data-file writes slower than this many ms (High at 5x)' + WHEN N'@significant_wait_threshold_pct' THEN 'minimum percent of uptime for a wait to be reported' + WHEN N'@wait_high_pct' THEN 'a resource wait at or above this percent of uptime is High priority' + WHEN N'@wait_medium_pct' THEN 'a resource wait at or above this percent of uptime is Medium priority' + WHEN N'@memory_grant_warning' THEN 'forced memory grants at or above this cumulative count are Medium' + WHEN N'@memory_grant_critical' THEN 'forced memory grants at or above this cumulative count are High' ELSE NULL END, valid_inputs = @@ -113,6 +127,13 @@ BEGIN WHEN N'@debug' THEN '0 or 1' WHEN N'@version' THEN 'OUTPUT parameter' WHEN N'@version_date' THEN 'OUTPUT parameter' + WHEN N'@slow_read_ms' THEN 'any positive number of milliseconds' + WHEN N'@slow_write_ms' THEN 'any positive number of milliseconds' + WHEN N'@significant_wait_threshold_pct' THEN 'any positive percentage' + WHEN N'@wait_high_pct' THEN 'any positive percentage' + WHEN N'@wait_medium_pct' THEN 'any positive percentage' + WHEN N'@memory_grant_warning' THEN 'any positive integer' + WHEN N'@memory_grant_critical' THEN 'any positive integer' ELSE NULL END, defaults = @@ -123,6 +144,13 @@ BEGIN WHEN N'@debug' THEN 'false' WHEN N'@version' THEN 'NULL' WHEN N'@version_date' THEN 'NULL' + WHEN N'@slow_read_ms' THEN '20.0' + WHEN N'@slow_write_ms' THEN '20.0' + WHEN N'@significant_wait_threshold_pct' THEN '10.0' + WHEN N'@wait_high_pct' THEN '50.0' + WHEN N'@wait_medium_pct' THEN '20.0' + WHEN N'@memory_grant_warning' THEN '100' + WHEN N'@memory_grant_critical' THEN '10000' ELSE NULL END FROM sys.all_parameters AS ap @@ -167,6 +195,27 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RETURN; END; + /* + Default any tuning threshold left NULL or negative back to its documented + value, so a caller can override only the ones they care about. A CASE on + ">= 0" catches both NULL (UNKNOWN) and negative in one shot. + */ + SELECT + @slow_read_ms = + CASE WHEN @slow_read_ms >= 0 THEN @slow_read_ms ELSE 20.0 END, + @slow_write_ms = + CASE WHEN @slow_write_ms >= 0 THEN @slow_write_ms ELSE 20.0 END, + @significant_wait_threshold_pct = + CASE WHEN @significant_wait_threshold_pct >= 0 THEN @significant_wait_threshold_pct ELSE 10.0 END, + @wait_high_pct = + CASE WHEN @wait_high_pct >= 0 THEN @wait_high_pct ELSE 50.0 END, + @wait_medium_pct = + CASE WHEN @wait_medium_pct >= 0 THEN @wait_medium_pct ELSE 20.0 END, + @memory_grant_warning = + CASE WHEN @memory_grant_warning >= 0 THEN @memory_grant_warning ELSE 100 END, + @memory_grant_critical = + CASE WHEN @memory_grant_critical >= 0 THEN @memory_grant_critical ELSE 10000 END; + /* Variable Declarations */ @@ -282,9 +331,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @size_difference_pct decimal(18, 2), @has_percent_growth bit, @has_fixed_growth bit, - /* Storage performance variables */ - @slow_read_ms decimal(10, 2) = 500.0, /* Threshold for slow reads (ms) */ - @slow_write_ms decimal(10, 2) = 500.0, /* Threshold for slow writes (ms) */ /* Set threshold for "slow" autogrowth (in ms) */ @slow_autogrow_ms integer = 1000, /* 1 second */ @trace_path nvarchar(260), @@ -292,7 +338,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Determine total waits, uptime, and significant waits */ @total_waits bigint, @uptime_ms bigint, - @significant_wait_threshold_pct decimal(5, 2) = 10.0, /* Only waits above 10% */ @significant_wait_threshold_avg decimal(10, 2) = 10.0, /* Or avg wait time > 10ms */ /* Threshold settings for stolen memory alert */ @buffer_pool_size_gb decimal(38, 2), @@ -972,14 +1017,21 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) SELECT check_id = 4101, - priority = 20, /* High: active memory spills */ + priority = + CASE + WHEN MAX(ders.forced_grant_count) >= @memory_grant_critical + THEN 20 /* High: heavy, sustained memory pressure */ + WHEN MAX(ders.forced_grant_count) >= @memory_grant_warning + THEN 30 /* Medium */ + ELSE 40 /* Low: a handful since startup, likely transient */ + END, category = N'Memory Pressure', finding = N'Memory-Starved Queries: Forced Grants', details = - N'dm_exec_query_resource_semaphores has ' + + N'dm_exec_query_resource_semaphores reports ' + CONVERT(nvarchar(10), MAX(ders.forced_grant_count)) + - N' forced memory grants. ' + - N'Queries are being forced to run with less memory than requested, which can cause spills to tempdb and poor performance.', + N' forced memory grants since startup. Queries ran with less memory than they asked for and spilled to tempdb. ' + + N'Review oversized grants (large sorts and hashes), query tuning, and max server memory.', url = N'https://erikdarling.com/sp_perfcheck/#MemoryStarved' FROM sys.dm_exec_query_resource_semaphores AS ders WHERE ders.forced_grant_count > 0 @@ -999,14 +1051,21 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) SELECT check_id = 4103, - priority = 20, /* High: queries can't get memory */ + priority = + CASE + WHEN MAX(ders.timeout_error_count) > 100 + THEN 20 /* High: queries repeatedly failing to get memory */ + WHEN MAX(ders.timeout_error_count) > 10 + THEN 30 /* Medium */ + ELSE 40 /* Low: a few since startup, likely transient */ + END, category = N'Memory Pressure', finding = N'Memory-Starved Queries: Grant Timeouts', details = - N'dm_exec_query_resource_semaphores has ' + + N'dm_exec_query_resource_semaphores reports ' + CONVERT(nvarchar(10), MAX(ders.timeout_error_count)) + - N' memory grant timeouts. ' + - N'Queries are waiting for memory for a long time and giving up.', + N' memory grant timeouts since startup. Queries waited a long time for a memory grant and gave up (error 8645). ' + + N'Review oversized grants, RESOURCE_SEMAPHORE waits, and max server memory.', url = N'https://erikdarling.com/sp_perfcheck/#MemoryStarved' FROM sys.dm_exec_query_resource_semaphores AS ders WHERE ders.timeout_error_count > 0 @@ -1403,6 +1462,60 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. N'' ); END; + + /* + Flag notable global trace flags with interpretation. The complete list + is in #server_info; here we call out the ones that change behavior + server-wide or are redundant on modern versions, with severity by how + much they matter. Benign flags (backup-message suppression, lightweight + profiling, deadlock logging) are left in the server-info list only. + */ + INSERT INTO + #results + ( + check_id, + priority, + category, + finding, + object_name, + details, + url + ) + SELECT + check_id = 1012, + priority = + CASE tf.trace_flag + WHEN 1211 THEN 20 /* High: disables lock escalation entirely */ + WHEN 3608 THEN 20 /* High: startup-only flag set globally */ + WHEN 3609 THEN 20 /* High: startup-only flag set globally */ + WHEN 1224 THEN 30 /* Medium */ + WHEN 834 THEN 30 /* Medium */ + ELSE 40 /* Low: notable but not dangerous */ + END, + category = N'Server Configuration', + finding = N'Notable Global Trace Flag', + object_name = N'TF ' + CONVERT(nvarchar(10), tf.trace_flag), + details = + N'Global trace flag ' + + CONVERT(nvarchar(10), tf.trace_flag) + + N' is enabled. ' + + CASE tf.trace_flag + WHEN 1211 THEN N'Disables lock escalation entirely, which can bloat lock memory and hurt concurrency. Almost never recommended; prefer 1224 if you truly must.' + WHEN 1224 THEN N'Disables count-based lock escalation (escalation still happens under memory pressure). Rarely necessary.' + WHEN 3608 THEN N'A startup-only flag (recover master only) that should not be set on a running production server.' + WHEN 3609 THEN N'A startup-only flag (skip tempdb creation) that should not be set on a running production server.' + WHEN 834 THEN N'Uses large-page allocations for the buffer pool. Can slow or block startup and interacts badly with columnstore; use deliberately.' + WHEN 4199 THEN N'Enables all query optimizer hotfixes globally, which can change plans server-wide. On 2016+ prefer the database-scoped QUERY_OPTIMIZER_HOTFIXES option.' + WHEN 8048 THEN N'Partitions memory objects per CPU to reduce spinlock contention. Only relevant on high-core NUMA servers.' + WHEN 1117 THEN N'Grows all files in a filegroup together. Redundant for tempdb on 2016+, where this is already the default.' + WHEN 1118 THEN N'Forces full-extent allocation. Redundant on 2016+, where this is already the default for tempdb.' + WHEN 2371 THEN N'Lowers the auto-update-statistics threshold. Redundant on 2016+ at compatibility level 130+.' + ELSE N'Review whether it is still needed.' + END, + url = N'https://erikdarling.com/sp_perfcheck/#TraceFlags' + FROM #trace_flags AS tf + WHERE tf.global = 1 + AND tf.trace_flag IN (1211, 1224, 3608, 3609, 834, 4199, 8048, 1117, 1118, 2371); END; /* Memory information - works on all platforms */ @@ -2068,7 +2181,17 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #wait_stats.wait_time_percent_of_uptime = (wait_time_ms * 100.0 / NULLIF(@uptime_ms, 0)); - /* Add only waits that represent >=10% of server uptime */ + /* + Surface the significant waits as a health-check diagnosis rather than a + flat list. Name each finding by what the wait means for the server + (Storage / Lock / Memory / TempDB / CPU / Log / Parallelism ...), put the + specific wait type and its plain-English meaning in object_name, and + calibrate severity by category: resource-pressure waits (locking, memory, + storage, tempdb, log, CPU) earn High by dominating uptime OR by long + average waits; parallelism is usually a cost threshold / MAXDOP symptom, + so it takes a very high share just to reach Medium; everything else stays + Low. SLEEP_TASK is collected for context but not surfaced as a finding. + */ INSERT INTO #results ( @@ -2080,41 +2203,85 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. details, url ) - SELECT TOP (10) /* Limit to top 10 most significant waits */ + SELECT TOP (10) /* the ten most significant waits */ 6001, priority = CASE - WHEN ws.wait_time_percent_of_uptime > 100 - THEN 20 /* High: >100% of uptime */ - WHEN ws.wait_time_percent_of_uptime > 75 - THEN 20 /* High: >75% of uptime */ - WHEN ws.wait_time_percent_of_uptime >= 50 - THEN 30 /* Medium: >=50% of uptime */ - ELSE 40 /* Low: >=10% of uptime */ + WHEN ws.category IN (N'Locking', N'Memory', N'I/O', N'TempDB Contention', N'Transaction Log', N'CPU') + THEN + CASE + WHEN ws.wait_time_percent_of_uptime >= @wait_high_pct + OR ws.avg_wait_ms >= 1000.0 + THEN 20 /* High: dominates uptime, or each wait is very long */ + WHEN ws.wait_time_percent_of_uptime >= @wait_medium_pct + OR ws.avg_wait_ms >= 250.0 + THEN 30 /* Medium */ + ELSE 40 /* Low */ + END + WHEN ws.category = N'Parallelism' + THEN + CASE + WHEN ws.wait_time_percent_of_uptime >= 100.0 + THEN 30 /* Medium at most: usually a cost threshold / MAXDOP symptom */ + ELSE 40 /* Low */ + END + ELSE 40 /* Low: network, query execution, stats, AG, throttling, other */ END, category = N'Wait Statistics', - finding = N'High Impact Wait Type', + finding = + CASE ws.category + WHEN N'I/O' THEN N'Storage-Related Waits' + WHEN N'Memory' THEN N'Memory-Related Waits' + WHEN N'Parallelism' THEN N'Parallelism Waits' + WHEN N'CPU' THEN N'CPU / Scheduling Waits' + WHEN N'TempDB Contention' THEN N'TempDB Contention Waits' + WHEN N'Locking' THEN N'Lock / Blocking Waits' + WHEN N'Transaction Log' THEN N'Transaction Log Waits' + WHEN N'Query Execution' THEN N'Query Execution Waits' + WHEN N'Network' THEN N'Network / Client Waits' + WHEN N'Availability Groups' THEN N'Availability Group Waits' + WHEN N'Azure SQL Throttling' THEN N'Azure SQL Throttling Waits' + WHEN N'Index Management' THEN N'Index Maintenance Waits' + WHEN N'Statistics' THEN N'Statistics Update Waits' + ELSE N'Other Significant Waits' + END, object_name = ws.wait_type + N' (' + - ws.category + + ws.description + N')', details = - N'Wait type: ' + + N'Wait type ' + ws.wait_type + - N' represents ' + + N' accounts for ' + CONVERT(nvarchar(10), CONVERT(decimal(10, 2), ws.wait_time_percent_of_uptime)) + N'% of server uptime (' + - CONVERT(nvarchar(20), CONVERT(decimal(10, 2), ws.wait_time_minutes)) + - N' minutes). ' + - N'Average wait: ' + + CONVERT(nvarchar(20), CONVERT(decimal(10, 2), ws.wait_time_hours)) + + N' hours), averaging ' + CONVERT(nvarchar(10), CONVERT(decimal(10, 2), ws.avg_wait_ms)) + N' ms per wait. ' + - N'Description: ' + - ws.description, + CASE ws.category + WHEN N'Locking' + THEN N'Time lost to blocking. Investigate long-running transactions and blocking chains; sp_PressureDetector shows live blockers.' + WHEN N'Memory' + THEN N'Queries are waiting on memory. Review max server memory and query memory grants; oversized grants force RESOURCE_SEMAPHORE waits.' + WHEN N'I/O' + THEN N'Reads are waiting on storage. Check the per-file latency findings and whether the working set fits in the buffer pool.' + WHEN N'TempDB Contention' + THEN N'Allocation-page contention in tempdb. Use equal-sized tempdb data files, and memory-optimized tempdb metadata on 2019+.' + WHEN N'Transaction Log' + THEN N'Commits are waiting on the log. Check log storage latency, transaction sizes, and log backup frequency.' + WHEN N'CPU' + THEN N'Scheduling pressure. Review parallelism settings and plan quality; THREADPOOL specifically means worker-thread exhaustion.' + WHEN N'Parallelism' + THEN N'Usually a cost threshold for parallelism / MAXDOP tuning issue rather than a problem in itself.' + WHEN N'Network' + THEN N'Usually client-side: the application consuming results slowly or a slow network, not the server.' + ELSE N'Description: ' + ws.description + N'.' + END, url = N'https://erikdarling.com/sp_perfcheck/#WaitStats' FROM #wait_stats AS ws - WHERE ws.wait_time_percent_of_uptime >= 10.0 /* Only include waits that are at least 10% of uptime */ + WHERE ws.wait_time_percent_of_uptime >= @significant_wait_threshold_pct AND ws.wait_type <> N'SLEEP_TASK' ORDER BY ws.wait_time_percent_of_uptime DESC; @@ -2840,9 +3007,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. check_id = 3001, priority = CASE - WHEN i.avg_read_latency_ms > @slow_read_ms * 2 - THEN 20 /* High: >1000ms is severe */ - ELSE 30 /* Medium: >500ms is significant */ + WHEN i.avg_read_latency_ms > @slow_read_ms * 5 + THEN 20 /* High: >100ms average reads (5x threshold) is bad storage */ + ELSE 30 /* Medium: over the 20ms threshold, worth investigating */ END, category = N'Storage Performance', finding = N'Slow Read Latency', @@ -2902,9 +3069,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. check_id = 3002, priority = CASE - WHEN i.avg_write_latency_ms > @slow_write_ms * 2 - THEN 20 /* High: >1000ms is severe */ - ELSE 30 /* Medium: >500ms is significant */ + WHEN i.avg_write_latency_ms > @slow_write_ms * 5 + THEN 20 /* High: >100ms average writes (5x threshold) is bad storage */ + ELSE 30 /* Medium: over the 20ms threshold, worth investigating */ END, category = N'Storage Performance', finding = N'Slow Write Latency', diff --git a/sp_PerfCheck/tests/run_tests.py b/sp_PerfCheck/tests/run_tests.py index 9b132fc2..8d1d605b 100644 --- a/sp_PerfCheck/tests/run_tests.py +++ b/sp_PerfCheck/tests/run_tests.py @@ -553,7 +553,7 @@ def inaccessible_database_tests(server, password, R): # metadata read sees a closed database. _sqlcmd(server, password, "ALTER DATABASE [%s] SET AUTO_CLOSE ON;" % db) _sqlcmd(server, password, - "ALTER DATABASE [%s] SET OFFLINE; " + "ALTER DATABASE [%s] SET OFFLINE WITH ROLLBACK IMMEDIATE; " "ALTER DATABASE [%s] SET ONLINE;" % (db, db)) out, err = run_perfcheck(server, password, "@database_name = N'%s'" % db) errs = find_sql_errors(out) + find_sql_errors(err) @@ -561,7 +561,7 @@ def inaccessible_database_tests(server, password, R): not errs, str(errs)) # OFFLINE is the more severe case, and the one a whole-instance run hits. - _sqlcmd(server, password, "ALTER DATABASE [%s] SET OFFLINE;" % db) + _sqlcmd(server, password, "ALTER DATABASE [%s] SET OFFLINE WITH ROLLBACK IMMEDIATE;" % db) out, err = run_perfcheck(server, password, "@database_name = N'%s'" % db) errs = find_sql_errors(out) + find_sql_errors(err) R.check(grp, "OFFLINE database scoped run does not crash (Msg 515)",