SQL SERVER: Batched identity-PK FILLFACTOR=100 rebuild runner (with backup pacing)

BatchedSqlIdentityWithFF.sql

/* Batched identity-PK FILLFACTOR=100 rebuild runner (with backup pacing)
 * GENERATED BY: *claude*
 *
 * PURPOSE
 *   Rebuilds every identity-keyed clustered index in the CURRENT database to
 *   FILLFACTOR = 100. Compression is NOT touched here — whatever compression
 *   level each index has today (NONE / ROW / PAGE) is preserved by omitting
 *   the DATA_COMPRESSION clause from the REBUILD. Handle PAGE compression in
 *   a separate script.
 *
 *   The candidate filter is purely based on fill_factor (1-99), so indexes
 *   that are already PAGE-compressed but at FF=90 are still picked up and
 *   rebuilt to FF=100.
 *
 *   Indexes are grouped into size-bounded batches; the script runs one batch
 *   at a time and waits for a backup between batches so the log drive never
 *   floods.
 *
 * HOW BATCHING WORKS
 *   Qualifying indexes are ordered by size (largest first) and greedily packed
 *   into batches whose running total does not exceed @batch_size_mb_target.
 *   A single index larger than the target gets its own batch.
 *
 * INTER-BATCH WAIT
 *   After each batch the script records the latest msdb.dbo.backupset entry
 *   for this database (any backup type: log, full, differential), then polls
 *   every 30 s until a NEWER backup entry appears OR @backup_wait_timeout_min
 *   elapses, whichever is first.
 *
 *   If the database is in SIMPLE/BULK_LOGGED recovery (log backup pacing is
 *   not applicable), the script falls back to a fixed inter-batch delay.
 *
 * USAGE
 *   1. Switch context to the target database:   USE [YourDatabase];
 *      (Run once per database you want to process.)
 *   2. Review parameters below.
 *   3. Run with @dry_run = 1 first to inspect the plan.
 *   4. Set @dry_run = 0 and run during an off-hours window.
 *
 * SAFETY
 *   - ONLINE = ON is used automatically when the server/edition supports it
 *     (Enterprise/Developer any version; Standard on SQL 2016 SP1+;
 *     Azure SQL DB / MI). On older Standard editions (e.g. SQL 2012 Standard),
 *     ONLINE = ON is not available and the script falls back to ONLINE = OFF
 *     per rebuild. OFFLINE rebuilds take a schema-modification lock on the
 *     table for the duration of that index rebuild — plan maintenance windows
 *     accordingly.
 *   - Per-index LOB detection: on SQL 2012/2014 (major < 13), ONLINE clustered
 *     rebuild is not supported when the table contains text / ntext / image
 *     columns. The script detects this per table and uses ONLINE = OFF for
 *     those specific rebuilds only; the rest continue online.
 *   - Each rebuild is its own auto-commit transaction.
 *   - Rebuild failures are logged and the run continues
 *     (unless @continue_on_error = 0).
 *   - The run stops after @stop_after_hours elapses so it never bleeds into
 *     business hours.
 *
 * ROLLBACK (per-index, if a rebuild causes regression)
 *   ALTER INDEX [<name>] ON [<schema>].[<table>] REBUILD WITH
 *     (FILLFACTOR = <previous value>, ONLINE = <ON|OFF per edition>);
 */

SET NOCOUNT ON;

-- =======================================================================
-- Parameters
-- =======================================================================
DECLARE
    @dry_run                        bit           = 1,       -- 1 = preview only, 0 = execute
    @batch_size_mb_target           int           = 20480,   -- ~20 GB reserved per batch
    @min_index_size_mb              int           = 0,       -- 0 = include all sizes; raise to skip tiny indexes
    @maxdop                         int           = NULL,    -- NULL = auto (half of visible cores, min 1, max 8); or set explicitly
    @backup_wait_timeout_min        int           = 30,      -- max wait for a backup
    @inter_batch_extra_sec          int           = 60,      -- settle wait after backup
    @stop_after_hours               decimal(5,2)  = 8.0,     -- total run budget, stop after N hours
    @continue_on_error              bit           = 1,       -- skip failing index and continue
    @simple_recovery_fixed_wait_min int           = 5;       -- SIMPLE/BULK_LOGGED fallback

DECLARE @db_name sysname         = DB_NAME();
DECLARE @recovery_model sysname;
SELECT  @recovery_model = recovery_model_desc FROM sys.databases WHERE database_id = DB_ID();

DECLARE @run_start datetime2 = SYSDATETIME();

-- =======================================================================
-- Detect ONLINE index rebuild support
--   EngineEdition: 2 = Standard (incl. Web, BI), 3 = Enterprise/Developer/Eval,
--                  4 = Express, 5 = Azure SQL DB, 8 = Managed Instance, 9 = Edge
--   Standard edition supports ONLINE rebuild only from SQL 2016 SP1 onward.
-- =======================================================================
DECLARE @product_version nvarchar(128) = CONVERT(nvarchar(128), SERVERPROPERTY('ProductVersion'));
DECLARE @product_level   nvarchar(128) = CONVERT(nvarchar(128), SERVERPROPERTY('ProductLevel'));
DECLARE @edition         nvarchar(128) = CONVERT(nvarchar(128), SERVERPROPERTY('Edition'));
DECLARE @engine_edition  int           = CONVERT(int, SERVERPROPERTY('EngineEdition'));
DECLARE @product_major   int           = CONVERT(int, PARSENAME(@product_version, 4));

DECLARE @online_available bit =
    CASE
        WHEN @engine_edition = 3                                   THEN 1  -- Enterprise / Developer / Evaluation
        WHEN @engine_edition IN (5, 8, 9)                          THEN 1  -- Azure SQL DB / MI / Edge
        WHEN @engine_edition = 2 AND @product_major >= 14          THEN 1  -- Standard 2017+
        WHEN @engine_edition = 2 AND @product_major  = 13
                                 AND @product_level <> 'RTM'       THEN 1  -- Standard 2016 SP1+
        ELSE 0
    END;

DECLARE @online_clause varchar(15) = CASE WHEN @online_available = 1 THEN 'ON' ELSE 'OFF' END;

-- =======================================================================
-- Auto-calculate MAXDOP if not explicitly set
--   Rule: half of visible online cores, floor of 1, cap of 8.
--   (Cap of 8 matches Microsoft's general guidance for single-NUMA workloads;
--    rebuild parallelism past that rarely helps and starves user queries.)
-- =======================================================================
DECLARE @cpu_count int;
SELECT @cpu_count = COUNT(*)
FROM   sys.dm_os_schedulers
WHERE  status = 'VISIBLE ONLINE';

IF @maxdop IS NULL OR @maxdop <= 0
BEGIN
    SET @maxdop =
        CASE
            WHEN @cpu_count <= 2                THEN 1
            WHEN (@cpu_count / 2) > 8           THEN 8
            ELSE (@cpu_count / 2)
        END;
END

RAISERROR('Identity-PK FF=100 runner starting on [%s] (recovery: %s). Existing compression preserved.',
          0, 1, @db_name, @recovery_model) WITH NOWAIT;
RAISERROR('Server: %s %s (major %d, engine edition %d). ONLINE rebuild available: %s. Cores visible: %d, MAXDOP: %d.',
          0, 1, @edition, @product_level, @product_major, @engine_edition, @online_clause, @cpu_count, @maxdop) WITH NOWAIT;

IF @online_available = 0
BEGIN
    RAISERROR(' ', 0, 1) WITH NOWAIT;
    RAISERROR('***********************************************************************', 0, 1) WITH NOWAIT;
    RAISERROR('WARNING: this server/edition does not support ONLINE index rebuilds.', 0, 1) WITH NOWAIT;
    RAISERROR('         All rebuilds will run with ONLINE = OFF.', 0, 1) WITH NOWAIT;
    RAISERROR('         Each rebuild takes a schema-modification lock on its table for', 0, 1) WITH NOWAIT;
    RAISERROR('         the duration of that index rebuild, blocking all readers and', 0, 1) WITH NOWAIT;
    RAISERROR('         writers against that table. Larger indexes = longer blocks.', 0, 1) WITH NOWAIT;
    RAISERROR('         Continuing anyway.', 0, 1) WITH NOWAIT;
    RAISERROR('***********************************************************************', 0, 1) WITH NOWAIT;
    RAISERROR(' ', 0, 1) WITH NOWAIT;
END

-- =======================================================================
-- 1. Build the work queue
-- =======================================================================
IF OBJECT_ID('tempdb..#queue') IS NOT NULL DROP TABLE #queue;
CREATE TABLE #queue (
    id                  int IDENTITY(1,1) PRIMARY KEY,
    schema_name         sysname        NOT NULL,
    table_name          sysname        NOT NULL,
    index_name          sysname        NOT NULL,
    size_mb             decimal(18,1)  NOT NULL,
    row_count           bigint         NOT NULL,
    current_compression varchar(10)    NOT NULL,
    online_mode         varchar(3)     NOT NULL,     -- 'ON' or 'OFF', set per-row at queue build time
    rebuild_sql         nvarchar(max)  NOT NULL,
    batch_number        int            NULL,
    status              varchar(20)    NOT NULL DEFAULT 'pending',
    started_at          datetime2      NULL,
    completed_at        datetime2      NULL,
    duration_sec        int            NULL,
    error_msg           nvarchar(4000) NULL
);

;WITH candidates AS (
    SELECT
        s.name AS schema_name,
        t.name AS table_name,
        i.name AS index_name,
        CAST(ps.reserved_page_count * 8.0 / 1024 AS decimal(18,1)) AS size_mb,
        ps.row_count AS row_count,
        CASE p.data_compression
             WHEN 0 THEN 'NONE'
             WHEN 1 THEN 'ROW'
             WHEN 2 THEN 'PAGE'
             ELSE CAST(p.data_compression AS varchar(10))
        END AS current_compression,
        -- Per-index ONLINE decision:
        --   1. If edition can't do ONLINE at all  -> OFF
        --   2. If SQL < 2016 AND the table has text/ntext/image columns,
        --      ONLINE clustered rebuild is blocked -> OFF for this one only
        --   3. Otherwise -> the auto-detected @online_clause
        CASE
            WHEN @online_available = 0 THEN 'OFF'
            WHEN @product_major < 13 AND EXISTS (
                SELECT 1
                FROM   sys.columns cc
                JOIN   sys.types   ty ON ty.user_type_id = cc.user_type_id
                WHERE  cc.object_id = t.object_id
                  AND  ty.name IN ('text','ntext','image')
            ) THEN 'OFF'
            ELSE @online_clause
        END AS online_mode
    FROM sys.indexes i
    JOIN sys.tables  t  ON t.object_id = i.object_id
    JOIN sys.schemas s  ON s.schema_id = t.schema_id
    JOIN sys.partitions p
            ON p.object_id = i.object_id
           AND p.index_id  = i.index_id
           AND p.partition_number = 1
    JOIN sys.dm_db_partition_stats ps
            ON ps.object_id = i.object_id
           AND ps.index_id  = i.index_id
           AND ps.partition_number = 1
    JOIN sys.columns c
            ON c.object_id = i.object_id
           AND c.is_identity = 1
    JOIN sys.index_columns ic
            ON ic.object_id = i.object_id
           AND ic.index_id  = i.index_id
           AND ic.column_id = c.column_id
           AND ic.key_ordinal = 1                       -- identity is the LEADING key
    WHERE i.type = 1                                     -- clustered
      AND i.fill_factor BETWEEN 1 AND 99                 -- 0 = default (100) already
      AND t.is_ms_shipped = 0
      AND ps.reserved_page_count * 8 >= @min_index_size_mb * 1024
)
INSERT INTO #queue (schema_name, table_name, index_name, size_mb, row_count,
                    current_compression, online_mode, rebuild_sql)
SELECT
    schema_name, table_name, index_name, size_mb, row_count,
    current_compression, online_mode,
    -- No DATA_COMPRESSION clause => preserve existing compression level.
    'ALTER INDEX ' + QUOTENAME(index_name)
      + ' ON ' + QUOTENAME(schema_name) + '.' + QUOTENAME(table_name)
      + ' REBUILD WITH ('
      + 'FILLFACTOR = 100'
      + ', ONLINE = ' + online_mode
      + ', SORT_IN_TEMPDB = ON'
      + ', MAXDOP = ' + CAST(@maxdop AS varchar(10))
      + ');'
FROM candidates;

DECLARE @total_indexes int, @total_gb decimal(18,2), @total_gb_str varchar(20);
SELECT  @total_indexes = COUNT(*),
        @total_gb      = ISNULL(SUM(size_mb)/1024.0, 0)
FROM    #queue;
SET     @total_gb_str  = CAST(@total_gb AS varchar(20));

IF @total_indexes = 0
BEGIN
    RAISERROR('No qualifying indexes in [%s]. Nothing to do.', 0, 1, @db_name) WITH NOWAIT;
    RETURN;
END

RAISERROR('Found %d qualifying indexes totalling %s GB.', 0, 1, @total_indexes, @total_gb_str) WITH NOWAIT;

-- If any indexes were forced to OFFLINE due to LOB columns on SQL < 2016, note it.
IF @online_available = 1
BEGIN
    DECLARE @lob_offline_count int;
    SELECT  @lob_offline_count = COUNT(*) FROM #queue WHERE online_mode = 'OFF';
    IF @lob_offline_count > 0
    BEGIN
        RAISERROR('Note: %d index(es) have text/ntext/image columns and will be rebuilt OFFLINE on this SQL version; remaining %d will use ONLINE = ON.',
                  0, 1, @lob_offline_count, @total_indexes - @lob_offline_count) WITH NOWAIT;
    END
END

-- =======================================================================
-- 2. Assign batches (greedy first-fit-decreasing by size)
-- =======================================================================
DECLARE
    @id             int,
    @size           decimal(18,1),
    @current_batch  int           = 1,
    @current_mb     decimal(18,1) = 0;

DECLARE pack_cur CURSOR LOCAL FAST_FORWARD FOR
    SELECT id, size_mb FROM #queue ORDER BY size_mb DESC, id;
OPEN pack_cur;
FETCH NEXT FROM pack_cur INTO @id, @size;
WHILE @@FETCH_STATUS = 0
BEGIN
    IF @current_mb > 0 AND (@current_mb + @size) > @batch_size_mb_target
    BEGIN
        SET @current_batch += 1;
        SET @current_mb     = 0;
    END
    UPDATE #queue SET batch_number = @current_batch WHERE id = @id;
    SET @current_mb += @size;
    FETCH NEXT FROM pack_cur INTO @id, @size;
END
CLOSE pack_cur; DEALLOCATE pack_cur;

DECLARE @max_batch int;
SELECT  @max_batch = MAX(batch_number) FROM #queue;

RAISERROR('Planned %d batches, target <= %d MB each.', 0, 1, @max_batch, @batch_size_mb_target) WITH NOWAIT;

-- Plan summary
SELECT batch_number,
       COUNT(*)                                AS indexes,
       CAST(SUM(size_mb) AS decimal(18,1))     AS batch_mb,
       CAST(SUM(size_mb)/1024.0 AS decimal(18,2)) AS batch_gb
FROM   #queue
GROUP BY batch_number
ORDER BY batch_number;

IF @dry_run = 1
BEGIN
    RAISERROR('DRY RUN: plan above. Set @dry_run = 0 and re-run to execute.', 0, 1) WITH NOWAIT;
    SELECT batch_number, id, schema_name, table_name, index_name,
           size_mb, row_count, current_compression, online_mode, rebuild_sql
    FROM   #queue
    ORDER BY batch_number, size_mb DESC;
    RETURN;
END

-- =======================================================================
-- 3. Execute batches
-- =======================================================================
DECLARE
    @batch       int           = 1,
    @last_bk     datetime,
    @new_bk      datetime,
    @sql         nvarchar(max),
    @cur_sch     sysname,
    @cur_tab     sysname,
    @cur_idx     sysname,
    @cur_size    decimal(18,1),
    @cur_size_s  varchar(20),
    @started     datetime2,
    @errmsg      nvarchar(4000),
    @wait_start  datetime2,
    @wait_str    varchar(10);

WHILE @batch <= @max_batch
BEGIN
    -- total-run time budget
    IF DATEDIFF(minute, @run_start, SYSDATETIME()) > @stop_after_hours * 60
    BEGIN
        DECLARE @budget_str varchar(20) = CAST(@stop_after_hours AS varchar(20));
        RAISERROR('Time budget (%s h) exceeded. Stopping before batch %d.', 0, 1, @budget_str, @batch) WITH NOWAIT;
        BREAK;
    END

    RAISERROR(' ', 0, 1) WITH NOWAIT;
    RAISERROR('=== Batch %d / %d ===', 0, 1, @batch, @max_batch) WITH NOWAIT;

    -- baseline backup timestamp (any backup type)
    IF @recovery_model = 'FULL'
    BEGIN
        SELECT @last_bk = MAX(backup_finish_date)
        FROM   msdb.dbo.backupset
        WHERE  database_name = @db_name
          AND  type IN ('L','D','I');               -- Log, Full DB, Differential
    END

    DECLARE idx_cur CURSOR LOCAL FAST_FORWARD FOR
        SELECT id, rebuild_sql, schema_name, table_name, index_name, size_mb
        FROM   #queue
        WHERE  batch_number = @batch
          AND  status       = 'pending'
        ORDER BY size_mb DESC, id;
    OPEN idx_cur;
    FETCH NEXT FROM idx_cur INTO @id, @sql, @cur_sch, @cur_tab, @cur_idx, @cur_size;
    WHILE @@FETCH_STATUS = 0
    BEGIN
        SET @started    = SYSDATETIME();
        SET @cur_size_s = CAST(@cur_size AS varchar(20));
        UPDATE #queue SET status = 'running', started_at = @started WHERE id = @id;

        RAISERROR('  [%s].[%s].[%s] (%s MB)...', 0, 1, @cur_sch, @cur_tab, @cur_idx, @cur_size_s) WITH NOWAIT;

        BEGIN TRY
            EXEC sp_executesql @sql;
            UPDATE #queue
            SET    status       = 'done',
                   completed_at = SYSDATETIME(),
                   duration_sec = DATEDIFF(second, @started, SYSDATETIME())
            WHERE  id = @id;
            RAISERROR('     done in %d sec.', 0, 1, DATEDIFF(second, @started, SYSDATETIME())) WITH NOWAIT;
        END TRY
        BEGIN CATCH
            SET @errmsg = ERROR_MESSAGE();
            UPDATE #queue
            SET    status       = 'failed',
                   completed_at = SYSDATETIME(),
                   duration_sec = DATEDIFF(second, @started, SYSDATETIME()),
                   error_msg    = @errmsg
            WHERE  id = @id;
            RAISERROR('     FAILED: %s', 0, 1, @errmsg) WITH NOWAIT;
            IF @continue_on_error = 0
            BEGIN
                CLOSE idx_cur; DEALLOCATE idx_cur;
                RAISERROR('continue_on_error = 0; aborting run.', 16, 1) WITH NOWAIT;
                RETURN;
            END
        END CATCH

        FETCH NEXT FROM idx_cur INTO @id, @sql, @cur_sch, @cur_tab, @cur_idx, @cur_size;
    END
    CLOSE idx_cur; DEALLOCATE idx_cur;

    -- inter-batch wait (skip after final batch)
    IF @batch < @max_batch
    BEGIN
        IF @recovery_model = 'FULL'
        BEGIN
            SET @wait_start = SYSDATETIME();
            RAISERROR('  waiting for next backup (baseline %s)...', 0, 1,
                      ISNULL(CONVERT(varchar(23), @last_bk, 121), '<none>')) WITH NOWAIT;

            WHILE 1 = 1
            BEGIN
                SELECT @new_bk = MAX(backup_finish_date)
                FROM   msdb.dbo.backupset
                WHERE  database_name = @db_name
                  AND  type IN ('L','D','I');

                IF @new_bk > ISNULL(@last_bk, '19000101')
                BEGIN
                    RAISERROR('  backup detected at %s.', 0, 1, CONVERT(varchar(23), @new_bk, 121)) WITH NOWAIT;
                    BREAK;
                END
                IF DATEDIFF(minute, @wait_start, SYSDATETIME()) >= @backup_wait_timeout_min
                BEGIN
                    RAISERROR('  timeout (%d min) waiting for backup. Continuing.', 0, 1, @backup_wait_timeout_min) WITH NOWAIT;
                    BREAK;
                END
                WAITFOR DELAY '00:00:30';
            END

            IF @inter_batch_extra_sec > 0
            BEGIN
                SET @wait_str = CONVERT(varchar(8),
                                        DATEADD(second, @inter_batch_extra_sec, CONVERT(datetime, 0)),
                                        108);
                WAITFOR DELAY @wait_str;
            END
        END
        ELSE
        BEGIN
            SET @wait_str = CONVERT(varchar(8),
                                    DATEADD(minute, @simple_recovery_fixed_wait_min, CONVERT(datetime, 0)),
                                    108);
            RAISERROR('  recovery model %s: fixed wait %d min between batches.', 0, 1,
                      @recovery_model, @simple_recovery_fixed_wait_min) WITH NOWAIT;
            WAITFOR DELAY @wait_str;
        END
    END

    SET @batch += 1;
END

-- =======================================================================
-- 4. Summary / after-measurements
-- =======================================================================
RAISERROR(' ', 0, 1) WITH NOWAIT;
RAISERROR('=== Run complete ===', 0, 1) WITH NOWAIT;

-- Roll-up by status
SELECT status,
       COUNT(*)                                   AS idx_count,
       CAST(SUM(size_mb)/1024.0 AS decimal(18,2)) AS total_gb,
       SUM(duration_sec)                          AS total_sec
FROM   #queue
GROUP BY status
ORDER BY status;

-- Per-index detail
SELECT batch_number, id, schema_name, table_name, index_name,
       size_mb, row_count, current_compression, online_mode, status,
       started_at, completed_at, duration_sec, error_msg
FROM   #queue
ORDER BY batch_number, size_mb DESC;

-- Page Life Expectancy right now (should improve over the following hours
-- as buffer pool refills with the compressed, densely-packed pages).
SELECT object_name, counter_name, cntr_value AS ple_seconds_now
FROM   sys.dm_os_performance_counters
WHERE  counter_name = 'Page life expectancy'
  AND  object_name LIKE '%Buffer Manager%';