Gallery smart search diagnostic — phase 2 (auto_explain + cache stats)

README.md

Gallery smart search — phase 2 diagnostic

Captures the real query plan the server runs on your box, plus Postgres cache stats, so we can see why smart search is slow.

Run

# Download the scripts
git clone https://gist.github.com/Deeds67/75bb0e5b2c1443402454068adb0cd102.git gallery-search-diag
cd gallery-search-diag
chmod +x *.sh

# 1. Turn on query plan logging (restarts Postgres)
./enable-auto-explain.sh

# 2. With GALLERY_SEARCH_TIMING=true still set on the server,
#    run 3–4 smart searches in the UI.

# 3. Collect outputs
docker exec -i immich_postgres psql -U postgres -d immich < cache-stats.sql > cache-stats.txt
docker logs immich_postgres --since=10m > postgres.log
docker logs immich_server   --since=10m > server.log

Send cache-stats.txt, postgres.log, and server.log.

Cleanup

./disable-auto-explain.sh

Also remove GALLERY_SEARCH_TIMING=true from your env once we're done.

Container names default to immich_postgres and db immich. If yours differ, pass them: ./enable-auto-explain.sh my_pg my_db.

cache-stats.sql

-- Postgres memory config + how much of the CLIP index / tables is in the page cache.
CREATE EXTENSION IF NOT EXISTS pg_buffercache;

\echo === settings ===
SHOW shared_buffers;
SHOW work_mem;
SHOW effective_cache_size;

\echo === relation sizes ===
SELECT relname,
       pg_size_pretty(pg_relation_size(oid)) AS size
FROM pg_class
WHERE relname IN ('clip_index', 'smart_search', 'asset', 'asset_exif')
ORDER BY pg_relation_size(oid) DESC;

\echo === buffer cache (pct of relation held in shared_buffers) ===
SELECT c.relname,
       pg_size_pretty(count(*) * 8192) AS cached,
       round(100.0 * count(*) / NULLIF(pg_relation_size(c.oid) / 8192, 0), 1) AS pct_cached
FROM pg_buffercache b
JOIN pg_class c ON c.oid = b.relfilenode
WHERE c.relname IN ('clip_index', 'smart_search', 'asset', 'asset_exif')
GROUP BY c.relname, c.oid
ORDER BY pct_cached DESC;

disable-auto-explain.sh

#!/usr/bin/env bash
# Undo enable-auto-explain.sh. Requires a restart.
set -euo pipefail

CONTAINER="${1:-immich_postgres}"
DB="${2:-immich}"
PGDATA="/var/lib/postgresql/data"

echo "==> Removing postgresql.override.conf"
docker exec "$CONTAINER" rm -f "$PGDATA/postgresql.override.conf"

echo "==> Restoring postgresql.auto.conf from backup"
docker exec "$CONTAINER" bash -c "
  if [ -f $PGDATA/postgresql.auto.conf.gallery-diag-backup ]; then
    mv $PGDATA/postgresql.auto.conf.gallery-diag-backup $PGDATA/postgresql.auto.conf
  else
    echo 'no backup found — leaving auto.conf alone'
  fi
"

echo "==> Restarting $CONTAINER"
docker restart "$CONTAINER" >/dev/null
until docker exec "$CONTAINER" pg_isready -U postgres -d "$DB" >/dev/null 2>&1; do sleep 1; done

echo "==> Verifying"
docker exec "$CONTAINER" psql -U postgres -d "$DB" -c "SHOW shared_preload_libraries;"
echo "==> Done."

enable-auto-explain.sh

#!/usr/bin/env bash
# Enable auto_explain in the Gallery Postgres container.
# Works around Postgres 14's ALTER SYSTEM list-quoting bug by editing
# postgresql.auto.conf directly. Preserves existing libraries (vchord, vectors).
# Requires a restart of the Postgres container.
set -euo pipefail

CONTAINER="${1:-immich_postgres}"
DB="${2:-immich}"
PGDATA="/var/lib/postgresql/data"

CURRENT_LIBS="$(docker exec "$CONTAINER" psql -U postgres -d "$DB" -tAXc "SHOW shared_preload_libraries;")"

if [[ "$CURRENT_LIBS" == *auto_explain* ]]; then
  NEW_LIBS="$CURRENT_LIBS"
  echo "==> auto_explain already in shared_preload_libraries"
else
  NEW_LIBS="$CURRENT_LIBS, auto_explain"
  echo "==> shared_preload_libraries: $CURRENT_LIBS → $NEW_LIBS"
fi

echo "==> Backing up postgresql.auto.conf"
docker exec "$CONTAINER" bash -c "
  if [ ! -f $PGDATA/postgresql.auto.conf.gallery-diag-backup ]; then
    cp $PGDATA/postgresql.auto.conf $PGDATA/postgresql.auto.conf.gallery-diag-backup
  fi
"

echo "==> Patching postgresql.auto.conf (strips existing shared_preload_libraries, appends new)"
docker exec -i "$CONTAINER" bash -c "
  grep -v '^shared_preload_libraries' $PGDATA/postgresql.auto.conf.gallery-diag-backup > $PGDATA/postgresql.auto.conf
  echo \"shared_preload_libraries = '$NEW_LIBS'\" >> $PGDATA/postgresql.auto.conf
"

echo "==> Writing postgresql.override.conf (auto_explain thresholds)"
docker exec -i "$CONTAINER" bash -c "cat > $PGDATA/postgresql.override.conf" <<'CONF'
# Managed by Gallery auto_explain diagnostic. Remove with disable-auto-explain.sh.
auto_explain.log_min_duration = '2000ms'
auto_explain.log_analyze = on
auto_explain.log_buffers = on
auto_explain.log_nested_statements = on
CONF

echo "==> Restarting $CONTAINER"
docker restart "$CONTAINER" >/dev/null
until docker exec "$CONTAINER" pg_isready -U postgres -d "$DB" >/dev/null 2>&1; do sleep 1; done

echo "==> Verifying"
docker exec "$CONTAINER" psql -U postgres -d "$DB" -c "SHOW shared_preload_libraries;"
docker exec "$CONTAINER" psql -U postgres -d "$DB" -c "SHOW auto_explain.log_min_duration;"
echo "==> Done. Queries >2s will now be logged with EXPLAIN ANALYZE."