stan/website
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases Wiki Activity

db-refresh-dev: own dev DB as personal_dev, no prod grants; PG_CONTAINER mode
All checks were successful
Deploy production / deploy (push) Successful in 3m49s
Details

Browse Source
This commit is contained in:
Stan Runge
2026-10-23 22:27:17 +00:00
parent 36af5675e9
commit 45f2bbb198
1 changed files with 52 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
scripts/db-refresh-dev.sh
View File

@@ -1,37 +1,58 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Refresh the local development database from a fresh production dump. # Refresh the development database (personal_dev) from a fresh production dump.
# #
# Prod is the source of truth for data; dev is a disposable copy of it. # Prod is the source of truth for data; dev is a disposable copy of it.
# #
# Required env (put them in .env.dev-refresh or export them; never commit real values): # Required env (put them in .env.dev-refresh or export them; never commit real values):
# PROD_DB_URL read access to prod, e.g. postgresql://personal:...@127.0.0.1:5440/personal # PROD_DB_URL read access to prod, e.g. postgresql://personal:...@127.0.0.1:5440/personal
# DEV_ADMIN_URL superuser on the dev cluster, maintenance DB, e.g. postgresql://postgres:...@127.0.0.1:5432/postgres # DEV_ADMIN_URL superuser on the dev cluster, maintenance DB, e.g. postgresql://postgres:...@127.0.0.1:5440/postgres
# (not needed with PG_CONTAINER, which uses the container's local superuser)
# Optional: # Optional:
# DEV_DB_NAME target database (default personal_dev) # DEV_DB_NAME target database (default personal_dev)
# DEV_OWNER role that owns every object in the dev DB (default personal_dev). It is the
# role the dev site logs in as, so it can also run migrations there. It must not
# be a member of any prod role; prod's CONNECT is not granted to it.
# PG_DOCKER=1 use the postgres:17-alpine image instead of local pg_dump/pg_restore # PG_DOCKER=1 use the postgres:17-alpine image instead of local pg_dump/pg_restore
# PG_CONTAINER name of the Postgres container (e.g. postgres-db-1): run every tool inside it
# via `docker exec` as the local superuser. URLs are then as seen from inside
# the container, e.g. PROD_DB_URL=postgresql://personal:...@127.0.0.1:5432/personal
# DOCKER docker command (default "docker"; e.g. "sudo -n docker")
# #
# What it does: dump prod (custom format) -> drop + recreate DEV_DB_NAME -> create # What it does: dump prod (custom format) -> drop + recreate DEV_DB_NAME owned by DEV_OWNER ->
# missing roles (NOLOGIN) -> restore with ownership/grants intact -> clear sessions # restore without prod ownership/grants, every object owned by DEV_OWNER -> clear sessions and
# and verification tokens so prod logins can't be reused in dev. It never writes to prod. # verification tokens so prod logins can't be reused in dev. It never writes to prod.
set -euo pipefail set -euo pipefail
cd "$(dirname "$0")/.." cd "$(dirname "$0")/.."
if [[ -f .env.dev-refresh ]]; then set -a; . ./.env.dev-refresh; set +a; fi if [[ -f .env.dev-refresh ]]; then set -a; . ./.env.dev-refresh; set +a; fi
: "${PROD_DB_URL:?set PROD_DB_URL (source, read-only use)}" : "${PROD_DB_URL:?set PROD_DB_URL (source, read-only use)}"
: "${DEV_ADMIN_URL:?set DEV_ADMIN_URL (superuser URL to the dev cluster maintenance DB)}"
DEV_DB_NAME="${DEV_DB_NAME:-personal_dev}" DEV_DB_NAME="${DEV_DB_NAME:-personal_dev}"
DEV_OWNER="${DEV_OWNER:-personal_dev}"
DOCKER="${DOCKER:-docker}"
if [[ -n "${PG_CONTAINER:-}" ]]; then
DEV_ADMIN_URL="${DEV_ADMIN_URL:-postgresql://postgres@localhost/postgres}"
fi
: "${DEV_ADMIN_URL:?set DEV_ADMIN_URL (superuser URL to the dev cluster maintenance DB)}"
if [[ ! "$DEV_DB_NAME" =~ ^[a-z_][a-z0-9_]*$ ]]; then for ident in "$DEV_DB_NAME" "$DEV_OWNER"; do
echo "DEV_DB_NAME must be a plain lowercase identifier" >&2 if [[ ! "$ident" =~ ^[a-z_][a-z0-9_]*$ ]]; then
echo "DEV_DB_NAME / DEV_OWNER must be plain lowercase identifiers" >&2
exit 1 exit 1
fi fi
done
case "$DEV_DB_NAME" in case "$DEV_DB_NAME" in
personal | personal_prod | postgres | template0 | template1) personal | personal_prod | postgres | template0 | template1)
echo "Refusing to overwrite '$DEV_DB_NAME'" >&2 echo "Refusing to overwrite '$DEV_DB_NAME'" >&2
exit 1 exit 1
;; ;;
esac esac
case "$DEV_OWNER" in
personal | personal_app | personal_ddl | personal_ro | postgres)
echo "Refusing: DEV_OWNER '$DEV_OWNER' is a prod/superuser role" >&2
exit 1
;;
esac
# host:port/db of a URL (no credentials), for the "same database" guard. # host:port/db of a URL (no credentials), for the "same database" guard.
endpoint() { bun -e 'const u=new URL(process.argv[1]); console.log(`${u.hostname}:${u.port||5432}${u.pathname}`)' "$1"; } endpoint() { bun -e 'const u=new URL(process.argv[1]); console.log(`${u.hostname}:${u.port||5432}${u.pathname}`)' "$1"; }
@@ -44,8 +65,12 @@ if [[ "$(endpoint "$dev_url")" == "$prod_endpoint" ]]; then
exit 1 exit 1
fi fi
if [[ "${PG_DOCKER:-0}" == 1 ]]; then # Dump goes to stdout and restore reads stdin, so the same commands work locally,
pg() { docker run --rm -i --network host -v "$work:$work" postgres:17-alpine "$@"; } # in a throwaway postgres image, or inside the Postgres container.
if [[ -n "${PG_CONTAINER:-}" ]]; then
pg() { $DOCKER exec -i -u postgres "$PG_CONTAINER" "$@"; }
elif [[ "${PG_DOCKER:-0}" == 1 ]]; then
pg() { $DOCKER run --rm -i --network host postgres:17-alpine "$@"; }
else else
pg() { "$@"; } pg() { "$@"; }
fi fi
@@ -55,27 +80,30 @@ trap 'rm -rf "$work"' EXIT
dump="$work/prod.dump" dump="$work/prod.dump"
echo "Dumping prod ($prod_endpoint)..." echo "Dumping prod ($prod_endpoint)..."
pg pg_dump --format=custom --no-password --file="$dump" "$PROD_DB_URL" pg pg_dump --format=custom --no-password "$PROD_DB_URL" >"$dump"
[[ -s "$dump" ]] || { echo "Empty dump; aborting before touching $DEV_DB_NAME" >&2; exit 1; }
pg pg_restore --list <"$dump" >/dev/null
echo "Recreating $DEV_DB_NAME..." echo "Recreating $DEV_DB_NAME (owner $DEV_OWNER)..."
pg psql -v ON_ERROR_STOP=1 -q "$DEV_ADMIN_URL" <<SQL pg psql -v ON_ERROR_STOP=1 -q "$DEV_ADMIN_URL" <<SQL
SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$DEV_DB_NAME' AND pid <> pg_backend_pid(); SELECT count(pg_terminate_backend(pid)) AS terminated_sessions FROM pg_stat_activity WHERE datname = '$DEV_DB_NAME' AND pid <> pg_backend_pid();
DROP DATABASE IF EXISTS "$DEV_DB_NAME"; DROP DATABASE IF EXISTS "$DEV_DB_NAME";
DO \$\$ DO \$\$
DECLARE r text;
BEGIN BEGIN
FOREACH r IN ARRAY ARRAY['personal','personal_app','personal_ddl','personal_ro'] LOOP IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '$DEV_OWNER') THEN
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r) THEN EXECUTE format('CREATE ROLE %I NOLOGIN', '$DEV_OWNER');
EXECUTE format('CREATE ROLE %I NOLOGIN', r); RAISE NOTICE 'created role $DEV_OWNER (NOLOGIN; give it a password + LOGIN for dev use)';
RAISE NOTICE 'created role % (NOLOGIN; give it a password + LOGIN for dev use)', r;
END IF; END IF;
END LOOP;
END \$\$; END \$\$;
CREATE DATABASE "$DEV_DB_NAME" OWNER personal; CREATE DATABASE "$DEV_DB_NAME" OWNER "$DEV_OWNER";
REVOKE CONNECT, TEMPORARY ON DATABASE "$DEV_DB_NAME" FROM PUBLIC;
SQL SQL
echo "Restoring into $DEV_DB_NAME..." echo "Restoring into $DEV_DB_NAME..."
pg pg_restore --no-password --exit-on-error --dbname="$dev_url" "$dump" # --no-owner/--no-privileges + --role: everything ends up owned by DEV_OWNER and none of
# prod's grants (personal_app, personal_ro, ...) are copied into dev.
pg pg_restore --no-password --exit-on-error --no-owner --no-privileges --role="$DEV_OWNER" \
--dbname="$dev_url" <"$dump"
echo "Clearing sessions and verification tokens..." echo "Clearing sessions and verification tokens..."
pg psql -v ON_ERROR_STOP=1 -q "$dev_url" -c 'TRUNCATE auth.session, auth.verification' pg psql -v ON_ERROR_STOP=1 -q "$dev_url" -c 'TRUNCATE auth.session, auth.verification'