424 lines
15 KiB
Bash
Executable File
424 lines
15 KiB
Bash
Executable File
#!/bin/bash
|
|
# ============================================================
|
|
# PX360 Database Sync Script
|
|
# ============================================================
|
|
# Copies the local Docker PostgreSQL database + media files
|
|
# to a production DB server.
|
|
#
|
|
# Usage:
|
|
# ./scripts/db_sync.sh # Dry-run: dump locally only
|
|
# ./scripts/db_sync.sh --deploy user@host # Full: dump → scp → restore
|
|
#
|
|
# Prerequisites:
|
|
# - Docker containers running locally (px360_db)
|
|
# - SSH access to the DB server
|
|
# - PostgreSQL client tools on the DB server (pg_restore, createdb)
|
|
# - .env.production file with DB_HOST, DB_USER, DB_NAME
|
|
# ============================================================
|
|
|
|
set -euo pipefail
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Paths
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
|
BACKUP_DIR="$PROJECT_DIR/backups"
|
|
|
|
# Default config (overridden by .env.production if available)
|
|
LOCAL_DB_CONTAINER="px360_db"
|
|
LOCAL_DB_USER="px360"
|
|
LOCAL_DB_NAME="px360"
|
|
|
|
PROD_DB_HOST=""
|
|
PROD_DB_USER="px360"
|
|
PROD_DB_NAME="px360"
|
|
PROD_SSH=""
|
|
|
|
# Tables to exclude from dump (dev-only data)
|
|
EXCLUDE_TABLES=(
|
|
"django_session"
|
|
"simulator_hisrequestlog"
|
|
"simulator_emaillog"
|
|
"simulator_smslog"
|
|
)
|
|
|
|
# Flags
|
|
DEPLOY=false
|
|
CONTAINER="" # DB container name (e.g. hh-dev-db-1) → restore via docker exec
|
|
MEDIA_CONTAINER="" # app container name (e.g. hh-dev-web_app) → media restore via docker exec
|
|
PROJECT="" # compose project name (e.g. hh-dev) → for stop/start app containers
|
|
NO_MEDIA=false # skip media dump+restore
|
|
|
|
# ============================================================
|
|
# Parse arguments
|
|
# ============================================================
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--deploy)
|
|
DEPLOY=true
|
|
PROD_SSH="$2"
|
|
shift 2
|
|
;;
|
|
--container)
|
|
CONTAINER="$2"
|
|
shift 2
|
|
;;
|
|
--media-container)
|
|
MEDIA_CONTAINER="$2"
|
|
shift 2
|
|
;;
|
|
--project)
|
|
PROJECT="$2"
|
|
shift 2
|
|
;;
|
|
--no-media)
|
|
NO_MEDIA=true
|
|
shift
|
|
;;
|
|
--help|-h)
|
|
echo "Usage: ./scripts/db_sync.sh [options] [--deploy user@host]"
|
|
echo ""
|
|
echo " Without --deploy: Creates a local dump only (dry-run)"
|
|
echo " With --deploy: Dumps locally, copies to prod DB server, restores"
|
|
echo ""
|
|
echo "Containerized target (DB runs in a Docker container on the server):"
|
|
echo " --container <name> DB container (e.g. hh-dev-db-1)"
|
|
echo " --media-container <n> app container for media (e.g. hh-dev-web_app)"
|
|
echo " --project <name> compose project (e.g. hh-dev) for stop/start app"
|
|
echo " --no-media skip media dump+restore"
|
|
echo ""
|
|
echo "Examples:"
|
|
echo " ./scripts/db_sync.sh # local dump"
|
|
echo " ./scripts/db_sync.sh --deploy root@10.10.1.100 # host-installed pg tools"
|
|
echo " ./scripts/db_sync.sh --deploy root@host \\\\"
|
|
echo " --container hh-dev-db-1 --media-container hh-dev-web_app --project hh-dev"
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo -e "${RED}Unknown option: $1${NC}"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# ============================================================
|
|
# Load production config
|
|
# ============================================================
|
|
if [ "$DEPLOY" = true ]; then
|
|
if [ -z "$PROD_SSH" ]; then
|
|
echo -e "${RED}Error: --deploy requires SSH target (user@host)${NC}"
|
|
exit 1
|
|
fi
|
|
|
|
ENV_FILE="$PROJECT_DIR/.env.production"
|
|
# In container mode the DB lives in the container; no DB_HOST needed.
|
|
# Only read .env.production for host-mode (no --container) or if it exists.
|
|
if [ -z "$CONTAINER" ] && [ -f "$ENV_FILE" ]; then
|
|
PROD_DB_HOST=$(grep -E "^DB_HOST=" "$ENV_FILE" | cut -d= -f2 || echo "")
|
|
PROD_DB_USER=$(grep -E "^DB_USER=" "$ENV_FILE" | cut -d= -f2 || echo "px360")
|
|
PROD_DB_NAME=$(grep -E "^DB_NAME=" "$ENV_FILE" | cut -d= -f2 || echo "px360")
|
|
if [ -z "$PROD_DB_HOST" ]; then
|
|
PROD_DB_HOST=$(grep -E "^DATABASE_URL=" "$ENV_FILE" | sed -E 's/.*@([^:]+):.*/\1/' || echo "")
|
|
fi
|
|
fi
|
|
|
|
if [ -z "$CONTAINER" ] && [ -z "$PROD_DB_HOST" ]; then
|
|
echo -e "${YELLOW}Warning: DB_HOST not found in .env.production${NC}"
|
|
echo -e "Enter DB server IP/hostname: "
|
|
read -r PROD_DB_HOST
|
|
fi
|
|
|
|
# Derive compose project from media container (target names it <project>-web_app) if not given
|
|
if [ -n "$MEDIA_CONTAINER" ] && [ -z "$PROJECT" ]; then
|
|
PROJECT="${MEDIA_CONTAINER%-web_app}"
|
|
fi
|
|
|
|
echo -e "${BLUE}Production target:${NC}"
|
|
echo -e " SSH: $PROD_SSH"
|
|
if [ -n "$CONTAINER" ]; then
|
|
echo -e " DB container: $CONTAINER"
|
|
echo -e " Media target: ${MEDIA_CONTAINER:-<none>}"
|
|
echo -e " Project: ${PROJECT:-<none>}"
|
|
else
|
|
echo -e " DB Host: $PROD_DB_HOST"
|
|
fi
|
|
echo -e " DB User: $PROD_DB_USER"
|
|
echo -e " DB Name: $PROD_DB_NAME"
|
|
echo ""
|
|
echo -e "${YELLOW}This will OVERWRITE the target database data. Continue? (y/N)${NC}"
|
|
read -r CONFIRM
|
|
if [[ ! "$CONFIRM" =~ ^[Yy]$ ]]; then
|
|
echo "Aborted."
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
# ============================================================
|
|
# Setup
|
|
# ============================================================
|
|
mkdir -p "$BACKUP_DIR"
|
|
|
|
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
|
DB_DUMP="$BACKUP_DIR/px360_${TIMESTAMP}.dump"
|
|
MEDIA_DUMP="$BACKUP_DIR/media_${TIMESTAMP}.tar.gz"
|
|
|
|
echo ""
|
|
echo -e "${BLUE}========================================${NC}"
|
|
echo -e "${BLUE} PX360 Database Sync${NC}"
|
|
echo -e "${BLUE} $(date '+%Y-%m-%d %H:%M:%S')${NC}"
|
|
echo -e "${BLUE}========================================${NC}"
|
|
echo ""
|
|
|
|
# ============================================================
|
|
# Step 1: Dump local database
|
|
# ============================================================
|
|
echo -e "${BLUE}[1/5] Dumping local database...${NC}"
|
|
|
|
# Build exclude flags
|
|
EXCLUDE_FLAGS=""
|
|
for table in "${EXCLUDE_TABLES[@]}"; do
|
|
EXCLUDE_FLAGS="$EXCLUDE_FLAGS --exclude-table-data=$table"
|
|
done
|
|
|
|
# Check if Docker DB is running
|
|
if ! docker ps --format '{{.Names}}' | grep -q "^${LOCAL_DB_CONTAINER}$"; then
|
|
echo -e "${YELLOW}Docker DB container not found. Trying local PostgreSQL...${NC}"
|
|
# Fall back to local pg_dump (for SQLite → PG migration or local PG)
|
|
if command -v pg_dump &> /dev/null; then
|
|
PGPASSWORD="${LOCAL_DB_USER}" pg_dump -U "$LOCAL_DB_USER" -d "$LOCAL_DB_NAME" \
|
|
--no-owner --no-privileges -Fc $EXCLUDE_FLAGS \
|
|
-f "$DB_DUMP" 2>/dev/null || {
|
|
echo -e "${RED}Failed to dump database. Neither Docker nor local PostgreSQL available.${NC}"
|
|
exit 1
|
|
}
|
|
else
|
|
echo -e "${RED}No Docker DB container and no pg_dump found.${NC}"
|
|
exit 1
|
|
fi
|
|
else
|
|
docker exec "$LOCAL_DB_CONTAINER" pg_dump -U "$LOCAL_DB_USER" -d "$LOCAL_DB_NAME" \
|
|
--no-owner --no-privileges -Fc $EXCLUDE_FLAGS > "$DB_DUMP"
|
|
fi
|
|
|
|
DB_SIZE=$(du -h "$DB_DUMP" | cut -f1)
|
|
echo -e "${GREEN} Database dump: $DB_DUMP ($DB_SIZE)${NC}"
|
|
|
|
# Show table counts
|
|
echo -e "${BLUE} Table counts:${NC}"
|
|
if docker ps --format '{{.Names}}' | grep -q "^${LOCAL_DB_CONTAINER}$"; then
|
|
docker exec "$LOCAL_DB_CONTAINER" psql -U "$LOCAL_DB_USER" -d "$LOCAL_DB_NAME" -t -c "
|
|
SELECT ' ' || relname || ': ' || n_live_tup
|
|
FROM pg_stat_user_tables
|
|
WHERE n_live_tup > 0
|
|
ORDER BY n_live_tup DESC
|
|
LIMIT 10;
|
|
" 2>/dev/null || true
|
|
fi
|
|
|
|
# ============================================================
|
|
# Step 2: Dump media files
|
|
# ============================================================
|
|
echo ""
|
|
echo -e "${BLUE}[2/5] Dumping media files...${NC}"
|
|
|
|
if [ "$NO_MEDIA" = true ]; then
|
|
echo -e "${YELLOW} Skipped (--no-media).${NC}"
|
|
MEDIA_DUMP=""
|
|
elif [ -d "$PROJECT_DIR/media" ] && [ "$(ls -A "$PROJECT_DIR/media" 2>/dev/null)" ]; then
|
|
tar -czf "$MEDIA_DUMP" -C "$PROJECT_DIR/media" .
|
|
MEDIA_SIZE=$(du -h "$MEDIA_DUMP" | cut -f1)
|
|
echo -e "${GREEN} Media dump: $MEDIA_DUMP ($MEDIA_SIZE)${NC}"
|
|
else
|
|
echo -e "${YELLOW} No media files found, skipping.${NC}"
|
|
MEDIA_DUMP=""
|
|
fi
|
|
|
|
# ============================================================
|
|
# Step 3: Cleanup old local backups (keep last 3)
|
|
# ============================================================
|
|
echo ""
|
|
echo -e "${BLUE}[3/5] Cleaning old local backups (keeping last 3)...${NC}"
|
|
|
|
ls -t "$BACKUP_DIR"/px360_*.dump 2>/dev/null | tail -n +4 | while read -r old_file; do
|
|
rm -f "$old_file"
|
|
echo -e " ${YELLOW}Removed: $(basename "$old_file")${NC}"
|
|
done
|
|
ls -t "$BACKUP_DIR"/media_*.tar.gz 2>/dev/null | tail -n +4 | while read -r old_file; do
|
|
rm -f "$old_file"
|
|
echo -e " ${YELLOW}Removed: $(basename "$old_file")${NC}"
|
|
done
|
|
|
|
# ============================================================
|
|
# Dry-run check
|
|
# ============================================================
|
|
if [ "$DEPLOY" = false ]; then
|
|
echo ""
|
|
echo -e "${GREEN}========================================${NC}"
|
|
echo -e "${GREEN} Dry-run complete!${NC}"
|
|
echo -e "${GREEN} Dump saved locally.${NC}"
|
|
echo -e "${GREEN} Run with --deploy user@host to push to prod.${NC}"
|
|
echo -e "${GREEN}========================================${NC}"
|
|
echo ""
|
|
echo -e "Database: ${DB_DUMP} (${DB_SIZE})"
|
|
if [ -n "$MEDIA_DUMP" ]; then
|
|
echo -e "Media: ${MEDIA_DUMP} (${MEDIA_SIZE})"
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
# ============================================================
|
|
# Step 4: Copy to production DB server
|
|
# ============================================================
|
|
echo ""
|
|
echo -e "${BLUE}[4/5] Copying to production server ($PROD_SSH)...${NC}"
|
|
|
|
echo -e " Copying database dump..."
|
|
scp -q "$DB_DUMP" "${PROD_SSH}:/tmp/px360_restore.dump"
|
|
echo -e "${GREEN} Database dump uploaded.${NC}"
|
|
|
|
if [ -n "$MEDIA_DUMP" ]; then
|
|
echo -e " Copying media files..."
|
|
scp -q "$MEDIA_DUMP" "${PROD_SSH}:/tmp/px360_media.tar.gz"
|
|
echo -e "${GREEN} Media files uploaded.${NC}"
|
|
fi
|
|
|
|
# ============================================================
|
|
# Step 5: Restore on production DB server
|
|
# ============================================================
|
|
echo ""
|
|
echo -e "${BLUE}[5/5] Restoring on production database...${NC}"
|
|
|
|
if [ -n "$CONTAINER" ]; then
|
|
# ----- Container mode: restore via docker exec into the DB container -----
|
|
|
|
# Stop app containers so nothing holds connections during --clean restore
|
|
if [ -n "$PROJECT" ]; then
|
|
echo -e "${BLUE} Stopping app containers (project $PROJECT)...${NC}"
|
|
ssh "$PROD_SSH" "cd \$HOME/.target/${PROJECT} && docker compose -p ${PROJECT} stop web_app celery celery-beat 2>&1 || true"
|
|
fi
|
|
|
|
ssh "$PROD_SSH" bash -s << REMOTE_SCRIPT
|
|
set -e
|
|
DB_USER="${PROD_DB_USER}"
|
|
DB_NAME="${PROD_DB_NAME}"
|
|
DB_CONTAINER="${CONTAINER}"
|
|
|
|
echo " Copying dump into DB container \$DB_CONTAINER..."
|
|
docker cp /tmp/px360_restore.dump "\$DB_CONTAINER":/tmp/px360_restore.dump
|
|
|
|
echo " Restoring database (this may take a minute)..."
|
|
docker exec "\$DB_CONTAINER" pg_restore -U "\$DB_USER" -d "\$DB_NAME" \
|
|
--no-owner --no-privileges --clean --if-exists --no-tablespaces \
|
|
/tmp/px360_restore.dump 2>&1 | grep -v "does not exist" || true
|
|
|
|
echo " Verifying restore..."
|
|
docker exec "\$DB_CONTAINER" psql -U "\$DB_USER" -d "\$DB_NAME" -t -c "
|
|
SELECT ' Total tables: ' || count(*) FROM pg_stat_user_tables;
|
|
" 2>/dev/null || true
|
|
docker exec "\$DB_CONTAINER" psql -U "\$DB_USER" -d "\$DB_NAME" -t -c "
|
|
SELECT ' ' || relname || ': ' || n_live_tup
|
|
FROM pg_stat_user_tables
|
|
WHERE n_live_tup > 0
|
|
ORDER BY n_live_tup DESC
|
|
LIMIT 10;
|
|
" 2>/dev/null || true
|
|
|
|
echo " Cleaning DB container temp..."
|
|
docker exec "\$DB_CONTAINER" rm -f /tmp/px360_restore.dump
|
|
echo " DB restore complete."
|
|
REMOTE_SCRIPT
|
|
|
|
# Start app containers back up
|
|
if [ -n "$PROJECT" ]; then
|
|
echo -e "${BLUE} Starting app containers...${NC}"
|
|
ssh "$PROD_SSH" "cd \$HOME/.target/${PROJECT} && docker compose -p ${PROJECT} up -d 2>&1 || true"
|
|
fi
|
|
|
|
# Media restore (web_app must be running, so after `up -d`)
|
|
if [ -n "$MEDIA_DUMP" ] && [ -n "$MEDIA_CONTAINER" ]; then
|
|
echo ""
|
|
echo -e "${BLUE}Restoring media into ${MEDIA_CONTAINER}...${NC}"
|
|
ssh "$PROD_SSH" bash -s << REMOTE_MEDIA
|
|
set -e
|
|
MC="${MEDIA_CONTAINER}"
|
|
docker cp /tmp/px360_media.tar.gz "\$MC":/tmp/px360_media.tar.gz
|
|
# web_app runs as appuser (UID 1000); chown so the app can write uploads later
|
|
docker exec "\$MC" sh -c "mkdir -p /app/media && tar -xzf /tmp/px360_media.tar.gz -C /app/media && chown -R 1000:1000 /app/media"
|
|
docker exec "\$MC" rm -f /tmp/px360_media.tar.gz
|
|
echo " Media restored to \$MC:/app/media"
|
|
REMOTE_MEDIA
|
|
fi
|
|
|
|
# Clean up host temp copies
|
|
ssh "$PROD_SSH" "rm -f /tmp/px360_restore.dump /tmp/px360_media.tar.gz" 2>/dev/null || true
|
|
|
|
else
|
|
# ----- Host mode: host-installed pg client tools (original behavior) -----
|
|
ssh "$PROD_SSH" bash -s << REMOTE_SCRIPT
|
|
set -e
|
|
DB_USER="${PROD_DB_USER}"
|
|
DB_NAME="${PROD_DB_NAME}"
|
|
DB_HOST="${PROD_DB_HOST}"
|
|
|
|
echo " Creating database if not exists..."
|
|
createdb -U "\$DB_USER" "\$DB_NAME" 2>/dev/null || echo " Database already exists."
|
|
|
|
echo " Restoring database (this may take a minute)..."
|
|
pg_restore -U "\$DB_USER" -d "\$DB_NAME" \
|
|
--no-owner --no-privileges --clean --if-exists \
|
|
--no-tablespaces \
|
|
/tmp/px360_restore.dump 2>&1 | grep -v "does not exist" || true
|
|
|
|
echo " Verifying restore..."
|
|
psql -U "\$DB_USER" -d "\$DB_NAME" -t -c "
|
|
SELECT ' Total tables: ' || count(*) FROM pg_stat_user_tables;
|
|
" 2>/dev/null || true
|
|
|
|
psql -U "\$DB_USER" -d "\$DB_NAME" -t -c "
|
|
SELECT ' ' || relname || ': ' || n_live_tup
|
|
FROM pg_stat_user_tables
|
|
WHERE n_live_tup > 0
|
|
ORDER BY n_live_tup DESC
|
|
LIMIT 10;
|
|
" 2>/dev/null || true
|
|
|
|
echo " Cleaning up temp files..."
|
|
rm -f /tmp/px360_restore.dump /tmp/px360_media.tar.gz
|
|
|
|
echo " Restore complete."
|
|
REMOTE_SCRIPT
|
|
|
|
if [ -n "$MEDIA_DUMP" ]; then
|
|
echo ""
|
|
echo -e "${YELLOW}Note: Media files were uploaded to the DB server at /tmp/px360_media.tar.gz${NC}"
|
|
echo -e "${YELLOW}Restore manually on the app server:${NC}"
|
|
echo -e " scp ${PROD_SSH}:/tmp/px360_media.tar.gz app-server:/tmp/"
|
|
echo -e " ssh app-server 'mkdir -p /path/to/media && tar -xzf /tmp/px360_media.tar.gz -C /path/to/media'"
|
|
fi
|
|
fi
|
|
|
|
# ============================================================
|
|
# Summary
|
|
# ============================================================
|
|
echo ""
|
|
echo -e "${GREEN}========================================${NC}"
|
|
echo -e "${GREEN} Sync complete!${NC}"
|
|
echo -e "${GREEN} $(date '+%Y-%m-%d %H:%M:%S')${NC}"
|
|
echo -e "${GREEN}========================================${NC}"
|
|
echo ""
|
|
echo -e "Local backup: ${DB_DUMP} (${DB_SIZE})"
|
|
if [ -n "$MEDIA_DUMP" ]; then
|
|
echo -e "Media backup: ${MEDIA_DUMP} (${MEDIA_SIZE})"
|
|
fi
|
|
echo ""
|
|
echo -e "${YELLOW}Next steps:${NC}"
|
|
echo -e " 1. Run migrations on app server: docker compose -f docker-compose.prod.yml exec web python manage.py migrate"
|
|
echo -e " 2. Clear sessions: docker compose -f docker-compose.prod.yml exec web python manage.py shell -c \"from django.contrib.sessions.models import Session; Session.objects.all().delete()\""
|
|
echo -e " 3. Restart services: docker compose -f docker-compose.prod.yml restart"
|