diff --git a/.github/workflows/push-trigger.yml b/.github/workflows/push-trigger.yml index 1cbc66f..3a1cef5 100644 --- a/.github/workflows/push-trigger.yml +++ b/.github/workflows/push-trigger.yml @@ -1,4 +1,4 @@ -name: Build postgres-init and masterdata-loader +name: Build postgres-init, postgres-upgrade and masterdata-loader on: release: @@ -18,6 +18,7 @@ on: - 1.* - develop - release* + - MOSIP* jobs: build-dockers: @@ -39,3 +40,63 @@ jobs: ACTOR_DOCKER_HUB: ${{ secrets.ACTOR_DOCKER_HUB }} RELEASE_DOCKER_HUB: ${{ secrets.RELEASE_DOCKER_HUB }} SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_DEVOPS }} + + get-db-upgrade-scripts: + runs-on: ubuntu-latest + env: + SERVICE_LOCATION: 'postgres-upgrade' + BUILD_ARTIFACT: 'postgres-upgrade-scripts' + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Setup branch and env + run: | + # PR -> target branch, others -> ref_name (branch or tag name) + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "BRANCH_NAME=${{ github.base_ref }}" >> "$GITHUB_ENV" + else + echo "BRANCH_NAME=${{ github.ref_name }}" >> "$GITHUB_ENV" + fi + echo "GPG_TTY=$(tty)" >> $GITHUB_ENV + + - name: Clone DB upgrade scripts + run: | + cd ${{ env.SERVICE_LOCATION }} + bash ./get_db_data.sh "${{ env.BRANCH_NAME }}" + - name: Ready the DB upgrade scripts + if: ${{ (github.event_name == 'pull_request' && github.base_ref != 'master' && github.base_ref != 'main') || (github.event_name != 'pull_request' && github.ref_name != 'master' && github.ref_name != 'main') }} + run: | + cd ${{ env.SERVICE_LOCATION }} + find ./repos/*/db_upgrade_scripts/ -type f -exec zip ${{ env.BUILD_ARTIFACT }}.zip {} + + + - name: Upload the upgrade scripts + if: ${{ (github.event_name == 'pull_request' && github.base_ref != 'master' && github.base_ref != 'main') || (github.event_name != 'pull_request' && github.ref_name != 'master' && github.ref_name != 'main') }} + uses: actions/upload-artifact@v4 + with: + name: ${{ env.BUILD_ARTIFACT }} + path: ${{ env.SERVICE_LOCATION }}/${{ env.BUILD_ARTIFACT }}.zip + + build-upgrade-dockers: + needs: get-db-upgrade-scripts + if: ${{ (github.event_name == 'pull_request' && github.base_ref != 'master' && github.base_ref != 'main') || (github.event_name != 'pull_request' && github.ref_name != 'master' && github.ref_name != 'main') }} + strategy: + matrix: + include: + - SERVICE_LOCATION: 'postgres-upgrade' + SERVICE_NAME: 'postgres-upgrade' + BUILD_ARTIFACT: 'postgres-upgrade-scripts' + ONLY_DOCKER: true + fail-fast: false + name: ${{ matrix.SERVICE_NAME }} + uses: mosip/kattu/.github/workflows/docker-build.yml@master + with: + SERVICE_LOCATION: ${{ matrix.SERVICE_LOCATION }} + SERVICE_NAME: ${{ matrix.SERVICE_NAME }} + BUILD_ARTIFACT: ${{ matrix.BUILD_ARTIFACT }} + ONLY_DOCKER: ${{ matrix.ONLY_DOCKER }} + secrets: + DEV_NAMESPACE_DOCKER_HUB: ${{ secrets.DEV_NAMESPACE_DOCKER_HUB }} + ACTOR_DOCKER_HUB: ${{ secrets.ACTOR_DOCKER_HUB }} + RELEASE_DOCKER_HUB: ${{ secrets.RELEASE_DOCKER_HUB }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_DEVOPS }} diff --git a/deploy/chart/README.md b/deploy/chart/README.md deleted file mode 100644 index e19bd81..0000000 --- a/deploy/chart/README.md +++ /dev/null @@ -1 +0,0 @@ -Local helm chart to install istio components. diff --git a/deploy/chart/istio-addons/.helmignore b/deploy/chart/istio-addons/.helmignore deleted file mode 100644 index 0e8a0eb..0000000 --- a/deploy/chart/istio-addons/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/deploy/chart/istio-addons/Chart.yaml b/deploy/chart/istio-addons/Chart.yaml deleted file mode 100644 index 9dd1488..0000000 --- a/deploy/chart/istio-addons/Chart.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v2 -name: istio-addons -description: A Helm chart for MOSIP postgres istio - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" diff --git a/deploy/chart/istio-addons/templates/gateway.yaml b/deploy/chart/istio-addons/templates/gateway.yaml deleted file mode 100644 index c1e96ab..0000000 --- a/deploy/chart/istio-addons/templates/gateway.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: networking.istio.io/v1alpha3 -kind: Gateway -metadata: - name: postgres -spec: - selector: - istio: ingressgateway-internal - servers: - - port: - number: 5432 - name: postgres - protocol: TCP - hosts: - - {{ .Values.postgresHost }} diff --git a/deploy/chart/istio-addons/templates/vs.yaml b/deploy/chart/istio-addons/templates/vs.yaml deleted file mode 100644 index dec6238..0000000 --- a/deploy/chart/istio-addons/templates/vs.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: networking.istio.io/v1alpha3 -kind: VirtualService -metadata: - name: postgres -spec: - hosts: - - "*" - gateways: - - postgres - tcp: - - match: - - port: 5432 - route: - - destination: - host: postgres-postgresql - port: - number: 5432 - - diff --git a/deploy/chart/istio-addons/values.yaml b/deploy/chart/istio-addons/values.yaml deleted file mode 100644 index 7770edb..0000000 --- a/deploy/chart/istio-addons/values.yaml +++ /dev/null @@ -1 +0,0 @@ -postgresHost: postgres.sandbox.xyz.net diff --git a/deploy/masterdata-loader/README.md b/deploy/masterdata-loader/README.md new file mode 100644 index 0000000..a9145cc --- /dev/null +++ b/deploy/masterdata-loader/README.md @@ -0,0 +1,9 @@ +# Contains deployment scripts for masterdata loader + +## Masterdata-loader + +### Init mosip_master db +* To incorporate your own master data, modify the `install.sh` with your `GithubBranch`, `GithubRepo` and `XlsfolderPath: /home/mosip/`. +```bash +./install.sh +``` diff --git a/deploy/copy-secrets-masterdata.sh b/deploy/masterdata-loader/copy_secret.sh old mode 100755 new mode 100644 similarity index 86% rename from deploy/copy-secrets-masterdata.sh rename to deploy/masterdata-loader/copy_secret.sh index 658faa2..895e261 --- a/deploy/copy-secrets-masterdata.sh +++ b/deploy/masterdata-loader/copy_secret.sh @@ -3,7 +3,7 @@ # DST_NS: Destination namespace function copying_secrets() { - UTIL_URL=https://github.com/mosip/mosip-infra/blob/master/deployment/v3/utils/copy_cm_func.sh + UTIL_URL=https://raw.githubusercontent.com/mosip/mosip-infra/master/deployment/v3/utils/copy_cm_func.sh COPY_UTIL=./copy_cm_func.sh DST_NS=masterdata-loader diff --git a/deploy/masterdata-install.sh b/deploy/masterdata-loader/install.sh old mode 100755 new mode 100644 similarity index 93% rename from deploy/masterdata-install.sh rename to deploy/masterdata-loader/install.sh index 29a5272..ba0927e --- a/deploy/masterdata-install.sh +++ b/deploy/masterdata-loader/install.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Loads sample masterdata +# Loads sample masterdata ## Usage: ./install.sh [kubeconfig] if [ $# -ge 1 ] ; then @@ -29,8 +29,8 @@ if [ $yn = "Y" ] helm repo update echo Copy configmaps - sed -i 's/\r$//' copy-secrets-masterdata.sh - ./copy-secrets-masterdata.sh + sed -i 's/\r$//' copy_secret.sh + ./copy_secret.sh echo Loading masterdata helm -n $NS install masterdata-loader mosip/masterdata-loader \ diff --git a/deploy/postgres-install.sh b/deploy/postgres-install.sh index 67efc8d..e926f36 100755 --- a/deploy/postgres-install.sh +++ b/deploy/postgres-install.sh @@ -7,7 +7,10 @@ if [ $# -ge 1 ] ; then fi NS=postgres +ISTIO_ADDONS_CHART_VERSION=1.0.0 +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo add mosip https://mosip.github.io/mosip-helm helm repo update echo Create $NS namespace kubectl create namespace $NS @@ -15,11 +18,13 @@ kubectl label ns $NS istio-injection=enabled --overwrite function installing_postgres() { echo Installing Postgres - helm -n $NS install postgres bitnami/postgresql --version 13.1.5 -f postgres-values.yaml --wait + helm -n $NS install postgres bitnami/postgresql \ + --set image.repository="mosipid/postgresql" \ + --set image.tag="16.0.0-debian-11-r13" \ + --version 13.1.5 -f postgres-values.yaml --wait echo Installed Postgres echo Installing gateways and virtual services - POSTGRES_HOST=$(kubectl get cm global -o jsonpath={.data.mosip-postgres-host}) - helm -n $NS install istio-addons chart/istio-addons --set postgresHost=$POSTGRES_HOST --wait + helm -n $NS install istio-addons mosip/istio-addons --version $ISTIO_ADDONS_CHART_VERSION -f postgres-istio-addons-values.yaml --wait return 0 } diff --git a/deploy/postgres-istio-addons-values.yaml b/deploy/postgres-istio-addons-values.yaml new file mode 100644 index 0000000..521c15d --- /dev/null +++ b/deploy/postgres-istio-addons-values.yaml @@ -0,0 +1,15 @@ +istio: + name: postgres + host: postgres.sandbox.xyz.net +# Note: If you have multiple ingressController then pass it in the form of list. + ingressController: ingressgateway-internal +# Note : By default, if you do **not** specify any gateways, the chart will bind to a single Gateway whose name is derived from the metadata name. If you have mutiple gateways to specify then pass it in the form of list like below. +# gateways: +# - public-gateway +# - internal-gateway + port: + name: tcp-postgresql + number: 5432 + protocol: TCP + serviceHost: postgres-postgresql + servicePort: 5432 diff --git a/deploy/postgres/ansible/README.md b/deploy/postgres/ansible/README.md new file mode 100644 index 0000000..57fa4d5 --- /dev/null +++ b/deploy/postgres/ansible/README.md @@ -0,0 +1,157 @@ +# PostgreSQL Secure Setup - MOSIP Infrastructure + +Production-grade PostgreSQL deployment with secure password generation, Kubernetes integration, and private subnet optimization. + +## Quick Start (3 Simple Steps) + +```bash +# 1. Interactive inventory setup with smart defaults +./setup-vm-inventory.sh + +# 2. Deploy PostgreSQL with secure setup +./run-postgresql-playbook.sh + +# 3. Verify installation status +./check-postgresql-status.sh +``` + +## Core Features + +### Deployment Features +- **Streamlined deployment** for private networks +- **Custom port configuration** (default 5433) +- **XFS filesystem** with optimized mount options +- **Connection pooling ready** with high max_connections + +### Production Ready +- **Automated password generation** - no manual password handling +- **Kubernetes integration** - generates secrets and ConfigMaps +- **Data migration support** - moves existing data to new storage +- **Complete lifecycle management** - setup, monitoring, cleanup + +## Essential Scripts + +### Production Scripts +- **`run-postgresql-playbook.sh`** - Main secure deployment script +- **`postgresql-setup.yml`** - Secure Ansible playbook +- **`check-postgresql-status.sh`** - Status monitoring and verification +- **`cleanup-postgresql.sh`** - Safe cleanup with backups + +### Configuration Files +- **`hosts.ini`** - Ansible inventory configuration +- **`postgresql-cleanup.yml`** - Cleanup playbook + +## Configuration + +Edit `hosts.ini` to configure your PostgreSQL server: + +```ini +[postgresql_servers] +postgres-vm ansible_host=10.0.2.176 ansible_user=mosipuser ansible_ssh_private_key_file=~/.ssh/mosip-key + +# Configuration Comments: +# PostgreSQL Version: 15 +# PostgreSQL Port: 5433 +# Storage Device: /dev/nvme2n1 +# Mount Point: /srv/postgres +# Network CIDR: 10.0.0.0/8 +# Kubernetes Namespace: postgres +``` + +## Advanced Usage + +### Security Configuration +```bash +# Use custom Kubernetes namespace and secret name +./run-postgresql-playbook.sh --namespace production --secret-name postgres-creds + +# Auto-confirm deployment (for automation) +./run-postgresql-playbook.sh --auto-confirm +``` + +### Kubernetes Integration +After deployment, Kubernetes files are generated in `/tmp/postgresql-secrets/`: +- `postgres-postgresql.yml` - Secret with credentials +- `postgres-setup-config.yml` - ConfigMap with connection details +- `DEPLOYMENT_INSTRUCTIONS.md` - Deployment guide + +**Apply the generated secrets to Kubernetes:** +```bash +# Create the postgres namespace +kubectl create ns postgres + +# Apply the generated YAML files +kubectl apply -f /tmp/postgresql-secrets/postgres-postgresql.yml +kubectl apply -f /tmp/postgresql-secrets/postgres-setup-config.yml +``` + +### Monitoring and Maintenance +```bash +# Check PostgreSQL status +./check-postgresql-status.sh + +# Safe cleanup (creates backups) +./cleanup-postgresql.sh --safe +``` + +## Security Features + +### Applied Security Measures +- 16-character secure passwords (mixed case, numbers, special chars) +- MD5 password encryption +- Private subnet deployment (no encryption overhead) +- Connection and statement audit logging +- Kubernetes secrets with proper base64 encoding +- Proper file permissions (0600 for secrets) +- Separation of sensitive and non-sensitive data +- No plaintext passwords in logs or files + +## Requirements Checklist + +### Server Requirements + +- [ ] Ubuntu 20.04+ server +- [ ] 4GB+ RAM (8GB+ recommended) +- [ ] Dedicated storage device (e.g., `/dev/nvme2n1`) +- [ ] SSH key-based access configured + +### Network Requirements + +- [ ] Private subnet (10.0.0.0/8 recommended) +- [ ] SSH access on port 22 +- [ ] PostgreSQL port 5433 accessible +- [ ] No encryption requirements (private network) + +### Software Dependencies + +- [ ] Ansible installed +- [ ] Python3-bcrypt package +- [ ] Python3-psycopg2 package +- [ ] SSH keys configured for target server + +## Workflow Overview + +```text +Edit hosts.ini → run-postgresql-playbook.sh → PostgreSQL Ready + ↓ + Kubernetes files generated + ↓ + check-postgresql-status.sh (verify) +``` + +## What Changed from Previous Version + +### Security Enhancements +- **Encryption Optimized**: Streamlined for private subnet deployment +- **Secure Password Generation**: Automated 16-character passwords +- **MD5 Encryption**: Standard password encryption +- **Kubernetes Integration**: Auto-generated secrets and ConfigMaps + +### Operational Excellence +- **Automated Deployment**: One-command secure setup +- **Production Ready**: Enterprise-grade security and monitoring +- **Easy Integration**: Kubernetes-ready with generated manifests + +--- + +*Secure, fast, and production-ready PostgreSQL for MOSIP infrastructure* diff --git a/deploy/postgres/ansible/check-postgresql-status.sh b/deploy/postgres/ansible/check-postgresql-status.sh new file mode 100644 index 0000000..566ac13 --- /dev/null +++ b/deploy/postgres/ansible/check-postgresql-status.sh @@ -0,0 +1,128 @@ +#!/bin/bash + +# PostgreSQL Environment Status Check +# This script checks the current state of PostgreSQL installation before cleanup + +set -e + +INVENTORY_FILE="${1:-hosts.ini}" +TARGET_GROUP="${2:-postgresql_servers}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +echo -e "${BLUE}=== PostgreSQL Environment Status Check ===${NC}" +echo "Inventory: $INVENTORY_FILE" +echo "Target Group: $TARGET_GROUP" +echo "==============================================" + +# Check connectivity +echo -e "${GREEN}Testing connectivity...${NC}" +if ! ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m ping > /dev/null 2>&1; then + echo -e "${RED}✗ Cannot connect to target hosts${NC}" + exit 1 +fi +echo -e "${GREEN}✓ Connectivity OK${NC}" + +echo "" +echo -e "${YELLOW}=== PostgreSQL Installation Status ===${NC}" + +# Check PostgreSQL packages +echo "PostgreSQL Packages:" +ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a 'dpkg -l | grep postgresql' 2>/dev/null || echo "No PostgreSQL packages found" + +echo "" +echo "PostgreSQL Processes:" +ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a 'ps aux | grep postgres | grep -v grep' 2>/dev/null || echo "No PostgreSQL processes running" + +echo "" +echo "PostgreSQL Service Status:" +ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a 'systemctl is-active postgresql' 2>/dev/null || echo "PostgreSQL service not active" + +echo "" +echo -e "${YELLOW}=== Storage and Mount Points ===${NC}" + +# Check common mount points +for mount_point in "/srv/postgres" "/data" "/mnt/postgresql" "/opt/postgresql"; do + echo "Mount point $mount_point:" + ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a "mount | grep '$mount_point'" 2>/dev/null || echo "Not mounted" +done + +echo "" +echo "Current mount points:" +ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a 'mount | grep -E "(srv|data|postgresql)" || echo "No relevant mount points"' 2>/dev/null + +echo "" +echo -e "${YELLOW}=== Storage Devices ===${NC}" + +# Check common storage devices +for device in "/dev/nvme2n1" "/dev/sdb" "/dev/vdb" "/dev/xvdb"; do + echo "Device $device:" + ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a "lsblk '$device' 2>/dev/null || echo 'Device not found'" 2>/dev/null +done + +echo "" +echo -e "${YELLOW}=== Configuration Files ===${NC}" + +# Check configuration directories +for config_dir in "/etc/postgresql" "/var/lib/postgresql"; do + echo "Directory $config_dir:" + ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a "ls -la '$config_dir' 2>/dev/null || echo 'Directory not found'" 2>/dev/null +done + +echo "" +echo -e "${YELLOW}=== Data Directories ===${NC}" + +# Check data directories +for data_dir in "/srv/postgres/postgresql" "/data/postgresql" "/var/lib/postgresql" "/mnt/postgresql"; do + echo "Data directory $data_dir:" + ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a "ls -la '$data_dir' 2>/dev/null || echo 'Directory not found'" 2>/dev/null +done + +echo "" +echo -e "${YELLOW}=== Network Configuration ===${NC}" + +echo "PostgreSQL listening ports:" +ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a 'netstat -tlnp | grep postgres || echo "No PostgreSQL ports listening"' 2>/dev/null + +echo "" +echo -e "${YELLOW}=== Users and Groups ===${NC}" + +echo "PostgreSQL user:" +ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a 'id postgres 2>/dev/null || echo "User postgres not found"' 2>/dev/null + +echo "" +echo -e "${GREEN}=== Status Summary ===${NC}" + +# Create a summary +ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a ' +echo "=== PostgreSQL Environment Summary ===" +echo "Hostname: $(hostname)" +echo "Date: $(date)" +echo "" +echo "PostgreSQL packages: $(dpkg -l | grep postgresql | wc -l) found" +echo "PostgreSQL processes: $(ps aux | grep postgres | grep -v grep | wc -l) running" +echo "PostgreSQL config dirs: $(find /etc -name "*postgresql*" -type d 2>/dev/null | wc -l) found" +echo "PostgreSQL data dirs: $(find /var/lib -name "*postgresql*" -type d 2>/dev/null | wc -l) found" +echo "Relevant mount points: $(mount | grep -E "(srv|data|postgresql)" | wc -l) found" +echo "" +if systemctl is-active postgresql >/dev/null 2>&1; then + echo "PostgreSQL service: ACTIVE" +else + echo "PostgreSQL service: INACTIVE" +fi +' 2>/dev/null + +echo "" +echo -e "${BLUE}=== Cleanup Options ===${NC}" +echo "Based on the current state, you can use:" +echo "" +echo "• Quick cleanup (no backups): ./cleanup-postgresql.sh --quick" +echo "• Safe cleanup (with backups): ./cleanup-postgresql.sh --safe" +echo "• Complete cleanup (wipe all): ./cleanup-postgresql.sh --complete" +echo "" +echo "For custom cleanup options, see: ./cleanup-postgresql.sh --help" diff --git a/deploy/postgres/ansible/cleanup-postgresql.sh b/deploy/postgres/ansible/cleanup-postgresql.sh new file mode 100644 index 0000000..7e56a2a --- /dev/null +++ b/deploy/postgres/ansible/cleanup-postgresql.sh @@ -0,0 +1,246 @@ +#!/bin/bash + +# PostgreSQL Cleanup Script - Complete Environment Restoration +# This script provides various cleanup options for PostgreSQL installations + +set -e + +# Default values +INVENTORY_FILE="hosts.ini" +CLEANUP_PLAYBOOK="postgresql-cleanup.yml" +TARGET_GROUP="postgresql_servers" +POSTGRESQL_VERSION="15" +STORAGE_DEVICE="/dev/nvme2n1" +MOUNT_POINT="/srv/postgres" +CREATE_BACKUP=false +WIPE_DEVICE=false +AUTO_CONFIRM=false + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +print_usage() { + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " -i, --inventory FILE Inventory file (default: hosts.ini)" + echo " -c, --cleanup FILE Cleanup playbook (default: postgresql-cleanup.yml)" + echo " -v, --version VERSION PostgreSQL version (default: 15)" + echo " -d, --device DEVICE Storage device (default: /dev/nvme2n1)" + echo " -m, --mount PATH Mount point (default: /srv/postgres)" + echo " -g, --group GROUP Target group (default: postgresql_servers)" + echo " -b, --backup Create safety backup before cleanup" + echo " -w, --wipe-device Wipe storage device completely (DANGEROUS!)" + echo " -y, --yes Auto-confirm (skip prompts)" + echo " --quick Quick cleanup (no backups, no device wipe)" + echo " --safe Safe cleanup (with backups, no device wipe)" + echo " --complete Complete cleanup (with backups and device wipe)" + echo " -t, --test Test connectivity only" + echo " -h, --help Show this help message" + echo "" + echo "Cleanup Modes:" + echo " Quick: Fast cleanup, no backups, preserves device data" + echo " Safe: Creates backups, removes PostgreSQL, preserves device" + echo " Complete: Creates backups, removes PostgreSQL, wipes device clean" + echo "" + echo "Examples:" + echo " $0 --safe # Safe cleanup with backups" + echo " $0 --quick -y # Quick cleanup, auto-confirm" + echo " $0 --complete # Complete cleanup with device wipe" + echo " $0 -b -d /dev/vdb -m /opt/postgresql # Custom device and mount with backup" + echo " $0 -t # Test connectivity only" +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -i|--inventory) + INVENTORY_FILE="$2" + shift 2 + ;; + -c|--cleanup) + CLEANUP_PLAYBOOK="$2" + shift 2 + ;; + -v|--version) + POSTGRESQL_VERSION="$2" + shift 2 + ;; + -d|--device) + STORAGE_DEVICE="$2" + shift 2 + ;; + -m|--mount) + MOUNT_POINT="$2" + shift 2 + ;; + -g|--group) + TARGET_GROUP="$2" + shift 2 + ;; + -b|--backup) + CREATE_BACKUP=true + shift + ;; + -w|--wipe-device) + WIPE_DEVICE=true + shift + ;; + -y|--yes) + AUTO_CONFIRM=true + shift + ;; + --quick) + CREATE_BACKUP=false + WIPE_DEVICE=false + shift + ;; + --safe) + CREATE_BACKUP=true + WIPE_DEVICE=false + shift + ;; + --complete) + CREATE_BACKUP=true + WIPE_DEVICE=true + shift + ;; + -t|--test) + TEST_ONLY=true + shift + ;; + -h|--help) + print_usage + exit 0 + ;; + *) + echo -e "${RED}Error: Unknown option $1${NC}" + print_usage + exit 1 + ;; + esac +done + +echo -e "${RED}=== PostgreSQL Cleanup Script ===${NC}" +echo -e "${YELLOW}WARNING: This will PERMANENTLY DELETE PostgreSQL and its data!${NC}" +echo "" +echo "Configuration:" +echo " Inventory: $INVENTORY_FILE" +echo " Playbook: $CLEANUP_PLAYBOOK" +echo " Target Group: $TARGET_GROUP" +echo " PostgreSQL Version: $POSTGRESQL_VERSION" +echo " Storage Device: $STORAGE_DEVICE" +echo " Mount Point: $MOUNT_POINT" +echo " Create Backup: $CREATE_BACKUP" +echo " Wipe Device: $WIPE_DEVICE" +echo " Auto Confirm: $AUTO_CONFIRM" +echo "==============================================" + +# Check if required files exist +if [[ ! -f "$INVENTORY_FILE" ]]; then + echo -e "${RED}Error: Inventory file '$INVENTORY_FILE' not found!${NC}" + exit 1 +fi + +if [[ ! -f "$CLEANUP_PLAYBOOK" ]]; then + echo -e "${RED}Error: Cleanup playbook '$CLEANUP_PLAYBOOK' not found!${NC}" + exit 1 +fi + +# Check if Ansible is installed +if ! command -v ansible &> /dev/null; then + echo -e "${RED}Error: Ansible not found. Please install Ansible first.${NC}" + exit 1 +fi + +# Test connectivity +echo -e "${GREEN}Testing connectivity...${NC}" +if ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m ping; then + echo -e "${GREEN}✓ Connectivity test passed${NC}" +else + echo -e "${RED}✗ Connectivity test failed${NC}" + echo "Please check your inventory file and SSH configuration" + exit 1 +fi + +# If test only, exit here +if [[ "${TEST_ONLY:-false}" == "true" ]]; then + echo -e "${GREEN}Test completed successfully!${NC}" + exit 0 +fi + +# Show what will be cleaned up +echo -e "${YELLOW}=== Cleanup Preview ===${NC}" +echo "The following will be PERMANENTLY DELETED:" +echo "• PostgreSQL $POSTGRESQL_VERSION service and packages" +echo "• All PostgreSQL configuration files" +echo "• All PostgreSQL data directories" +echo "• Mount point $MOUNT_POINT" +if [[ "$WIPE_DEVICE" == "true" ]]; then + echo -e "${RED}• ALL DATA on storage device $STORAGE_DEVICE (COMPLETE WIPE!)${NC}" +fi +echo "" + +if [[ "$CREATE_BACKUP" == "true" ]]; then + echo -e "${GREEN}Safety measures enabled:${NC}" + echo "• Configuration files will be backed up" + echo "• Data will be backed up before deletion" +fi +echo "" + +# Final confirmation +if [[ "$AUTO_CONFIRM" != "true" ]]; then + if [[ "$WIPE_DEVICE" == "true" ]]; then + echo -e "${RED}⚠️ DANGER: Device wipe is enabled! This will destroy ALL data on $STORAGE_DEVICE${NC}" + echo -e "${RED}⚠️ This action is IRREVERSIBLE!${NC}" + read -p "Type 'DELETE EVERYTHING' to confirm device wipe: " device_confirm + if [[ "$device_confirm" != "DELETE EVERYTHING" ]]; then + echo "Device wipe cancelled." + exit 1 + fi + fi + + read -p "Are you sure you want to proceed with PostgreSQL cleanup? (y/N): " confirm + if [[ $confirm != [yY] && $confirm != [yY][eE][sS] ]]; then + echo "Cleanup cancelled." + exit 0 + fi +fi + +# Show the actual ansible-playbook command +echo -e "${YELLOW}Executing cleanup command:${NC}" +echo "ansible-playbook -i $INVENTORY_FILE $CLEANUP_PLAYBOOK \\" +echo " --limit $TARGET_GROUP \\" +echo " --extra-vars \"postgresql_version=$POSTGRESQL_VERSION\" \\" +echo " --extra-vars \"storage_device=$STORAGE_DEVICE\" \\" +echo " --extra-vars \"mount_point=$MOUNT_POINT\" \\" +echo " --extra-vars \"create_backup=$CREATE_BACKUP\" \\" +echo " --extra-vars \"wipe_device=$WIPE_DEVICE\" \\" +echo " --extra-vars \"auto_confirm=$AUTO_CONFIRM\" \\" +echo " -v" +echo "" + +# Execute the cleanup +echo -e "${RED}Starting PostgreSQL cleanup...${NC}" +ansible-playbook -i "$INVENTORY_FILE" "$CLEANUP_PLAYBOOK" \ + --limit "$TARGET_GROUP" \ + --extra-vars "postgresql_version=$POSTGRESQL_VERSION" \ + --extra-vars "storage_device=$STORAGE_DEVICE" \ + --extra-vars "mount_point=$MOUNT_POINT" \ + --extra-vars "create_backup=$CREATE_BACKUP" \ + --extra-vars "wipe_device=$WIPE_DEVICE" \ + --extra-vars "auto_confirm=$AUTO_CONFIRM" \ + -v + +echo -e "${GREEN}PostgreSQL cleanup completed successfully!${NC}" +echo "" +echo -e "${BLUE}=== Post-Cleanup Verification ===${NC}" +echo "You can verify the cleanup with these commands:" +echo "ansible $TARGET_GROUP -i $INVENTORY_FILE -m shell -a 'dpkg -l | grep postgresql'" +echo "ansible $TARGET_GROUP -i $INVENTORY_FILE -m shell -a 'ps aux | grep postgres'" +echo "ansible $TARGET_GROUP -i $INVENTORY_FILE -m shell -a 'mount | grep $MOUNT_POINT'" +echo "ansible $TARGET_GROUP -i $INVENTORY_FILE -m shell -a 'lsblk $STORAGE_DEVICE'" diff --git a/deploy/postgres/ansible/hosts.ini b/deploy/postgres/ansible/hosts.ini new file mode 100644 index 0000000..a4d206e --- /dev/null +++ b/deploy/postgres/ansible/hosts.ini @@ -0,0 +1,34 @@ +# Ansible Inventory for PostgreSQL Setup on Virtual Machine +# Generated by setup-vm-inventory.sh on Tue 26 Aug 2025 12:35:42 AM IST +# +# Configuration Summary: +# - PostgreSQL Version: 15 +# - Storage Device: /dev/nvme2n1 +# - Mount Point: /srv/postgres +# - Network CIDR: 10.0.0.0/8 +# - PostgreSQL Port: 5433 + +[postgresql_servers] +vm_server ansible_host= ansible_user=ubuntu ansible_ssh_private_key_file=/path/sandbox.pem ansible_port=22 + +[postgresql_servers:vars] +# SSH and Ansible Configuration +ansible_become=yes +ansible_become_method=sudo +ansible_ssh_common_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' +ansible_python_interpreter=/usr/bin/python3 +ansible_ssh_timeout=30 +ansible_connect_timeout=30 + +# PostgreSQL Configuration Variables +postgresql_version=15 +storage_device=/dev/nvme2n1 +mount_point=/srv/postgres +network_cidr=10.0.0.0/8 + +# Additional PostgreSQL Variables +postgresql_port=5433 + +# For local testing (if needed) +[local] +localhost ansible_connection=local diff --git a/deploy/postgres/ansible/postgresql-cleanup.yml b/deploy/postgres/ansible/postgresql-cleanup.yml new file mode 100644 index 0000000..dd16e34 --- /dev/null +++ b/deploy/postgres/ansible/postgresql-cleanup.yml @@ -0,0 +1,232 @@ +--- +# PostgreSQL Cleanup - Complete Removal and Restoration +# This playbook completely removes PostgreSQL and restores the system to original state + +- name: PostgreSQL Complete Cleanup + hosts: postgresql_servers + become: yes + vars: + # Configuration - should match the setup playbook + postgres_version: "{{ postgresql_version | default('15') }}" + postgres_port: "{{ postgresql_port | default('5433') }}" + postgres_device: "{{ storage_device | default('/dev/nvme2n1') }}" + postgres_mount: "{{ mount_point | default('/srv/postgres') }}" + postgres_data_dir: "{{ postgres_mount }}/postgresql/{{ postgres_version }}/main" + postgres_old_dir: "/var/lib/postgresql/{{ postgres_version }}/main" + postgres_config_dir: "/etc/postgresql/{{ postgres_version }}/main" + + tasks: + - name: Display cleanup summary + debug: + msg: | + PostgreSQL {{ postgres_version }} Cleanup Starting + Port: {{ postgres_port }} + Data Directory: {{ postgres_data_dir }} + Mount Point: {{ postgres_mount }} + Device: {{ postgres_device }} + WARNING: This will PERMANENTLY DELETE all PostgreSQL data! + + - name: Pause for confirmation + pause: + prompt: "Are you sure you want to proceed with cleanup? This will DELETE ALL PostgreSQL data! Press ENTER to continue or Ctrl+C to abort" + when: not (auto_confirm | default(false) | bool) + + # Stop PostgreSQL Service + - name: Stop PostgreSQL service + systemd: + name: postgresql + state: stopped + ignore_errors: yes + + - name: Disable PostgreSQL service + systemd: + name: postgresql + enabled: no + ignore_errors: yes + + # Backup current data (optional safety measure) + - name: Create backup directory (safety measure) + file: + path: "/tmp/postgresql-cleanup-backup-{{ ansible_date_time.epoch }}" + state: directory + register: backup_dir + when: create_backup | default(false) | bool + + - name: Backup PostgreSQL configuration files + copy: + src: "{{ postgres_config_dir }}/" + dest: "{{ backup_dir.path }}/config/" + remote_src: yes + when: create_backup | default(false) | bool and backup_dir is defined + ignore_errors: yes + + - name: Backup custom data directory + shell: | + if [ -d "{{ postgres_data_dir }}" ]; then + tar -czf "{{ backup_dir.path }}/data-backup.tar.gz" -C "{{ postgres_data_dir }}" . + fi + when: create_backup | default(false) | bool and backup_dir is defined + ignore_errors: yes + + # Remove PostgreSQL completely + - name: Remove PostgreSQL packages + apt: + name: + - postgresql-{{ postgres_version }} + - postgresql-client-{{ postgres_version }} + - postgresql-contrib-{{ postgres_version }} + state: absent + purge: yes + autoremove: yes + + - name: Remove PostgreSQL common packages + apt: + name: + - postgresql-common + - postgresql-client-common + state: absent + purge: yes + autoremove: yes + ignore_errors: yes + + # Remove PostgreSQL APT repository + - name: Remove PostgreSQL APT repository key + apt_key: + url: https://www.postgresql.org/media/keys/ACCC4CF8.asc + state: absent + ignore_errors: yes + + - name: Remove PostgreSQL APT repository + apt_repository: + repo: "deb http://apt.postgresql.org/pub/repos/apt {{ ansible_distribution_release }}-pgdg main" + state: absent + ignore_errors: yes + + # Clean up directories and files + - name: Remove PostgreSQL configuration directory + file: + path: /etc/postgresql + state: absent + + - name: Remove PostgreSQL log directory + file: + path: /var/log/postgresql + state: absent + + - name: Remove PostgreSQL lib directory + file: + path: /var/lib/postgresql + state: absent + + - name: Remove custom data directory + file: + path: "{{ postgres_data_dir }}" + state: absent + when: postgres_data_dir != "/var/lib/postgresql/{{ postgres_version }}/main" + + # Unmount and clean storage + - name: Check if mount point is mounted + shell: "mount | grep '{{ postgres_mount }}'" + register: mount_check + failed_when: false + changed_when: false + + - name: Unmount the storage device + mount: + path: "{{ postgres_mount }}" + state: unmounted + when: mount_check.rc == 0 + + - name: Remove mount entry from fstab + mount: + path: "{{ postgres_mount }}" + state: absent + + - name: Remove mount directory + file: + path: "{{ postgres_mount }}" + state: absent + + # Optional: Wipe the storage device (DANGEROUS!) + - name: Wipe storage device (if requested) + shell: | + if [ -b "{{ postgres_device }}" ]; then + wipefs -a "{{ postgres_device }}" + dd if=/dev/zero of="{{ postgres_device }}" bs=1M count=100 + fi + when: wipe_device | default(false) | bool + ignore_errors: yes + + # Remove postgres user and group + - name: Remove postgres user + user: + name: postgres + state: absent + remove: yes + ignore_errors: yes + + - name: Remove postgres group + group: + name: postgres + state: absent + ignore_errors: yes + + # Clean up any remaining packages + - name: Remove orphaned packages + shell: apt autoremove -y + ignore_errors: yes + + - name: Clean package cache + shell: apt autoclean + ignore_errors: yes + + # Restore original pg_hba.conf backup if exists + - name: Check for original pg_hba.conf backup + stat: + path: "{{ postgres_config_dir }}/pg_hba.conf.*~" + register: original_backup + ignore_errors: yes + + # Final verification + - name: Verify PostgreSQL removal + shell: | + echo "=== PostgreSQL Cleanup Verification ===" + echo "PostgreSQL packages:" + dpkg -l | grep postgresql || echo "No PostgreSQL packages found" + echo "" + echo "PostgreSQL processes:" + ps aux | grep postgres | grep -v grep || echo "No PostgreSQL processes running" + echo "" + echo "Mount points:" + mount | grep "{{ postgres_mount }}" || echo "No mount points found" + echo "" + echo "Storage device:" + lsblk "{{ postgres_device }}" 2>/dev/null || echo "Device available" + echo "" + echo "=== Cleanup Complete ===" + register: cleanup_verification + changed_when: false + + - name: Display cleanup results + debug: + var: cleanup_verification.stdout_lines + + - name: Display backup information + debug: + msg: | + Cleanup completed successfully! + {% if create_backup | default(false) | bool and backup_dir is defined %} + Safety backup created at: {{ backup_dir.path }} + {% endif %} + + System Status: + - PostgreSQL service: Stopped and disabled + - PostgreSQL packages: Removed + - Configuration files: Deleted + - Data directories: Deleted + - Mount points: Unmounted and removed + {% if wipe_device | default(false) | bool %} + - Storage device: Wiped clean + {% endif %} + + The system has been restored to its original state. diff --git a/deploy/postgres/ansible/postgresql-setup.yml b/deploy/postgres/ansible/postgresql-setup.yml new file mode 100644 index 0000000..d418520 --- /dev/null +++ b/deploy/postgres/ansible/postgresql-setup.yml @@ -0,0 +1,591 @@ +--- +# PostgreSQL Setup with Secure Password Generation - Production Grade +# Enhanced playbook with secure password generation and Kubernetes secret creation + +- name: PostgreSQL Secure Setup + hosts: postgresql_servers + become: yes + vars: + # Default configuration - easily overridable + postgres_version: "{{ postgresql_version | default('15') }}" + postgres_port: "{{ postgresql_port | default('5433') }}" + postgres_device: "{{ storage_device | default('/dev/nvme2n1') }}" + postgres_mount: "{{ mount_point | default('/srv/postgres') }}" + postgres_data_dir: "{{ postgres_mount }}/postgresql/{{ postgres_version }}/main" + postgres_old_dir: "{% if ansible_os_family == 'Debian' %}/var/lib/postgresql/{{ postgres_version }}/main{% else %}/var/lib/pgsql/{{ postgres_version }}/data{% endif %}" + postgres_config_dir: "{% if ansible_os_family == 'Debian' %}/etc/postgresql/{{ postgres_version }}/main{% else %}/var/lib/pgsql/{{ postgres_version }}/data{% endif %}" + postgres_bin_dir: "{% if ansible_os_family == 'Debian' %}/usr/lib/postgresql/{{ postgres_version }}/bin{% else %}/usr/pgsql-{{ postgres_version }}/bin{% endif %}" + postgres_service_name: "{% if ansible_os_family == 'Debian' %}postgresql{% else %}postgresql-{{ postgres_version }}{% endif %}" + postgres_network_cidr: "{{ network_cidr | default('10.0.0.0/8') }}" + postgres_namespace: "{{ kubernetes_namespace | default('postgres') }}" + postgres_secret_name: "{{ secret_name | default('postgres-postgresql') }}" + # Use localhost for local operations, but actual IP for ConfigMap + postgres_local_host: "{{ local_connection_host | default('localhost') }}" + postgres_external_host: "{{ ansible_host }}" + + tasks: + # OS Compatibility and Prerequisites + - name: Gather OS facts + setup: + gather_subset: + - 'distribution' + - 'distribution_version' + + - name: Ensure Python3 is available (CentOS/RHEL compatibility) + package: + name: python3 + state: present + when: ansible_os_family == "RedHat" + become: yes + + - name: Verify Python3 with secrets module availability + command: python3 -c "import secrets; print('Python secrets module available')" + register: python_secrets_check + failed_when: python_secrets_check.rc != 0 + changed_when: false + + # Prerequisites Check + - name: Check if device exists + stat: + path: "{{ postgres_device }}" + follow: yes + register: device_check + failed_when: not device_check.stat.exists or not device_check.stat.isblk + + # Secure Password Generation - Idempotent approach + - name: Check if PostgreSQL is already configured with a password + shell: | + sudo -u postgres psql -p {{ postgres_port }} -c "SELECT 1;" 2>/dev/null + register: postgres_status_check + failed_when: false + changed_when: false + + - name: Try to read existing encrypted password from stored file + shell: | + if [ -f /var/lib/postgresql/.postgres_password_enc ] && [ -f /var/lib/postgresql/.postgres_encryption_key ]; then + ENCRYPTION_KEY=$(cat /var/lib/postgresql/.postgres_encryption_key) + openssl enc -aes-256-cbc -d -salt -pbkdf2 -pass pass:"$ENCRYPTION_KEY" -in /var/lib/postgresql/.postgres_password_enc 2>/dev/null + else + exit 1 + fi + register: existing_encrypted_password + failed_when: false + changed_when: false + no_log: true + + - name: Set existing password if file exists and is valid + set_fact: + postgres_password: "{{ existing_encrypted_password.stdout }}" + when: + - existing_encrypted_password.rc == 0 + - existing_encrypted_password.stdout is defined + - (existing_encrypted_password.stdout | length) == 16 + no_log: true + + - name: Generate secure PostgreSQL password only if needed (OS-independent method) + shell: | + # Primary Method: Python secrets (works on all OS with Python 3.6+) + python3 -c " + import secrets + import string + # Use cryptographically secure random generator + uppercase = ''.join(secrets.choice(string.ascii_uppercase) for _ in range(4)) + lowercase = ''.join(secrets.choice(string.ascii_lowercase) for _ in range(4)) + digits = ''.join(secrets.choice(string.digits) for _ in range(4)) + special = ''.join(secrets.choice('!@#\$%^&*') for _ in range(4)) + password = list(uppercase + lowercase + digits + special) + secrets.SystemRandom().shuffle(password) + print(''.join(password)) + " + register: generated_password_primary + when: postgres_password is not defined + no_log: true + + - name: Generate secure PostgreSQL password using OpenSSL fallback (Linux/macOS only) + shell: | + # Fallback Method: OpenSSL (Linux/macOS optimized) + openssl rand -base64 24 | tr -d "=+/" | cut -c1-16 + register: generated_password_fallback + when: + - postgres_password is not defined + - generated_password_primary.failed is defined + - generated_password_primary.failed == true + no_log: true + failed_when: false + + - name: Set password from OS-independent generation + set_fact: + postgres_password: "{{ generated_password_primary.stdout if generated_password_primary.stdout is defined else generated_password_fallback.stdout }}" + when: postgres_password is not defined + no_log: true + + - name: Validate password complexity + assert: + that: + - postgres_password | length == 16 + - postgres_password | regex_search('[A-Z]') + - postgres_password | regex_search('[a-z]') + - postgres_password | regex_search('[0-9]') + - postgres_password | regex_search('[!@#$%^&*]') + fail_msg: "Generated password does not meet complexity requirements" + success_msg: "Password meets security requirements" + no_log: true + + # Store password in a fact to ensure consistency throughout playbook + - name: Store password in host facts for consistency + set_fact: + postgres_password: "{{ postgres_password }}" + no_log: true + + # Storage Setup + - name: Check if device is already mounted + command: mountpoint -q {{ postgres_mount }} + register: mount_check + failed_when: false + changed_when: false + + - name: Format device if not mounted + filesystem: + fstype: xfs + dev: "{{ postgres_device }}" + force: no + when: mount_check.rc != 0 + + - name: Create mount point and mount device + mount: + path: "{{ postgres_mount }}" + src: "{{ postgres_device }}" + fstype: xfs + opts: defaults,noatime + state: mounted + + # PostgreSQL Installation - Multi-OS Support + + # Ubuntu/Debian Installation + - name: Install required packages for repository (Ubuntu/Debian) + apt: + name: + - wget + - ca-certificates + - gnupg + - lsb-release + - python3-bcrypt + - python3-psycopg2 + state: present + update_cache: yes + when: ansible_os_family == "Debian" + + # CentOS/RHEL Installation + - name: Install PostgreSQL repository (CentOS/RHEL) + yum: + name: "https://download.postgresql.org/pub/repos/yum/reporpms/EL-{{ ansible_distribution_major_version }}-x86_64/pgdg-redhat-repo-latest.noarch.rpm" + state: present + disable_gpg_check: yes + when: ansible_os_family == "RedHat" + + - name: Install required packages (CentOS/RHEL) + yum: + name: + - postgresql{{ postgres_version }}-server + - postgresql{{ postgres_version }} + - python3-bcrypt + - python3-psycopg2 + - wget + - ca-certificates + state: present + when: ansible_os_family == "RedHat" + + # Ubuntu/Debian Repository Setup + - name: Get Ubuntu codename + command: lsb_release -cs + register: ubuntu_codename + changed_when: false + when: ansible_os_family == "Debian" + + - name: Add PostgreSQL signing key + apt_key: + url: https://www.postgresql.org/media/keys/ACCC4CF8.asc + state: present + when: ansible_os_family == "Debian" + + - name: Add PostgreSQL APT repository + apt_repository: + repo: "deb http://apt.postgresql.org/pub/repos/apt {{ ubuntu_codename.stdout }}-pgdg main" + state: present + when: ansible_os_family == "Debian" + + - name: Update package cache after adding PostgreSQL repo + apt: + update_cache: yes + when: ansible_os_family == "Debian" + + # Install PostgreSQL (Ubuntu/Debian) + - name: Install PostgreSQL packages (Ubuntu/Debian) + apt: + name: + - "postgresql-{{ postgres_version }}" + - "postgresql-client-{{ postgres_version }}" + state: present + when: ansible_os_family == "Debian" + + # Initialize PostgreSQL on CentOS/RHEL (required step) + - name: Initialize PostgreSQL database (CentOS/RHEL only) + command: "/usr/pgsql-{{ postgres_version }}/bin/postgresql-{{ postgres_version }}-setup initdb" + when: + - ansible_os_family == "RedHat" + become: yes + register: postgres_init_result + failed_when: false + changed_when: postgres_init_result.rc == 0 + + # Generate encryption key AFTER PostgreSQL installation (so postgres user and directories exist) + - name: Generate or retrieve encryption key for password storage + shell: | + if [ ! -f /var/lib/postgresql/.postgres_encryption_key ]; then + # Generate a secure random key for encryption + openssl rand -base64 32 > /var/lib/postgresql/.postgres_encryption_key + chmod 600 /var/lib/postgresql/.postgres_encryption_key + chown postgres:postgres /var/lib/postgresql/.postgres_encryption_key + fi + cat /var/lib/postgresql/.postgres_encryption_key + register: encryption_key + no_log: true + + - name: Store password securely for future runs (after PostgreSQL installation) + shell: | + # Create encrypted password file using secure encryption key + echo "{{ postgres_password }}" | openssl enc -aes-256-cbc -salt -pbkdf2 -pass pass:"{{ encryption_key.stdout }}" -out /var/lib/postgresql/.postgres_password_enc + chmod 600 /var/lib/postgresql/.postgres_password_enc + chown postgres:postgres /var/lib/postgresql/.postgres_password_enc + # Also store a hash for verification + echo "{{ postgres_password }}" | sha256sum | cut -d' ' -f1 > /var/lib/postgresql/.postgres_password_hash + chmod 600 /var/lib/postgresql/.postgres_password_hash + chown postgres:postgres /var/lib/postgresql/.postgres_password_hash + when: postgres_password is defined + no_log: true + + - name: Stop PostgreSQL for data migration + systemd: + name: postgresql + state: stopped + + # Data Migration + - name: Check if old data directory exists + stat: + path: "{{ postgres_old_dir }}/PG_VERSION" + register: old_data_exists + + - name: Check if new data directory is empty + find: + paths: "{{ postgres_data_dir }}" + file_type: any + register: new_data_check + failed_when: false + + - name: Create new data directory + file: + path: "{{ postgres_data_dir }}" + state: directory + owner: postgres + group: postgres + mode: '0700' + recurse: yes + + - name: Copy data to new location + shell: | + sudo -u postgres cp -a {{ postgres_old_dir }}/* {{ postgres_data_dir }}/ + sudo -u postgres cp -a {{ postgres_old_dir }}/.[^.]* {{ postgres_data_dir }}/ 2>/dev/null || true + when: + - old_data_exists.stat.exists + - new_data_check.matched == 0 + + - name: Fix permissions on data directory + file: + path: "{{ postgres_data_dir }}" + owner: postgres + group: postgres + mode: '0700' + recurse: yes + + # Configuration + - name: Configure PostgreSQL settings + lineinfile: + path: "{{ postgres_config_dir }}/postgresql.conf" + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + backup: yes + loop: + - { regexp: '^#?data_directory\s*=', line: "data_directory = '{{ postgres_data_dir }}'" } + - { regexp: '^#?port\s*=', line: "port = {{ postgres_port }}" } + - { regexp: '^#?listen_addresses\s*=', line: "listen_addresses = '*'" } + - { regexp: '^#?max_connections\s*=', line: "max_connections = 1000" } + notify: restart postgresql + + - name: Configure pg_hba.conf for access + copy: + dest: "{{ postgres_config_dir }}/pg_hba.conf" + content: | + # Database administrative login by Unix domain socket + local all postgres peer + + # "local" is for Unix domain socket connections only + local all all peer + + # IPv4 local connections: + host all all 0.0.0.0/0 md5 + + # IPv6 local connections: + host all all ::1/128 md5 + + # Allow replication connections from localhost, by a user with the + # replication privilege. + local replication all peer + host replication all 127.0.0.1/32 md5 + host replication all ::1/128 md5 + owner: postgres + group: postgres + mode: '0640' + backup: yes + notify: restart postgresql + + # Start Service + - name: Start and enable PostgreSQL + systemd: + name: postgresql + state: started + enabled: yes + + # Wait for PostgreSQL to be ready + - name: Wait for PostgreSQL to be ready + wait_for: + port: "{{ postgres_port }}" + timeout: 30 + + # Secure Password Setup - Only set if not already configured + - name: Test if password is already set correctly + shell: | + export PGPASSWORD='{{ postgres_password }}' + psql -h localhost -p {{ postgres_port }} -U postgres -c "SELECT 'Password already configured' as status;" + register: existing_password_test + no_log: true + failed_when: false + changed_when: false + + - name: Set PostgreSQL postgres user password only if needed + shell: | + export PGPASSWORD_NEW='{{ postgres_password }}' + sudo -u postgres psql -p {{ postgres_port }} </dev/null || true + find /var/tmp -name "*postgres*" -type f -mmin -5 ! -name "*.yml" -delete 2>/dev/null || true + # Clear any environment variables that might contain passwords + unset PGPASSWORD PGPASSWORD_NEW 2>/dev/null || true + changed_when: false + failed_when: false + + # Clear sensitive variables from memory + - name: Clear sensitive variables from Ansible facts + set_fact: + postgres_password: "" + postgres_password_b64: "" + generated_password: "" + generated_password_primary: "" + generated_password_fallback: "" + no_log: true + + # Verify Kubernetes files were created + - name: Verify Kubernetes files were created successfully on localhost + stat: + path: "{{ item }}" + register: k8s_files_check + loop: + - /tmp/postgresql-secrets/{{ postgres_secret_name }}.yml + - /tmp/postgresql-secrets/postgres-setup-config.yml + delegate_to: localhost + run_once: true + become: false + when: postgres_password_b64 is defined + + - name: Display Kubernetes files status + debug: + msg: | + Kubernetes Files Status: + {% for file in k8s_files_check.results %} + - {{ file.item }}: {{ 'EXISTS' if file.stat.exists else 'MISSING' }} + {% endfor %} + when: k8s_files_check is defined + + # Final verification and summary + - name: Display setup summary + debug: + msg: | + ✅ PostgreSQL {{ postgres_version }} Secure Setup Complete + + Configuration: + - Host: {{ ansible_host }} + - Port: {{ postgres_port }} + - Data Directory: {{ postgres_data_dir }} + - Mount Point: {{ postgres_mount }} + - Password: {{ 'REUSED EXISTING' if existing_password_test.rc == 0 else 'NEWLY GENERATED' }} + - Password Storage: /var/lib/postgresql/.postgres_password_enc (AES-256-CBC encrypted) + - Connection Test: {{ 'PASSED' if pg_test is succeeded else 'FAILED' }} + + Kubernetes Files Generated: + - Secret: /tmp/postgresql-secrets/{{ postgres_secret_name }}.yml + - ConfigMap: /tmp/postgresql-secrets/postgres-setup-config.yml + + Security Features: + - 16-character secure password (persistent across runs) + - AES-256-CBC encrypted password storage with PBKDF2 + - Cryptographically secure encryption key (randomly generated) + - MD5 authentication for database connections + - No plaintext passwords in logs or process lists + - Environment variable isolation + - Secure memory cleanup + - Private subnet deployment + - Idempotent execution + - Kubernetes secrets with proper encoding + + Next Steps: + 1. Review files in /tmp/postgresql-secrets/ + 2. Deploy to Kubernetes cluster + 3. Password will be reused on subsequent playbook runs + + Note: Password is securely encrypted with a randomly generated key stored at: + /var/lib/postgresql/.postgres_encryption_key + + handlers: + - name: restart postgresql + systemd: + name: postgresql + state: restarted diff --git a/deploy/postgres/ansible/run-postgresql-playbook.sh b/deploy/postgres/ansible/run-postgresql-playbook.sh new file mode 100644 index 0000000..22046ed --- /dev/null +++ b/deploy/postgres/ansible/run-postgresql-playbook.sh @@ -0,0 +1,260 @@ +#!/bin/bash + +# PostgreSQL Secure Setup Script +# Production-grade deployment with secure password generation and Kubernetes integration + +set -euo pipefail + +# Configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INVENTORY_FILE="${SCRIPT_DIR}/hosts.ini" +PLAYBOOK_FILE="${SCRIPT_DIR}/postgresql-setup.yml" +SECRETS_DIR="/tmp/postgresql-secrets" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging function +log() { + echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1" +} + +error() { + echo -e "${RED}[ERROR]${NC} $1" >&2 +} + +success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +# Print banner +print_banner() { + cat << 'EOF' +╔══════════════════════════════════════════════════════════════╗ +║ PostgreSQL Secure Setup ║ +║ Production-Grade with Kubernetes Integration ║ +╚══════════════════════════════════════════════════════════════╝ +EOF +} + +# Check prerequisites +check_prerequisites() { + log "Checking prerequisites..." + + # Check if running as non-root user + if [[ $EUID -eq 0 ]]; then + error "This script should not be run as root for security reasons" + exit 1 + fi + + # Check if inventory file exists + if [[ ! -f "$INVENTORY_FILE" ]]; then + error "Inventory file not found: $INVENTORY_FILE" + exit 1 + fi + + # Check if playbook exists + if [[ ! -f "$PLAYBOOK_FILE" ]]; then + error "Playbook file not found: $PLAYBOOK_FILE" + exit 1 + fi + + # Check if ansible is installed + if ! command -v ansible-playbook >/dev/null 2>&1; then + error "ansible-playbook is not installed" + exit 1 + fi + + # Check Python dependencies + if ! python3 -c "import bcrypt" >/dev/null 2>&1; then + warning "python3-bcrypt not found, installing..." + sudo apt-get update && sudo apt-get install -y python3-bcrypt + fi + + success "Prerequisites check passed" +} + +# Show configuration +show_configuration() { + log "Reading configuration from $INVENTORY_FILE..." + + # Extract configuration from inventory file + POSTGRES_VERSION=$(grep -E "^\s*#.*PostgreSQL Version:" "$INVENTORY_FILE" | awk '{print $NF}' || echo "15") + POSTGRES_PORT=$(grep -E "^\s*#.*PostgreSQL Port:" "$INVENTORY_FILE" | awk '{print $NF}' || echo "5433") + STORAGE_DEVICE=$(grep -E "^\s*#.*Storage Device:" "$INVENTORY_FILE" | awk '{print $NF}' || echo "/dev/nvme2n1") + MOUNT_POINT=$(grep -E "^\s*#.*Mount Point:" "$INVENTORY_FILE" | awk '{print $NF}' || echo "/srv/postgres") + TARGET_HOST=$(grep "ansible_host=" "$INVENTORY_FILE" | awk -F'ansible_host=' '{print $2}' | awk '{print $1}') + + cat << EOF + +Configuration Summary: + PostgreSQL Version: $POSTGRES_VERSION + Target Host: $TARGET_HOST + Port: $POSTGRES_PORT + Storage Device: $STORAGE_DEVICE + Mount Point: $MOUNT_POINT + Secrets Directory: $SECRETS_DIR + Inventory File: $INVENTORY_FILE + Playbook: $PLAYBOOK_FILE + +Security Features: + ✓ 16-character secure password generation + ✓ MD5 password encryption + ✓ Private subnet deployment (no SSL overhead) + ✓ Connection and statement audit logging + ✓ Kubernetes secrets with base64 encoding + ✓ Proper file permissions (0600 for secrets) + ✓ Separation of sensitive and non-sensitive data + +EOF +} + +# Test connectivity +test_connectivity() { + log "Testing connectivity to PostgreSQL servers..." + + if ansible postgresql_servers -i "$INVENTORY_FILE" -m ping >/dev/null 2>&1; then + success "Connectivity test passed" + else + error "Connectivity test failed" + echo "Please check your inventory file and SSH configuration" + exit 1 + fi +} + +# Clean up previous secrets +cleanup_previous_secrets() { + if [[ -d "$SECRETS_DIR" ]]; then + warning "Previous secrets directory found, cleaning up..." + rm -rf "$SECRETS_DIR" + fi + + # Create secure directory + mkdir -p "$SECRETS_DIR" + chmod 700 "$SECRETS_DIR" + log "Created secure secrets directory: $SECRETS_DIR" +} + +# Main execution +main() { + print_banner + + # Parse command line arguments + KUBERNETES_NAMESPACE="postgres" + SECRET_NAME="postgres-postgresql" + AUTO_CONFIRM=false + + while [[ $# -gt 0 ]]; do + case $1 in + --namespace) + KUBERNETES_NAMESPACE="$2" + shift 2 + ;; + --secret-name) + SECRET_NAME="$2" + shift 2 + ;; + --auto-confirm) + AUTO_CONFIRM=true + shift + ;; + -h|--help) + cat << EOF +PostgreSQL Secure Setup Script + +Usage: $0 [OPTIONS] + +Options: + --namespace NAMESPACE Kubernetes namespace for secrets (default: postgres) + --secret-name NAME Name for Kubernetes secret (default: postgres-postgresql) + --auto-confirm Skip confirmation prompts + -h, --help Show this help message + +Examples: + $0 + $0 --namespace production --secret-name postgres-creds + $0 --auto-confirm + +EOF + exit 0 + ;; + *) + error "Unknown option: $1" + exit 1 + ;; + esac + done + + check_prerequisites + show_configuration + test_connectivity + cleanup_previous_secrets + + # Confirmation prompt + if [[ "$AUTO_CONFIRM" == "false" ]]; then + echo + read -p "Do you want to proceed with PostgreSQL secure setup? (y/N): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + log "Setup cancelled by user" + exit 0 + fi + fi + + log "Starting PostgreSQL secure setup..." + echo "==============================================" + + # Execute the secure playbook + if ansible-playbook -i "$INVENTORY_FILE" "$PLAYBOOK_FILE" \ + --extra-vars "kubernetes_namespace=$KUBERNETES_NAMESPACE" \ + --extra-vars "secret_name=$SECRET_NAME" \ + -v; then + + echo + echo "==============================================" + success "PostgreSQL secure setup completed successfully!" + + # Show generated files + if [[ -d "$SECRETS_DIR" ]]; then + echo + log "Generated Kubernetes files:" + ls -la "$SECRETS_DIR" + echo + success "Kubernetes secret files are ready for deployment" + success "Review the DEPLOYMENT_INSTRUCTIONS.md file for next steps" + + # Show file permissions + echo + log "File permissions (security check):" + find "$SECRETS_DIR" -type f -exec ls -l {} \; + + else + warning "Secrets directory not found. Check playbook execution." + fi + + else + error "PostgreSQL setup failed" + exit 1 + fi +} + +# Cleanup function for signals +cleanup() { + log "Script interrupted, cleaning up..." + # Add any cleanup operations here if needed + exit 1 +} + +# Set up signal handlers +trap cleanup SIGINT SIGTERM + +# Run main function +main "$@" diff --git a/deploy/postgres/ansible/setup-vm-inventory.sh b/deploy/postgres/ansible/setup-vm-inventory.sh new file mode 100644 index 0000000..21c8f51 --- /dev/null +++ b/deploy/postgres/ansible/setup-vm-inventory.sh @@ -0,0 +1,296 @@ +#!/bin/bash + +# VM Inventory Setup Script +# This script helps you create or update the Ansible inventory file for PostgreSQL setup + +set -e + +# Default values +INVENTORY_FILE="hosts.ini" +DEFAULT_USER="ubuntu" +DEFAULT_PORT="22" +DEFAULT_PG_VERSION="15" +DEFAULT_DEVICE="/dev/nvme2n1" +DEFAULT_MOUNT_POINT="/srv/postgres" +DEFAULT_NETWORK="10.1.0.0/16" +DEFAULT_PG_PORT="5433" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +print_usage() { + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " -f, --file FILE Inventory file name (default: hosts.ini)" + echo " -h, --help Show this help message" + echo "" + echo "This script will guide you through setting up your VM inventory." + echo "" + echo "Examples:" + echo " $0 # Interactive setup with default inventory file" + echo " $0 -f my-hosts.ini # Use custom inventory file name" +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -f|--file) + INVENTORY_FILE="$2" + shift 2 + ;; + -h|--help) + print_usage + exit 0 + ;; + *) + echo "Unknown option: $1" + print_usage + exit 1 + ;; + esac +done + +echo -e "${GREEN}=== VM Inventory Setup ===${NC}" +echo "This script will help you create the inventory file for PostgreSQL automation." +echo "" + +# Check if inventory file already exists +if [[ -f "$INVENTORY_FILE" ]]; then + echo -e "${YELLOW}Warning: Inventory file '$INVENTORY_FILE' already exists.${NC}" + echo "Current content:" + echo "----------------------------------------" + cat "$INVENTORY_FILE" + echo "----------------------------------------" + echo "" + read -p "Do you want to overwrite it? (y/N): " overwrite + if [[ $overwrite != [yY] && $overwrite != [yY][eE][sS] ]]; then + echo "Setup cancelled." + exit 0 + fi +fi + +# Collect VM information +echo -e "${BLUE}=== VM Connection Details ===${NC}" +echo "" + +# VM IP Address +while true; do + read -p "Enter your VM IP address: " vm_ip + if [[ -n "$vm_ip" ]]; then + break + else + echo -e "${RED}IP address cannot be empty.${NC}" + fi +done + +# SSH User +read -p "Enter SSH username (default: $DEFAULT_USER): " ssh_user +ssh_user="${ssh_user:-$DEFAULT_USER}" + +# SSH Key Path +echo "" +echo "SSH Key Configuration:" +echo "Please provide the path to your SSH private key file." +echo "Common locations:" +echo " • ~/.ssh/id_rsa" +echo " • ~/.ssh/your-key.pem" +echo " • ~/Downloads/your-aws-key.pem" +echo "" + +while true; do + read -p "Enter SSH private key file path: " ssh_key + if [[ -n "$ssh_key" ]]; then + # Expand tilde to home directory + ssh_key="${ssh_key/#\~/$HOME}" + + if [[ -f "$ssh_key" ]]; then + break + else + echo -e "${RED}File not found: $ssh_key${NC}" + echo "Please check the path and try again." + fi + else + echo -e "${RED}SSH key path cannot be empty.${NC}" + fi +done + +# SSH Port (optional) +read -p "Enter SSH port (default: $DEFAULT_PORT): " ssh_port +ssh_port="${ssh_port:-$DEFAULT_PORT}" + +# Server Name +read -p "Enter server name for inventory (default: vm_server): " server_name +server_name="${server_name:-vm_server}" + +echo "" +echo -e "${BLUE}=== PostgreSQL Configuration ===${NC}" +echo "These parameters will be used for PostgreSQL installation:" +echo "" + +# PostgreSQL Version +read -p "Enter PostgreSQL version (default: $DEFAULT_PG_VERSION): " pg_version +pg_version="${pg_version:-$DEFAULT_PG_VERSION}" + +# Storage Device +echo "" +echo "Storage Device Configuration:" +echo "This is the block device where PostgreSQL data will be stored." +echo "Common devices: /dev/sdb, /dev/nvme2n1, /dev/xvdf" +echo "" +read -p "Enter storage device path (default: $DEFAULT_DEVICE): " storage_device +storage_device="${storage_device:-$DEFAULT_DEVICE}" + +# Mount Point +echo "" +echo "Mount Point Configuration:" +echo "This is where the storage device will be mounted for PostgreSQL data." +echo "" +read -p "Enter mount point (default: $DEFAULT_MOUNT_POINT): " mount_point +mount_point="${mount_point:-$DEFAULT_MOUNT_POINT}" + +# Network CIDR +echo "" +echo "Network Configuration:" +echo "This is the network CIDR for PostgreSQL access control." +echo "Examples: 10.1.0.0/16, 192.168.1.0/24, 172.16.0.0/12" +echo "" +read -p "Enter network CIDR (default: $DEFAULT_NETWORK): " network_cidr +network_cidr="${network_cidr:-$DEFAULT_NETWORK}" + + +# PostgreSQL Port +echo "" +echo "PostgreSQL Port Configuration:" +echo "This is the port PostgreSQL will listen on." +echo "Default is 5432, but you might want to use a different port for security." +echo "" +read -p "Enter PostgreSQL port (default: $DEFAULT_PG_PORT): " pg_port +pg_port="${pg_port:-$DEFAULT_PG_PORT}" +echo "" +echo -e "${BLUE}=== Configuration Summary ===${NC}" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "VM Connection:" +echo " Server Name: $server_name" +echo " IP Address: $vm_ip" +echo " SSH User: $ssh_user" +echo " SSH Key: $ssh_key" +echo " SSH Port: $ssh_port" +echo "" +echo "PostgreSQL Configuration:" +echo " Version: $pg_version" +echo " Storage Device: $storage_device" +echo " Mount Point: $mount_point" +echo " Network CIDR: $network_cidr" +echo " PostgreSQL Port: $pg_port" +echo "" +echo "Inventory File: $INVENTORY_FILE" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" + +read -p "Create inventory file with these settings? (y/N): " confirm +if [[ $confirm != [yY] && $confirm != [yY][eE][sS] ]]; then + echo "Setup cancelled." + exit 0 +fi + +# Create the inventory file +echo -e "${GREEN}Creating inventory file...${NC}" + +cat > "$INVENTORY_FILE" << INVENTORY_EOF +# Ansible Inventory for PostgreSQL Setup on Virtual Machine +# Generated by setup-vm-inventory.sh on $(date) +# +# Configuration Summary: +# - PostgreSQL Version: $pg_version +# - Storage Device: $storage_device +# - Mount Point: $mount_point +# - Network CIDR: $network_cidr +# - PostgreSQL Port: $pg_port + +[postgresql_servers] +$server_name ansible_host=$vm_ip ansible_user=$ssh_user ansible_ssh_private_key_file=$ssh_key ansible_port=$ssh_port + +[postgresql_servers:vars] +# SSH and Ansible Configuration +ansible_become=yes +ansible_become_method=sudo +ansible_ssh_common_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' +ansible_python_interpreter=/usr/bin/python3 +ansible_ssh_timeout=30 +ansible_connect_timeout=30 + +# PostgreSQL Configuration Variables +postgresql_version=$pg_version +storage_device=$storage_device +mount_point=$mount_point +network_cidr=$network_cidr + +# Additional PostgreSQL Variables +postgresql_port=$pg_port + +# For local testing (if needed) +[local] +localhost ansible_connection=local +INVENTORY_EOF + +echo -e "${GREEN}✓ Inventory file '$INVENTORY_FILE' created successfully!${NC}" +echo "" + +# Test connectivity +echo -e "${BLUE}=== Testing Connectivity ===${NC}" +echo "Testing SSH connection to your VM..." + +if command -v ansible &> /dev/null; then + if ansible postgresql_servers -i "$INVENTORY_FILE" -m ping; then + echo -e "${GREEN}✓ Connection test successful!${NC}" + echo "" + echo -e "${GREEN}=== Setup Complete ===${NC}" + echo "Your inventory is ready with the following configuration:" + echo " • PostgreSQL Version: $pg_version" + echo " • Storage Device: $storage_device" + echo " • Mount Point: $mount_point" + echo " • Network CIDR: $network_cidr" +echo " • PostgreSQL Port: $pg_port" + echo "" + echo "You can now run your PostgreSQL automation scripts:" + echo " ./run-postgresql-playbook.sh" + echo "" + echo "Or test with manual commands like:" + echo " ansible-playbook -i $INVENTORY_FILE your-playbook.yml" + echo "" + else + echo -e "${RED}✗ Connection test failed${NC}" + echo "" + echo -e "${YELLOW}=== Troubleshooting Tips ===${NC}" + echo "1. Check if your VM is running and accessible" + echo "2. Verify the IP address is correct" + echo "3. Ensure SSH key has correct permissions (600)" + echo "4. Check if the SSH user has sudo privileges" + echo "5. Verify firewall/security groups allow SSH access" + echo "6. Make sure the storage device exists on the target VM" + echo "" + echo "You can test manually with:" + echo " ssh -i $ssh_key $ssh_user@$vm_ip" + echo "" + echo "The inventory file has been created, but please fix connectivity issues before proceeding." + fi +else + echo -e "${YELLOW}⚠ Ansible not found - skipping connectivity test${NC}" + echo "Install Ansible to test connectivity: sudo apt install ansible" + echo "" + echo -e "${GREEN}=== Setup Complete ===${NC}" + echo "Inventory file created with your configuration." +fi + +echo "" +echo -e "${BLUE}=== Inventory File Created ===${NC}" +echo "File: $INVENTORY_FILE" +echo "Location: $(pwd)/$INVENTORY_FILE" +echo "" +echo "You can review the inventory file with:" +echo " cat $INVENTORY_FILE" diff --git a/helm/postgres-upgrade/templates/postgres-upgrade-job.yaml b/helm/postgres-upgrade/templates/postgres-upgrade-job.yaml index 077889a..91770a4 100644 --- a/helm/postgres-upgrade/templates/postgres-upgrade-job.yaml +++ b/helm/postgres-upgrade/templates/postgres-upgrade-job.yaml @@ -3,35 +3,35 @@ kind: Job metadata: name: postgres-upgrade-job labels: - {{- include "postgres-upgrade.labels" . | nindent 4 }} +{{- include "postgres-upgrade.labels" . | nindent 4 }} spec: backoffLimit: 0 template: metadata: labels: - {{- include "postgres-upgrade.selectorLabels" . | nindent 8 }} +{{- include "postgres-upgrade.selectorLabels" . | nindent 8 }} app.kubernetes.io/name: postgres-upgrade-job sidecar.istio.io/inject: "false" spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: - {{- toYaml . | nindent 8 }} +{{- toYaml . | nindent 8 }} {{- end }} serviceAccountName: {{ include "postgres-upgrade.serviceAccountName" . }} securityContext: - {{- toYaml .Values.jobSecurityContext | nindent 8 }} +{{- toYaml .Values.jobSecurityContext | nindent 8 }} restartPolicy: Never # This is one time job containers: - name: {{ .Chart.Name }} securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} +{{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} env: - - name: SU_USER_PWD - value: {{ .Values.database.su.su_password }} - - name: DB_USER_PWD - value: {{ .Values.database.dbu.dbu_password }} + - name: SU_USER_PWD + value: "{{ .Values.database.su.su_password }}" + - name: DB_USER_PWD + value: "{{ .Values.database.dbu.dbu_password }}" envFrom: - configMapRef: name: postgres-upgrade-env-config diff --git a/helm/postgres-upgrade/values.yaml b/helm/postgres-upgrade/values.yaml index 086ca20..83f54c7 100644 --- a/helm/postgres-upgrade/values.yaml +++ b/helm/postgres-upgrade/values.yaml @@ -5,10 +5,10 @@ replicaCount: 1 image: - repository: mosipdev/postgres-upgrade + repository: mosipqa/postgres-upgrade pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: release-1.2.0.1 + tag: develop imagePullSecrets: [] nameOverride: "" diff --git a/postgres-upgrade/Dockerfile b/postgres-upgrade/Dockerfile index b4bdcfe..fba0746 100644 --- a/postgres-upgrade/Dockerfile +++ b/postgres-upgrade/Dockerfile @@ -25,6 +25,8 @@ COPY ./repos/admin-services/db_upgrade_scripts . COPY ./repos/keymanager/db_upgrade_scripts . COPY ./repos/audit-manager/db_upgrade_scripts . COPY ./repos/resident-services/db_upgrade_scripts . +COPY ./repos/otp-manager/db_upgrade_scripts . +COPY ./repos/digital-card-service/db_upgrade_scripts . ENV MOSIP_DB_NAME= ENV DB_SERVERIP= diff --git a/postgres-upgrade/get_db_data.sh b/postgres-upgrade/get_db_data.sh index 94b02d8..710b95a 100755 --- a/postgres-upgrade/get_db_data.sh +++ b/postgres-upgrade/get_db_data.sh @@ -10,6 +10,7 @@ while IFS= read -r line; do branch=release-1.3.0 git clone -b $branch https://github.com/mosip/$REPO_NAME.git ./repos/$REPO_NAME elif [ $REPO_NAME == "esignet" ] + then branch=release-1.2.x git clone -b $branch https://github.com/mosip/$REPO_NAME.git ./repos/$REPO_NAME else diff --git a/postgres-upgrade/repo-list.txt b/postgres-upgrade/repo-list.txt index 750c89c..d14c836 100644 --- a/postgres-upgrade/repo-list.txt +++ b/postgres-upgrade/repo-list.txt @@ -9,4 +9,6 @@ id-repository partner-management-services pre-registration registration -resident +resident-services +digital-card-service +otp-manager diff --git a/postgres-upgrade/upgrade.csv b/postgres-upgrade/upgrade.csv index 8c3fe9d..89638e9 100644 --- a/postgres-upgrade/upgrade.csv +++ b/postgres-upgrade/upgrade.csv @@ -31,3 +31,63 @@ mosip_resident,1.2.0.1-B1,1.2.0.1-B2,upgrade mosip_resident,1.2.0.1-B2,1.2.0.1,upgrade mosip_hotlist,1.1.5.5,1.2.0.1-B1,upgrade mosip_hotlist,1.2.0.1-B1,1.2.0.1,upgrade +mosip_kernel,1.2.0.1,1.2.0.2,upgrade +mosip_kernel,1.2.0.2,1.3.0,upgrade +mosip_prereg,1.2.0.1,1.2.0.2,upgrade +mosip_prereg,1.2.0.2,1.2.0.3,upgrade +mosip_prereg,1.2.0.3,1.3.0,upgrade +mosip_regprc,1.2.0.1,1.2.0.2,upgrade +mosip_regprc,1.2.0.2,1.2.1.0,upgrade +mosip_regprc,1.2.1.0,1.2.1.1,upgrade +mosip_regprc,1.2.1.1,1.2.1.2,upgrade +mosip_regprc,1.2.1.2,1.3.0,upgrade +mosip_credential,1.2.0.1,1.2.1.0,upgrade +mosip_credential,1.2.1.0,1.2.2.0,upgrade +mosip_credential,1.2.2.0,1.2.2.1,upgrade +mosip_credential,1.2.2.1,1.2.2.2,upgrade +mosip_credential,1.2.2.2,1.2.2.3,upgrade +mosip_credential,1.2.2.3,1.2.2.4,upgrade +mosip_credential,1.2.2.4,1.3.0,upgrade +mosip_idmap,1.2.0.1,1.2.1.0,upgrade +mosip_idmap,1.2.1.0,1.2.2.0,upgrade +mosip_idmap,1.2.2.0,1.2.2.1,upgrade +mosip_idmap,1.2.2.1,1.2.2.2,upgrade +mosip_idmap,1.2.2.2,1.2.2.3,upgrade +mosip_idmap,1.2.2.3,1.2.2.4,upgrade +mosip_idmap,1.2.2.4,1.3.0,upgrade +mosip_idrepo,1.2.0.1,1.2.1.0,upgrade +mosip_idrepo,1.2.1.0,1.2.2.0,upgrade +mosip_idrepo,1.2.2.0,1.2.2.1,upgrade +mosip_idrepo,1.2.2.1,1.2.2.2,upgrade +mosip_idrepo,1.2.2.2,1.2.2.3,upgrade +mosip_idrepo,1.2.2.3,1.2.2.4,upgrade +mosip_idrepo,1.2.2.4,1.3.0,upgrade +mosip_ida,1.2.0.1,1.2.1.0,upgrade +mosip_ida,1.2.1.0,1.2.1.1,upgrade +mosip_ida,1.2.1.1,1.2.1.2,upgrade +mosip_ida,1.2.1.2,1.3.0,upgrade +mosip_pms,1.2.0.1,1.2.1.0,upgrade +mosip_pms,1.2.1.0,1.2.2.0,upgrade +mosip_pms,1.2.2.0,1.2.2.1,upgrade +mosip_pms,1.2.2.1,1.2.2.2,upgrade +mosip_master,1.2.0.1,1.2.1.0,upgrade +mosip_master,1.2.1.0,1.2.1.1,upgrade +mosip_master,1.2.1.1,1.2.1.2,upgrade +mosip_master,1.2.1.2,1.2.1.3,upgrade +mosip_master,1.2.1.3,1.2.1.4,upgrade +mosip_master,1.2.1.4,1.3.0,upgrade +mosip_keymgr,1.2.0.1,1.2.1.0,upgrade +mosip_keymgr,1.2.1.0,1.3.0,upgrade +mosip_audit,1.2.0.1,1.3.0,upgrade +mosip_resident,1.2.0.1,1.2.1.0,upgrade +mosip_resident,1.2.1.0,1.2.1.1,upgrade +mosip_resident,1.2.1.1,1.2.1.2,upgrade +mosip_resident,1.2.1.2,1.3.0,upgrade +mosip_hotlist,1.2.0.1,1.2.1.0,upgrade +mosip_hotlist,1.2.1.0,1.2.1.1,upgrade +mosip_hotlist,1.2.1.1,1.2.1.2,upgrade +mosip_hotlist,1.2.1.2,1.2.1.3,upgrade +mosip_hotlist,1.2.1.3,1.2.1.4,upgrade +mosip_hotlist,1.2.1.4,1.3.0,upgrade +mosip_otp,1.2.0.1,1.3.0,upgrade +mosip_digitalcard,1.2.0.1,1.3.0,upgrade