diff --git a/blueprints/fedramp-high/gemini-enterprise/README.md b/blueprints/fedramp-high/gemini-enterprise/README.md index bb9d3225b..6ed805d99 100644 --- a/blueprints/fedramp-high/gemini-enterprise/README.md +++ b/blueprints/fedramp-high/gemini-enterprise/README.md @@ -1,6 +1,6 @@ # Gemini Enterprise for FedRAMP High - Comprehensive Documentation -**Version:** 1.0.0 +**Version:** 1.2.0 **Compliance:** FedRAMP High / IL4+ **Scope:** Full System Documentation @@ -22,11 +22,11 @@ ## 1. Executive Overview -This blueprint deploys a secure and compliant environment for hosting Gemini Enterprise on Google Cloud Platform, specifically tailored for FedRAMP High requirements. It leverages Vertex AI Search and Discovery Engine. The deployment is divided into two main Terraform stages (`gemini-stage-0` and `gemini-stage-1`) and interacts with the `gem4gov` CLI tool. +This blueprint deploys a secure and compliant environment for hosting Gemini Enterprise on Google Cloud Platform, specifically tailored for FedRAMP High requirements. It leverages the Vertex AI Search and Discovery Engine APIs. The deployment is divided into two main Terraform stages (`gemini-stage-0` and `gemini-stage-1`) and interacts with the `gem4gov` CLI tool. **This blueprint supports both EXTERNAL and INTERNAL load balancer deployments, configurable via the `deployment_type` variable in `gemini-stage-0/terraform.tfvars`.** -It is designed to be **fully automated** via the `deploy.sh` script, which serves as the central management interface for the entire lifecycle of the application—from initial infrastructure provisioning to application updates and certificate management. +It is designed to be **fully automated** via the `deploy.sh` script, which serves as the central management interface for the entire lifecycle of the application—from initial infrastructure provisioning to application updates and ongoing maintenance. ### Overall Goal @@ -36,34 +36,47 @@ The primary goal is to provide a turnkey ("Push Button") solution for setting up The blueprint establishes a robust infrastructure including: -1. **Networking:** - - **Greenfield:** Deploys a dedicated Virtual Private Cloud (VPC) with private subnets to isolate the environment. - - **Brownfield (Stellar Engine):** Automatically discovers and attaches to the existing Shared VPC and subnets provided by the Stellar Engine Host Project. - - **Load Balancing:** - - **Regional External LB:** Equipped with Cloud Armor (WAF) and Identity Aware Proxy (IAP) for zero-trust, hardened external access. - - **Regional Internal LB:** Limits access to traffic from the VPC/VPN/Interconnect. -2. **Data Storage:** CMEK-encrypted Google Cloud Storage (GCS) buckets and BigQuery datasets to securely store data for Discovery Engine. -3. **Discovery Engine:** Configuration of Discovery Engine data stores, and connectors for GCS and BigQuery. -4. **Security Controls:** - - **Identity-Aware Proxy (IAP):** Enforces fine-grained access control based on user identity and context (Supports Google Identity & Workforce Identity). - - **Access Context Manager:** Defines granular access policies (Time, Location, Device). - - **Chrome Enterprise Premium (Zero Trust):** Optional integration for strict device-based access policies. - - **Cloud Armor:** WAF capabilities and DDoS protection (US-only geo-fencing). - - **CMEK (Customer-managed encryption key):** Ensures data at rest is encrypted with customer-managed keys. - - **IAM & Org Policies:** Least privilege roles and automated policy validation. +**1. Core Infrastructure (`gemini-stage-0`)** +- **Networking:** + - **VPC & Subnets:** `google_compute_network` and `google_compute_subnetwork` for private and proxy-only subnets (Greenfield) or data source attachment to Shared VPC (Brownfield). + - **IP Addressing:** `google_compute_address` for reserved internal/external Load Balancer IP. + - **Network Endpoints:** `google_compute_region_network_endpoint_group` and `google_compute_region_network_endpoint` mapping to the Discovery Engine FQDN (`vertexaisearch.cloud.google.com`). + - **HTTP Redirect (External LB):** `google_compute_region_url_map`, `google_compute_region_target_http_proxy`, and `google_compute_forwarding_rule` to ensure all HTTP traffic upgrades to HTTPS. +- **Security & Access Control:** + - **Cloud Armor (WAF):** `google_compute_region_security_policy` with predefined OWASP rules and US-only geo-fencing. + - **Access Context Manager:** `google_access_context_manager_access_level` defining conditions like Time of Day, IP Restrictions, Expiration Dates, and leniency tiers for Chrome Enterprise Premium device identity. + - **IAM Bindings:** `google_project_iam_member` ensuring least privilege for Gemini Enterprise Admins, Gemini Enterprise End Users, and required Service Accounts. +- **Data Storage & Encryption:** + - **KMS / CMEK:** `google_kms_key_ring`, `google_kms_crypto_key`, and `google_kms_crypto_key_iam_member` for encrypting Discovery Engine data stores. + - **Discovery Engine Settings:** `google_discovery_engine_cmek_config` and `google_discovery_engine_acl_config`. + - **Data Sources:** `google_storage_bucket` (GCS) and `google_bigquery_dataset` (BQ) acting as safe data hubs. + +**2. Gemini Enterprise (`gem4gov-cli`):** +- **Gemini Application:** Creates and configures the core Search Engine resource. +- **Data Stores:** Configures and attaches Cloud Storage and BigQuery data stores to the Gemini Enterprise application. + +**3. Application Frontend (`gemini-stage-1`)** +- **Gemini Application:** Creates the core Discovery Engine Application. +- **Data Stores:** Configures and attaches Cloud Storage and BigQuery data stores to the Gemini application. +- **Load Balancing:** + - **Backend Service:** `google_compute_region_backend_service` pointing to the Stage 0 NEG. + - **HTTPS Routing:** `google_compute_region_url_map` and `google_compute_region_target_https_proxy` (utilizing the managed/unmanaged SSL certificate). + - **Forwarding Rule:** `google_compute_forwarding_rule` to accept external/internal HTTPS traffic. +- **Identity-Aware Proxy (IAP):** + - **IAP Access Control:** `google_iap_web_region_backend_service_iam_member` binding the specific Admin/User Groups (or Workforce Identity Principals) to the Backend Service, enforcing the zero-trust boundary. ### Deployment Automation (`deploy.sh`) The `deploy.sh` script is the recommended way to interact with this blueprint. It handles: -1. **Interactive Configuration:** Guides you through every step, including Project selection, Authentication, and Deployment Topology (Greenfield vs. Brownfield). +1. **Interactive Configuration:** Guides you through every step, including authentication, project selection, prerequisite checking,and deployment topology selection (Greenfield vs. Brownfield). 2. **Automated Discovery:** - - **Context Awareness:** Automatically detects if you are in a "Bootstrap" or "Stellar Engine" environment. + - **Session Persistence:** Uses remote Terraform state to track resources that have been already deployed in a separate session - **Resource Discovery:** Finds existing constraints, keys, networks, and subnets to prevent misconfiguration. 3. **Variable Generation:** Auto-generates `terraform.tfvars` files for both stages, eliminating manual copy-pasting errors. 4. **Lifecycle Management:** Contains a **"Helper Functions"** menu for post-deployment tasks: - **Update App Compliance:** Ensure an existing Gemini Enterprise application meets the most recent compliance standards and includes any recently authorized features - - **Replace App / Routing:** Seamlessly swap the backend Gemini App while maintaining the Load Balancer. + - **Replace App / Routing:** Seamlessly swap the backend Gemini Enterprise application while maintaining the Load Balancer. - **Import Documents:** Interactive utility to ingest data into GCS/BigQuery Data Stores. - **Upload SSL Certificate:** Validates and uploads PEM certificates to GCP Certificate Manager. diff --git a/blueprints/fedramp-high/gemini-enterprise/deploy.sh b/blueprints/fedramp-high/gemini-enterprise/deploy.sh index 39b5da0a1..bd0550ba9 100755 --- a/blueprints/fedramp-high/gemini-enterprise/deploy.sh +++ b/blueprints/fedramp-high/gemini-enterprise/deploy.sh @@ -17,7 +17,6 @@ IS_CUSTOM="false" BUCKET_NAME="" STATE_BUCKET="" TENANT_IAC_PROJECT="" -DEFAULT_CMEK_KEY="" KMS_KEY_ID="" SKIP_PROMPTS="false" @@ -28,15 +27,7 @@ YELLOW='\033[0;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color -# --- Helper Functions --- - -get_tfvar_value() { - local file="$1" - local key="$2" - if [[ -f "$file" ]]; then - grep "^${key}\s*=" "$file" | head -n 1 | cut -d'=' -f2- | tr -d ' "' - fi -} +# --- Script Helper Functions --- print_header() { echo -e "${GREEN}============================================================${NC}" @@ -47,7 +38,7 @@ print_header() { pause() { echo "" - read -p "Press Enter to continue..." + read -p "Press Enter to acknowledge and continue..." } normalize_environment() { @@ -58,6 +49,64 @@ normalize_environment() { } check_dependencies() { + echo "" + echo -e "${BLUE}--- Check Dependencies ---${NC}" + echo "Validating required dependencies: tfenv, gcloud, terraform, pip3, python3, jq..." + + # Ensure ~/.tfenv/bin is in PATH early if it exists (resolves precedence issues) + if [[ -d "$HOME/.tfenv/bin" ]] && [[ ":$PATH:" != *":$HOME/.tfenv/bin:"* ]]; then + export PATH="$HOME/.tfenv/bin:$PATH" + hash -r 2>/dev/null || true + fi + + if command -v tfenv &> /dev/null; then + echo -e "${GREEN}tfenv is installed. Setting Terraform version to 1.12.2...${NC}" + tfenv install 1.12.2 + tfenv use 1.12.2 + else + echo -e "${YELLOW}tfenv is not installed. Checking OS...${NC}" + if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || "$OSTYPE" == "win32" ]]; then + echo -e "${RED}Windows detected.${NC}" + echo "Please manually install Terraform v1.12.2:" + echo "1. Download the binary from: https://releases.hashicorp.com/terraform/1.12.2/terraform_1.12.2_windows_amd64.zip" + echo "2. Extract the zip file." + echo "3. Add the dir containing terraform.exe to your system's PATH environment variable." + exit 1 + elif [[ "$OSTYPE" == "darwin"* || "$OSTYPE" == "linux-gnu"* ]]; then + echo -e "${YELLOW}MacOS/Linux detected. Installing tfenv manually...${NC}" + if [[ ! -d "$HOME/.tfenv" ]]; then + git clone --depth=1 https://github.com/tfutils/tfenv.git ~/.tfenv + else + echo -e "${GREEN}tfenv directory already exists at $HOME/.tfenv. Skipping clone.${NC}" + fi + + # Add to bashrc/bash_profile to ensure Linux/MacOS compat + for PROFILE in ~/.bash_profile ~/.bashrc; do + if [[ -f "$PROFILE" ]] && ! grep -q 'export PATH="$HOME/.tfenv/bin:$PATH"' "$PROFILE" 2>/dev/null; then + echo 'export PATH="$HOME/.tfenv/bin:$PATH"' >> "$PROFILE" + fi + done + # If neither file existed, just create .bashrc for Linux + if [[ ! -f ~/.bash_profile && ! -f ~/.bashrc ]]; then + echo 'export PATH="$HOME/.tfenv/bin:$PATH"' >> ~/.bashrc + fi + + export PATH="$HOME/.tfenv/bin:$PATH" + hash -r 2>/dev/null || true + echo -e "${GREEN}tfenv installed. Setting Terraform version to 1.12.2...${NC}" + tfenv install 1.12.2 + tfenv use 1.12.2 + + if [[ "$CLOUD_SHELL" == "true" ]]; then + echo -e "${YELLOW}IMPORTANT: You are running in Google Cloud Shell.${NC}" + echo -e "${YELLOW}To use the 'tfenv' or 'terraform' commands in your terminal AFTER this script finishes, you MUST run: ${GREEN}source ~/.bashrc${NC}" + fi + else + echo -e "${RED}Unsupported OS: $OSTYPE. Please install Terraform 1.12.2 manually before running this script.${NC}" + exit 1 + fi + fi + local missing=0 for cmd in gcloud terraform pip3 python3 jq; do if ! command -v $cmd &> /dev/null; then @@ -73,6 +122,10 @@ check_dependencies() { configure_data_stores() { # Expects GCS_LIST and BQ_LIST to be defined arrays in the calling scope + + # Clear existing import config to prevent duplicate import blocks on replay + > gemini-stage-0/import.tf + while true; do echo "" echo -e "${BLUE}--- Data Store Configuration ---${NC}" @@ -83,22 +136,120 @@ configure_data_stores() { case $DS_MENU_SEL in 1) - read -p "Enter Bucket Name (e.g., company-docs): " GCS_NAME - if [[ -n "$GCS_NAME" ]]; then - GCS_LIST+=("\"$GCS_NAME\"") - echo -e "${GREEN}Added GCS Bucket: ${GCS_NAME}${NC}" + read -p "Does the GCS bucket already exist? [y/N]: " BUCKET_EXISTS + + if [[ "$BUCKET_EXISTS" =~ ^[Yy]$ ]]; then + read -p "Enter Bucket Name (exclude 'gs://' prefix, e.g., company-docs): " GCS_NAME + GCS_NAME=$(echo "$GCS_NAME" | tr -dc 'a-z0-9_.-') # Sanitize bucket name + read -p "Enter Display Name for the Data Store: " DISPLAY_NAME + + if [[ -n "$GCS_NAME" && -n "$DISPLAY_NAME" ]]; then + CREATE_BUCKET="false" + echo -e "${YELLOW}GCS Bucket '${GCS_NAME}' already exists. It will NOT be created by Terraform.${NC}" + + read -p "Would you like to import this bucket into Terraform state to be managed? [y/N]: " IMPORT_GCS + if [[ "$IMPORT_GCS" =~ ^[Yy]$ ]]; then + CREATE_BUCKET="true" + GCS_INDEX=$(echo "$DISPLAY_NAME" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g' | sed 's/^-//;s/-$//') + cat <> gemini-stage-0/import.tf +import { + to = google_storage_bucket.gemini_enterprise_gcs_bucket["${GCS_INDEX}"] + id = "${PROJECT_ID}/${GCS_NAME}" +} +EOF + echo -e "${GREEN}Import configuration generated for ${GCS_NAME}.${NC}" + fi + + GCS_LIST+=("\"$GCS_INDEX\" = {name = \"$GCS_NAME\", create_bucket = $CREATE_BUCKET, display_name = \"$DISPLAY_NAME\"}") + echo -e "${GREEN}Added GCS Data Store: ${DISPLAY_NAME} (Bucket: ${GCS_NAME})${NC}" + else + echo -e "${RED}Invalid Bucket Name or Display Name.${NC}" + fi else - echo -e "${RED}Invalid Bucket Name.${NC}" + read -p "Enter Display Name for the new Data Store: " DISPLAY_NAME + + if [[ -n "$DISPLAY_NAME" ]]; then + # Clean display name: lowercase, replace spaces/special chars with hyphens + CLEAN_NAME=$(echo "$DISPLAY_NAME" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g' | sed 's/^-//;s/-$//') + + # Terraform appends project ID and '-data' to the key. + # Total length: len(PROJECT_ID) + 1 (hyphen) + len(KEY) + 5 ('-data') <= 63 + PREFIX_LEN=$((${#PROJECT_ID} + 6)) + MAX_LEN=$((63 - PREFIX_LEN)) + + if [[ $MAX_LEN -le 0 ]]; then + echo -e "${RED}Project ID is too long to automatically generate a valid bucket name. Please use an existing bucket.${NC}" + else + # Truncate and ensure it doesn't start/end with hyphen + CLEAN_NAME=$(echo "${CLEAN_NAME:0:$MAX_LEN}" | sed 's/^-//;s/-$//') + GCS_NAME="${PROJECT_ID}-${CLEAN_NAME}-data" + + CREATE_BUCKET="true" + echo -e "${GREEN}Data Store '${DISPLAY_NAME}' will generate Terraform Bucket key: '${GCS_NAME}'${NC}" + + GCS_LIST+=("\"$CLEAN_NAME\" = {name = \"$GCS_NAME\", create_bucket = $CREATE_BUCKET, display_name = \"$DISPLAY_NAME\"}") + echo -e "${GREEN}Added GCS Data Store: ${DISPLAY_NAME}${NC}" + fi + else + echo -e "${RED}Invalid Display Name.${NC}" + fi fi ;; 2) - read -p "Enter Dataset ID (must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_)): " BQ_DATASET - read -p "Enter Table ID: " BQ_TABLE - if [[ -n "$BQ_DATASET" && -n "$BQ_TABLE" ]]; then - BQ_LIST+=("{dataset_id = \"$BQ_DATASET\", table_id = \"$BQ_TABLE\"}") - echo -e "${GREEN}Added BigQuery Table: ${BQ_DATASET}.${BQ_TABLE}${NC}" + read -p "Does the BigQuery dataset already exist? [y/N]: " DATASET_EXISTS + + if [[ "$DATASET_EXISTS" =~ ^[Yy]$ ]]; then + read -p "Enter Dataset ID (e.g., my_dataset): " BQ_DATASET + BQ_DATASET=$(echo "$BQ_DATASET" | tr -dc 'a-zA-Z0-9_') # Sanitize dataset ID + read -p "Enter Table ID (e.g., my_table): " BQ_TABLE + BQ_TABLE=$(echo "$BQ_TABLE" | tr -dc 'a-zA-Z0-9_-') # Sanitize table ID + read -p "Enter Display Name for the Data Store: " DISPLAY_NAME + + if [[ -n "$BQ_DATASET" && -n "$BQ_TABLE" && -n "$DISPLAY_NAME" ]]; then + CREATE_DATASET="false" + echo -e "${YELLOW}BigQuery Dataset '${BQ_DATASET}' already exists. It will NOT be created by Terraform.${NC}" + + read -p "Would you like to import this dataset into Terraform state to be managed? [y/N]: " IMPORT_BQ + if [[ "$IMPORT_BQ" =~ ^[Yy]$ ]]; then + CREATE_DATASET="true" + BQ_INDEX=$(echo "$DISPLAY_NAME" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g' | sed 's/^-//;s/-$//') + cat <> gemini-stage-0/import.tf +import { + to = google_bigquery_dataset.gemini_enterprise_bq_dataset["${BQ_INDEX}"] + id = "projects/${PROJECT_ID}/datasets/${BQ_DATASET}" +} +EOF + echo -e "${GREEN}Import configuration generated for ${BQ_DATASET}.${NC}" + fi + + BQ_INDEX=$(echo "$DISPLAY_NAME" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g' | sed 's/^-//;s/-$//') + BQ_LIST+=("\"$BQ_INDEX\" = {dataset_id = \"$BQ_DATASET\", table_id = \"$BQ_TABLE\", create_dataset = $CREATE_DATASET, display_name = \"$DISPLAY_NAME\"}") + echo -e "${GREEN}Added BigQuery Data Store: ${DISPLAY_NAME} (Table: ${BQ_DATASET}.${BQ_TABLE})${NC}" + else + echo -e "${RED}Invalid Dataset ID, Table ID, or Display Name.${NC}" + fi else - echo -e "${RED}Invalid Dataset or Table ID.${NC}" + read -p "Enter Display Name for the new Data Store: " DISPLAY_NAME + + if [[ -n "$DISPLAY_NAME" ]]; then + # Clean display name for BQ Dataset: underscores and alphanumeric only + BQ_DATASET=$(echo "$DISPLAY_NAME" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/_/g' | sed 's/^_//;s/_$//') + read -p "Enter Table ID for the new Data Store (e.g., my_table): " BQ_TABLE + BQ_TABLE=$(echo "$BQ_TABLE" | tr -dc 'a-zA-Z0-9_-') # Sanitize table ID + + if [[ -n "$BQ_TABLE" ]]; then + CREATE_DATASET="true" + echo -e "${GREEN}Data Store '${DISPLAY_NAME}' will generate Terraform Dataset ID: '${BQ_DATASET}'${NC}" + + BQ_INDEX=$(echo "$DISPLAY_NAME" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g' | sed 's/^-//;s/-$//') + BQ_LIST+=("\"$BQ_INDEX\" = {dataset_id = \"$BQ_DATASET\", table_id = \"$BQ_TABLE\", create_dataset = $CREATE_DATASET, display_name = \"$DISPLAY_NAME\"}") + echo -e "${GREEN}Added BigQuery Data Store: ${DISPLAY_NAME} (Table: ${BQ_DATASET}.${BQ_TABLE})${NC}" + else + echo -e "${RED}Invalid Table ID.${NC}" + fi + else + echo -e "${RED}Invalid Display Name.${NC}" + fi fi ;; 3) @@ -109,11 +260,17 @@ configure_data_stores() { ;; esac done + + # Clean up empty import.tf if no existing resources were imported + if [[ ! -s "gemini-stage-0/import.tf" ]]; then + rm -f gemini-stage-0/import.tf + fi } # --- Authentication & Setup --- auth_and_project_setup() { + echo "" echo -e "${BLUE}--- Authentication & Project Selection ---${NC}" # 1. Google Account Check @@ -127,21 +284,7 @@ auth_and_project_setup() { echo -e "Now authenticated as: ${YELLOW}${CURRENT_ACCOUNT}${NC}" fi - # 2. ADC Check - echo "Checking Application Default Credentials (ADC)..." - if gcloud auth application-default print-access-token &>/dev/null; then - echo -e "${GREEN}ADC is configured.${NC}" - else - echo -e "${YELLOW}Application Default Credentials not found.${NC}" - read -p "Do you want to authenticate ADC now? (y/N): " DO_AUTH - if [[ "$DO_AUTH" == "y" || "$DO_AUTH" == "Y" ]]; then - gcloud auth application-default login - else - echo "Warning: Proceeding without ADC. Terraform might fail." - fi - fi - - # 3. Project ID Selection + # 2. Project ID Selection CURRENT_PROJECT_ID=$(gcloud config get-value project 2>/dev/null) if [[ -n "$CURRENT_PROJECT_ID" ]]; then echo -e "Current Project ID: ${YELLOW}${CURRENT_PROJECT_ID}${NC}" @@ -150,6 +293,7 @@ auth_and_project_setup() { PROJECT_ID=$CURRENT_PROJECT_ID else read -p "Enter the Google Cloud Project ID: " PROJECT_ID + PROJECT_ID=$(echo "$PROJECT_ID" | tr -dc 'a-z0-9-') if [[ -n "$PROJECT_ID" ]]; then gcloud config set project "${PROJECT_ID}" fi @@ -158,6 +302,7 @@ auth_and_project_setup() { if [[ -z "$PROJECT_ID" ]]; then read -p "Enter the Google Cloud Project ID: " PROJECT_ID + PROJECT_ID=$(echo "$PROJECT_ID" | tr -dc 'a-z0-9-') if [[ -n "$PROJECT_ID" ]]; then gcloud config set project "${PROJECT_ID}" fi @@ -169,15 +314,87 @@ auth_and_project_setup() { fi # Set billing quota project - echo "Setting billing quota project..." - gcloud config set billing/quota_project "${PROJECT_ID}" - echo "Setting application default quota project..." - gcloud auth application-default set-quota-project "${PROJECT_ID}" + CURRENT_QUOTA_PROJ=$(gcloud config get-value billing/quota_project 2>/dev/null || echo "") + if [[ "$CURRENT_QUOTA_PROJ" != "$PROJECT_ID" ]]; then + echo "Setting billing quota project..." + if ! gcloud config set billing/quota_project "${PROJECT_ID}" --quiet 2>/dev/null; then + echo -e "${YELLOW}Notice: Could not set billing/quota_project. Access may be restricted.${NC}" + fi + fi + + # Enable Service Usage API (Required for quota project validation) + if ! gcloud services list --enabled --project "${PROJECT_ID}" --filter="config.name:serviceusage.googleapis.com" --format="value(config.name)" 2>/dev/null | grep -q "serviceusage.googleapis.com"; then + echo "Ensuring Service Usage API is enabled..." + if ! gcloud --quiet services enable serviceusage.googleapis.com --project "${PROJECT_ID}" 2>/dev/null; then + echo -e "${YELLOW}Notice: Could not verify/enable Service Usage API. Proceeding...${NC}" + fi + fi + + # Set application default quota project + ADC_FILE="$HOME/.config/gcloud/application_default_credentials.json" + CURRENT_ADC_QUOTA="" + if [[ -f "$ADC_FILE" ]]; then + CURRENT_ADC_QUOTA=$(jq -r '.quota_project_id // empty' "$ADC_FILE" 2>/dev/null || echo "") + fi + + if [[ "$CURRENT_ADC_QUOTA" != "$PROJECT_ID" ]]; then + echo "Setting application default quota project..." + if ! gcloud --quiet auth application-default set-quota-project "${PROJECT_ID}" 2>/dev/null; then + echo -e "${YELLOW}Notice: ADC Quota project not set to '${PROJECT_ID}'. (Missing 'serviceusage.services.use'?)${NC}" + + if [[ "$SKIP_PROMPTS" != "true" ]]; then + echo -e "${BLUE}Please enter a project ID where you have 'serviceusage.services.use' permission to use for quota.${NC}" + read -p "Fallback Quota Project ID (leave blank to skip): " FALLBACK_PROJECT_ID + if [[ -n "$FALLBACK_PROJECT_ID" ]]; then + if gcloud --quiet auth application-default set-quota-project "${FALLBACK_PROJECT_ID}" &>/dev/null; then + echo -e "${GREEN}Quota project set to '${FALLBACK_PROJECT_ID}'.${NC}" + else + echo -e "${YELLOW}Notice: Failed to set fallback quota project.${NC}" + fi + fi + fi + fi + fi + + # 3. ADC Check + echo "Checking Application Default Credentials (ADC)..." + if [[ "$CLOUD_SHELL" == "true" ]]; then + echo -e "${YELLOW}Google Cloud Shell detected. Forcing interactive Application Default Credentials setup for Terraform compatibility...${NC}" + gcloud auth application-default login + elif gcloud auth application-default print-access-token &>/dev/null; then + echo -e "${GREEN}ADC is configured.${NC}" + + # Optional: Check if ADC matches current account (Best Effort) + # Note: We can't easily extract the account from the token without an API call, + # but we can ask the user if they want to be sure. + echo -e "${YELLOW}Note: Ensure your ADC matches the current account: ${CURRENT_ACCOUNT}${NC}" + if [[ "$SKIP_PROMPTS" != "true" ]]; then + read -p "Do you want to force refresh ADC credentials? (y/N): " REFRESH_ADC + if [[ "$REFRESH_ADC" =~ ^[Yy]$ ]]; then + gcloud auth application-default login + fi + fi + else + echo -e "${YELLOW}Application Default Credentials not found.${NC}" + read -p "Do you want to authenticate ADC now? (y/N): " DO_AUTH + if [[ "$DO_AUTH" == "y" || "$DO_AUTH" == "Y" ]]; then + gcloud auth application-default login + else + echo -e "${RED}WARNING: Proceeding without ADC. Terraform might fail.${NC}" + fi + fi # Discover Org ID echo "Discovering Organization ID..." - ORG_ID=$(gcloud projects get-ancestors "${PROJECT_ID}" --format="value(id)" | tail -n 1) - echo -e "Found Organization ID: ${YELLOW}${ORG_ID}${NC}" + ANCESTORS_INFO=$(gcloud projects get-ancestors "${PROJECT_ID}" --format="json" 2>/dev/null || echo "[]") + ORG_ID=$(echo "$ANCESTORS_INFO" | jq -r 'last(.[] | select(.type == "organization")) | .id // empty') + + if [[ -z "$ORG_ID" ]]; then + echo -e "${RED}WARNING: This project is not part of a GCP Organization ancestry chain.${NC}" + echo -e "${YELLOW}Discovery Engine AclConfig applied via Terraform will likely fail with 'Organization not associated with Cloud Identity' error.${NC}" + else + echo -e "Found Organization ID: ${YELLOW}${ORG_ID}${NC}" + fi # Discover Domain echo "Discovering Organization Domain..." @@ -186,7 +403,7 @@ auth_and_project_setup() { DOMAIN="${ORG_DOMAIN}" echo -e "Found Organization Domain: ${YELLOW}${DOMAIN}${NC}" else - echo -e "${YELLOW}Warning: Could not auto-discover Organization Domain.${NC}" + echo -e "${RED}WARNING: Could not auto-discover Organization Domain.${NC}" fi return 0 @@ -335,7 +552,29 @@ discover_infrastructure() { # Prioritize US Multi-Region for CMEK_US_KEYRING echo "Searching for CMEK Keyring in the US multi-region..." - CMEK_PROJECT_ID="${TENANT_IAC_PROJECT}" + # Determine the target project for CMEK + if [[ "$IS_BROWNFIELD" == "true" ]]; then + echo "Searching for 'cmek-*' project under 'StellarEngine-*' Assured Workloads folder..." + STELLAR_FOLDER_ID=$(gcloud resource-manager folders list --organization="${ORG_ID}" --filter="displayName~^StellarEngine-" --format="value(name)" 2>/dev/null | head -n 1) + + if [[ -n "$STELLAR_FOLDER_ID" ]]; then + STELLAR_FOLDER_ID=$(basename "$STELLAR_FOLDER_ID") + FOUND_CMEK_PROJECT=$(gcloud projects list --filter="name:cmek-* AND parent.id:${STELLAR_FOLDER_ID}" --format="value(projectId)" 2>/dev/null | head -n 1) + + if [[ -n "$FOUND_CMEK_PROJECT" ]]; then + echo -e "Found CMEK Project: ${GREEN}${FOUND_CMEK_PROJECT}${NC}" + CMEK_PROJECT_ID="${FOUND_CMEK_PROJECT}" + else + echo -e "${YELLOW}Could not find 'cmek-*' project under StellarEngine folder. Defaulting to ${TENANT_IAC_PROJECT}.${NC}" + CMEK_PROJECT_ID="${TENANT_IAC_PROJECT}" + fi + else + echo -e "${YELLOW}Could not find 'StellarEngine-*' folder. Defaulting to ${TENANT_IAC_PROJECT}.${NC}" + CMEK_PROJECT_ID="${TENANT_IAC_PROJECT}" + fi + else + CMEK_PROJECT_ID="${TENANT_IAC_PROJECT}" + fi # Capitalize first letter of Environment for KeyRing name (e.g. prod -> Prod) US_KEYRING_NAME="${CAP_ENV}-${TENANT}-keyring" US_KEYRING_ID="projects/${CMEK_PROJECT_ID}/locations/us/keyRings/${US_KEYRING_NAME}" @@ -397,10 +636,12 @@ discover_infrastructure() { elif [[ "$IS_CUSTOM" == "true" ]]; then read -p "Enter Environment identifier (e.g., prod): " ENVIRONMENT normalize_environment - read -p "Enter Tenant IaC Project ID: " TENANT_IAC_PROJECT + read -p "Enter Tenant IaC Project ID [${TENANT_IAC_PROJECT}]: " INPUT_TENANT_IAC_PROJECT + TENANT_IAC_PROJECT=${INPUT_TENANT_IAC_PROJECT:-$TENANT_IAC_PROJECT} # State Bucket - read -p "Enter Terraform State Bucket Name (leave blank to create): " STATE_BUCKET + read -p "Enter Terraform State Bucket Name (leave blank to create) [${STATE_BUCKET}]: " INPUT_STATE_BUCKET + STATE_BUCKET=${INPUT_STATE_BUCKET:-$STATE_BUCKET} if [[ -n "$STATE_BUCKET" ]]; then # Validate Encryption BUCKET_JSON=$(gcloud storage buckets describe "gs://${STATE_BUCKET}" --format="json" 2>/dev/null || echo "{}") @@ -416,9 +657,14 @@ discover_infrastructure() { fi fi - read -p "Enter CMEK Project ID: " CMEK_PROJECT_ID - read -p "Enter US Multi-Region Keyring ID (optional): " CMEK_US_KEYRING - read -p "Enter US Gemini Resources Key ID (optional): " CMEK_US_RESOURCES_KEY + read -p "Enter CMEK Project ID [${CMEK_PROJECT_ID}]: " INPUT_CMEK_PROJECT + CMEK_PROJECT_ID=${INPUT_CMEK_PROJECT:-$CMEK_PROJECT_ID} + + read -p "Enter US Multi-Region Keyring ID (optional) [${CMEK_US_KEYRING}]: " INPUT_CMEK_KEYRING + CMEK_US_KEYRING=${INPUT_CMEK_KEYRING:-$CMEK_US_KEYRING} + + read -p "Enter US Gemini Resources Key ID (optional) [${CMEK_US_RESOURCES_KEY}]: " INPUT_CMEK_GEMINI_KEY + CMEK_US_RESOURCES_KEY=${INPUT_CMEK_GEMINI_KEY:-$CMEK_US_RESOURCES_KEY} else # Greenfield @@ -435,6 +681,50 @@ discover_infrastructure() { return 0 } +# --- State Hydration --- + +hydrate_from_state() { + # Check if we have a bucket to read from (either BUCKET_NAME or derived from STATE_BUCKET) + local bucket="" + if [[ -n "$BUCKET_NAME" ]]; then + bucket="$BUCKET_NAME" + elif [[ -n "$STATE_BUCKET" ]]; then + bucket=$(echo "$STATE_BUCKET" | sed 's#gs://##' | sed 's/\/$//') + export BUCKET_NAME="$bucket" + fi + + if [[ -z "$bucket" ]]; then + return 0 + fi + + echo "Checking for existing state in gs://${bucket}..." + STATE_CONTENT=$(gcloud storage cat "gs://${bucket}/terraform/state/stage-0/default.tfstate" 2>/dev/null || echo "{}") + + # Project ID + if [[ -z "$PROJECT_ID" ]]; then + VAL=$(echo "$STATE_CONTENT" | jq -r '.outputs.main_project_id.value // empty') + if [[ -n "$VAL" ]]; then + PROJECT_ID="$VAL" + echo -e "Hydrated Project ID from state: ${YELLOW}${PROJECT_ID}${NC}" + fi + fi + + # Region + if [[ -z "$REGION" ]]; then + VAL=$(echo "$STATE_CONTENT" | jq -r '.outputs.region.value // empty') + if [[ -n "$VAL" ]]; then + REGION="$VAL" + echo -e "Hydrated Region from state: ${YELLOW}${REGION}${NC}" + fi + fi + + # Load Balancer IP (Useful for later steps) + VAL=$(echo "$STATE_CONTENT" | jq -r '.outputs.gemini_enterprise_ip.value // empty') + if [[ -n "$VAL" ]]; then + export GEMINI_IP="$VAL" + fi +} + ensure_prerequisites() { echo "" echo -e "${BLUE}--- Ensuring Prerequisites ---${NC}" @@ -495,6 +785,9 @@ ensure_prerequisites() { BUCKET_PROJECT="${PROJECT_ID}" fi + # Ensure Storage Service Agent exists + gcloud beta services identity create --service=storage.googleapis.com --project="${BUCKET_PROJECT}" &>/dev/null || true + BUCKET_PROJECT_NUMBER=$(gcloud projects describe "${BUCKET_PROJECT}" --format="value(projectNumber)") STORAGE_SA="service-${BUCKET_PROJECT_NUMBER}@gs-project-accounts.iam.gserviceaccount.com" if gcloud kms keys add-iam-policy-binding "${CMEK_STATE_KEY}" \ @@ -502,7 +795,7 @@ ensure_prerequisites() { --role="roles/cloudkms.cryptoKeyEncrypterDecrypter" --quiet &>/dev/null; then echo -e "${GREEN}Granted Storage SA (${STORAGE_SA}) access to CMEK State Key.${NC}" else - echo -e "${YELLOW}Warning: Could not grant Storage SA access. Check permissions.${NC}" + echo -e "${RED}WARNING: Could not grant Storage SA access. Check permissions.${NC}" fi fi @@ -520,8 +813,14 @@ ensure_prerequisites() { if ! gcloud storage buckets describe "gs://${BUCKET_NAME}" &>/dev/null; then echo "Creating state bucket gs://${BUCKET_NAME}..." + local create_output + local create_status=0 + # Grant Storage Service Agent access to CMEK if used (Double Check / Re-grant just in case) if [[ -n "$KMS_KEY_ID" ]]; then + # Ensure Storage Service Agent exists + gcloud beta services identity create --service=storage.googleapis.com --project="${PROJECT_ID}" &>/dev/null || true + PROJECT_NUMBER=$(gcloud projects describe "${PROJECT_ID}" --format="value(projectNumber)") STORAGE_SA="service-${PROJECT_NUMBER}@gs-project-accounts.iam.gserviceaccount.com" @@ -529,11 +828,24 @@ ensure_prerequisites() { gcloud kms keys add-iam-policy-binding "${KMS_KEY_ID}" \ --member="serviceAccount:${STORAGE_SA}" \ --role="roles/cloudkms.cryptoKeyEncrypterDecrypter" \ - --project="${CMEK_PROJECT_ID}" &>/dev/null || echo "Warning: Failed to grant IAM binding on key." + --project="${CMEK_PROJECT_ID}" &>/dev/null || echo -e "${RED}WARNING: Failed to grant IAM binding on key.${NC}" - gcloud storage buckets create "gs://${BUCKET_NAME}" --project "${PROJECT_ID}" --location "us" --uniform-bucket-level-access --default-encryption-key="${KMS_KEY_ID}" + create_output=$(gcloud storage buckets create "gs://${BUCKET_NAME}" --project "${PROJECT_ID}" --location "us" --uniform-bucket-level-access --default-encryption-key="${KMS_KEY_ID}" 2>&1) || create_status=$? else - gcloud storage buckets create "gs://${BUCKET_NAME}" --project "${PROJECT_ID}" --location "us" --uniform-bucket-level-access + create_output=$(gcloud storage buckets create "gs://${BUCKET_NAME}" --project "${PROJECT_ID}" --location "us" --uniform-bucket-level-access 2>&1) || create_status=$? + fi + + if [[ $create_status -eq 0 ]]; then + echo -e "${GREEN}Bucket created successfully!${NC}" + elif [[ "$create_output" == *"409"* && "$create_output" == *"namespace"* ]]; then + echo -e "${RED}${create_output}${NC}" + echo -e "${RED}The bucket name '${BUCKET_NAME}' is already taken globally.${NC}" + echo -e "${YELLOW}Please restart the script and select a new, unique PREFIX and/or ENVIRONMENT identifier to ensure clean infrastructure alignment.${NC}" + exit 1 + else + echo -e "${RED}Failed to create state bucket:${NC}" + echo "$create_output" + exit 1 fi else echo -e "Using Terraform State Bucket: ${GREEN}${BUCKET_NAME}${NC}" @@ -550,33 +862,56 @@ ensure_prerequisites() { BUCKET_PROJECT="${PROJECT_ID}" fi - # Construct Name + # Construct Initial Name NEW_BUCKET_NAME="${PREFIX}-${ENVIRONMENT}-${TENANT}-iac-0" echo "Creating Bucket '${NEW_BUCKET_NAME}' in ${REGION}..." + # Note: If REGION != 'us' and Key is 'us', this might fail if not dual-region. - # Attempting creation. - if ! gcloud storage buckets create "gs://${NEW_BUCKET_NAME}" \ - --project="${BUCKET_PROJECT}" \ - --location="${REGION}" \ - --default-encryption-key="${CMEK_STATE_KEY}" \ - --uniform-bucket-level-access; then - - echo -e "${RED}Failed to create bucket. Retrying with 'US' location if Key is US...${NC}" + # Capture output and exit status + local create_output + local create_status=0 + + if [[ -n "$CMEK_STATE_KEY" ]]; then + create_output=$(gcloud storage buckets create "gs://${NEW_BUCKET_NAME}" \ + --project="${BUCKET_PROJECT}" \ + --location="${REGION}" \ + --default-encryption-key="${CMEK_STATE_KEY}" \ + --uniform-bucket-level-access 2>&1) || create_status=$? + # Fallback logic if region mismatch suspected - if [[ "$CMEK_STATE_KEY" == *"/locations/us/"* ]]; then - gcloud storage buckets create "gs://${NEW_BUCKET_NAME}" \ + if [[ $create_status -ne 0 && "$create_output" != *"409"* && "$CMEK_STATE_KEY" == *"/locations/us/"* ]]; then + echo -e "${RED}Failed to create bucket with CMEK in ${REGION}. Retrying with 'US' location...${NC}" + create_output=$(gcloud storage buckets create "gs://${NEW_BUCKET_NAME}" \ --project="${BUCKET_PROJECT}" \ --location="us" \ --default-encryption-key="${CMEK_STATE_KEY}" \ - --uniform-bucket-level-access - else - return 1 - fi + --uniform-bucket-level-access 2>&1) || create_status=$? + fi + else + create_output=$(gcloud storage buckets create "gs://${NEW_BUCKET_NAME}" \ + --project="${BUCKET_PROJECT}" \ + --location="${REGION}" \ + --uniform-bucket-level-access 2>&1) || create_status=$? + fi + + if [[ $create_status -eq 0 ]]; then + # Success + echo -e "${GREEN}Bucket created successfully!${NC}" + STATE_BUCKET="${NEW_BUCKET_NAME}" + echo -e "Using Terraform State Bucket: ${GREEN}${STATE_BUCKET}${NC}" + elif [[ "$create_output" == *"409"* && "$create_output" == *"namespace"* ]]; then + # Conflict on globally unique name + echo -e "${RED}${create_output}${NC}" + echo -e "${RED}The bucket name '${NEW_BUCKET_NAME}' is already taken globally.${NC}" + echo -e "${YELLOW}Please restart the script and select a new, unique PREFIX and/or ENVIRONMENT identifier to ensure clean infrastructure alignment.${NC}" + exit 1 + else + # Other error + echo -e "${RED}Failed to create bucket:${NC}" + echo "$create_output" + exit 1 fi - - STATE_BUCKET="${NEW_BUCKET_NAME}" - echo -e "Using Terraform State Bucket: ${GREEN}${STATE_BUCKET}${NC}" fi echo -e "${GREEN}Prerequisites met successfully${NC}" @@ -666,7 +1001,7 @@ check_org_policies() { fi if [[ "$failed" -eq 1 ]]; then - echo -e "${YELLOW}WARNING: One or more Organization Policies may prevent deployment.${NC}" + echo -e "${RED}WARNING: One or more Organization Policies may prevent deployment.${NC}" read -p "Do you want to proceed anyway? (y/N): " PROCEED if [[ "$PROCEED" != "y" && "$PROCEED" != "Y" ]]; then return 1 @@ -676,7 +1011,6 @@ check_org_policies() { } configure_access_policies() { - echo -e "${BLUE}--- Configure Access Policies ---${NC}" # Initialize Defaults CREATE_IP_BASED_ACCESS="true" @@ -728,7 +1062,7 @@ configure_access_policies() { echo "" echo "Enter IP ranges allowed to access the Load Balancer (CIDR format)." echo "RECOMMENDED: Set this to the IP range of the agency's corporate gateway." - read -p "Enter IP Ranges (comma-separated, e.g., 203.0.113.0/24): " IP_RANGES_INPUT + read -p "Enter IP Ranges (comma-separated, e.g., 203.0.113.0/24) (leave blank if not required): " IP_RANGES_INPUT if [[ -n "$IP_RANGES_INPUT" ]]; then IFS=',' read -ra IP_ADDRS <<< "$IP_RANGES_INPUT" @@ -742,6 +1076,9 @@ configure_access_policies() { fi done ALLOWED_IPS="[$JSON_IPS]" + else + echo "No IPs provided. Updating configuration to disable IP based access." + CREATE_IP_BASED_ACCESS="false" fi fi @@ -889,6 +1226,7 @@ configure_stage_0() { # Check if we can reuse existing config if [[ -f "gemini-stage-0/terraform.tfvars" ]]; then echo -e "${YELLOW}Found existing configuration.${NC}" + echo -e "${RED}WARNING: Answering 'n' will OVERWRITE existing gemini-stage-0/terraform.tfvars${NC}" read -p "Reuse existing configuration? (Y/n): " REUSE_CONFIG if [[ "$REUSE_CONFIG" != "n" && "$REUSE_CONFIG" != "N" ]]; then echo -e "${GREEN}Using existing configuration.${NC}" @@ -916,6 +1254,9 @@ configure_stage_0() { fi if [[ -n "$BUCKET_NAME" ]]; then + # Ensure the tfvars file is updated with the sanitized bucket name + sed -i '' "s/terraform_state_bucket *= *\".*\"/terraform_state_bucket = \"${BUCKET_NAME}\"/" terraform.tfvars 2>/dev/null || sed -i "s/terraform_state_bucket *= *\".*\"/terraform_state_bucket = \"${BUCKET_NAME}\"/" terraform.tfvars + if terraform init -migrate-state -backend-config="bucket=${BUCKET_NAME}" -backend-config="prefix=terraform/state/stage-0" &>/dev/null; then if terraform state list | grep -q "google_kms_key_ring.created"; then echo -e "${YELLOW}KeyRing found in Terraform State. Updating existing config to use managed resource.${NC}" @@ -932,19 +1273,18 @@ configure_stage_0() { if terraform state list | grep -q "google_access_context_manager_access_level"; then echo -e "${YELLOW}Access Levels found in Terraform State. Setting flags to preserve resources.${NC}" - # If we find generic access levels we might want to default everything to true? - # Or just rely on the granular discovery logic below if we don't overwrite them? - # Requirement is to remove 'create_access_policies'. - # The new logic relies on 'configure_access_policies' which is called later. - # If reusing, users might skip 'configure_access_policies' if they say 'Reuse config'. - # If 'terraform.tfvars' exists, it has the granular flags. - # We should ensure granular flags are set to true if they are missing? - # Actually, if reusing config, we trust the tfvars file. - # So we probably don't need to SED replace create_access_policies anymore. - # We might want to remove the SED command that sets it. fi fi fi + + if [[ -n "$ENVIRONMENT" ]]; then + sed -i '' "s/environment *= *\".*\"/environment = \"${ENVIRONMENT}\"/" terraform.tfvars 2>/dev/null || sed -i "s/environment *= *\".*\"/environment = \"${ENVIRONMENT}\"/" terraform.tfvars + fi + + if [[ -n "$PREFIX" ]]; then + sed -i '' "s/prefix *= *\".*\"/prefix = \"${PREFIX}\"/" terraform.tfvars 2>/dev/null || sed -i "s/prefix *= *\".*\"/prefix = \"${PREFIX}\"/" terraform.tfvars + fi + cd .. return 0 @@ -973,7 +1313,8 @@ configure_stage_0() { echo -e "${BLUE}--- Compliance Regime (Assured Workloads) ---${NC}" echo "1. FedRAMP High (Default)" echo "2. IL4" - echo "3. None" + echo "3. IL5" + echo "4. None" read -p "What compliance regime will you be using? [1]: " REGIME_CHOICE REGIME_CHOICE=${REGIME_CHOICE:-1} @@ -990,9 +1331,14 @@ configure_stage_0() { REGIME_DISPLAY="IL4" ;; 3) - echo -e "${YELLOW}WARNING: Gemini for Government currently only supports deployment within FedRAMP High / IL4 Assured Workloads folders.${NC}" - echo -e "${YELLOW}Proceed at your own risk.${NC}" - read -p "Press Enter to acknowledge..." + COMPLIANCE_REGIME="IL5" + REGIME_DISPLAY="IL5" + ;; + 4) + echo -e "${RED}WARNING: Gemini for Government currently only supports deployment within FedRAMP High / IL4 Assured Workloads folders.${NC}" + echo -e "${RED}Proceed at your own risk.${NC}" + echo "" + read -p "Press Enter to acknowledge and continue..." ;; *) echo -e "${RED}Invalid selection. Defaulting to FedRAMP High.${NC}" @@ -1001,18 +1347,43 @@ configure_stage_0() { ;; esac + # Enable APIs based on compliance regime + if [[ "$COMPLIANCE_REGIME" == "IL5" ]]; then + echo "" + echo -e "${YELLOW}WARNING: Discovery Engine API is not currently included in the Assured Workloads Service Usage Allowlist Org Policy for IL5.${NC}" + echo -e "${YELLOW}Gemini for Government can only be used by creating an exception and adding discoveryengine.googleapis.com to the allowlist.${NC}" + read -p "Do you want to attempt to enable Discovery Engine and Certificate Manager APIs? [y/N]: " ENABLE_APIS_NOW + if [[ "$ENABLE_APIS_NOW" =~ ^[Yy]$ ]]; then + echo "Attempting to enable APIs..." + if ! gcloud services enable discoveryengine.googleapis.com certificatemanager.googleapis.com --project "${PROJECT_ID}"; then + echo -e "${RED}Warning: Failed to enable Discovery Engine or Certificate Manager APIs.${NC}" + echo -e "${YELLOW}This is expected if the APIs are not in your allowlist and you have not created an exception.${NC}" + fi + else + echo -e "${YELLOW}Skipping API enablement. You may need to enable them manually after configuring exceptions.${NC}" + fi + else + echo -e "${GREEN}Enabling Discovery Engine and Certificate Manager APIs automatically...${NC}" + if ! gcloud services enable discoveryengine.googleapis.com certificatemanager.googleapis.com --project "${PROJECT_ID}"; then + echo -e "${RED}Warning: Failed to enable Discovery Engine or Certificate Manager APIs.${NC}" + echo -e "${YELLOW}Please ensure you have permissions to enable these APIs or they are allowed by your Org Policy.${NC}" + fi + fi + if [[ -n "$COMPLIANCE_REGIME" ]]; then read -p "Is this project deployed in a ${REGIME_DISPLAY} Assured Workloads folder? (y/N): " IS_ASSURED if [[ "$IS_ASSURED" == "y" || "$IS_ASSURED" == "Y" ]]; then read -p "Enter the region (e.g., us-east4): " WORKLOAD_REGION if [[ -n "$WORKLOAD_REGION" ]]; then - echo "Fetching ${REGIME_DISPLAY} Assured Workload folders in ${WORKLOAD_REGION}..." + echo -n "Fetching ${REGIME_DISPLAY} Assured Workload folders in ${WORKLOAD_REGION}..." WORKLOAD_NAME=$(gcloud assured workloads list --location="${WORKLOAD_REGION}" --organization="${ORG_ID}" --filter="complianceRegime=${COMPLIANCE_REGIME}" --format="value(displayName)" 2>/dev/null | head -n 1 || true) if [[ -z "$WORKLOAD_NAME" ]]; then - echo -e "${YELLOW}Warning: Could not find ${REGIME_DISPLAY} Assured Workload folder in ${WORKLOAD_REGION}.${NC}" + echo -e "\n${RED}WARNING: Could not find ${REGIME_DISPLAY} Assured Workload folder in ${WORKLOAD_REGION}.${NC}" echo -e "${YELLOW}Skipping automated Assured Workloads updates.${NC}" else + echo -e " [${GREEN}OK${NC}]" + echo -e "Found: ${GREEN}${WORKLOAD_NAME}${NC}" echo "" echo -e "${YELLOW}ACTION REQUIRED: Please update your Assured Workload environment manually.${NC}" echo -e "1. Navigate to the following URL in your browser:" @@ -1020,13 +1391,23 @@ configure_stage_0() { echo -e "2. Click on the ${REGIME_DISPLAY} Assured Workload named: ${GREEN}${WORKLOAD_NAME}${NC}" echo -e "3. Click on the button to ${GREEN}\"Review available updates\"${NC} and apply them." echo "" - read -p "Press Enter after you have confirmed the updates have been made..." + read -p "Press Enter to acknowledge and continue..." echo -e "${GREEN}Assured Workload folder ${WORKLOAD_NAME} validated / updated${NC}" fi fi fi fi - + # 2. Access Transparency (Conditional on Compliance Regime) + if [[ "$COMPLIANCE_REGIME" == "FEDRAMP_HIGH" || "$COMPLIANCE_REGIME" == "IL4" || "$COMPLIANCE_REGIME" == "IL5" ]]; then + echo "" + echo -e "${BLUE}--- Access Transparency ---${NC}" + echo -e "${YELLOW}Access Transparency is highly recommended/required for this compliance regime.${NC}" + echo -e "1. Navigate to the following URL in your browser:" + echo -e "${BLUE}https://console.cloud.google.com/iam-admin/settings?organizationId=${ORG_ID}${NC}" + echo -e "2. Under 'Access Transparency', ensure it is enabled." + echo "" + read -p "Press Enter to acknowledge and continue..." + fi # 2. Shared VPC USE_SHARED_VPC="false" @@ -1123,23 +1504,77 @@ configure_stage_0() { # 3. Region if [[ -z "$REGION" ]]; then - REGION=$(gcloud config get-value compute/region 2>/dev/null) - REGION=${REGION:-"us-east4"} - read -p "Enter Region [${REGION}]: " INPUT_REGION - REGION=${INPUT_REGION:-$REGION} + echo "" + echo -e "Select Network Region:" + echo "1) us-central1" + echo "2) us-central2" + echo "3) us-east1" + echo "4) us-east4 (Default)" + echo "5) us-east5" + echo "6) us-south1" + echo "7) us-west1" + echo "8) us-west2" + echo "9) us-west3" + echo "10) us-west4" + read -p "Enter selection [4]: " REGION_SEL + + case $REGION_SEL in + 1) REGION="us-central1" ;; + 2) REGION="us-central2" ;; + 3) REGION="us-east1" ;; + 4|"") REGION="us-east4" ;; + 5) REGION="us-east5" ;; + 6) REGION="us-south1" ;; + 7) REGION="us-west1" ;; + 8) REGION="us-west2" ;; + 9) REGION="us-west3" ;; + 10) REGION="us-west4" ;; + *) + echo -e "${YELLOW}Invalid selection. Defaulting to us-east4.${NC}" + REGION="us-east4" + ;; + esac fi - echo -e "Using Region: ${YELLOW}${REGION}${NC}" + echo -e "Using Network Region: ${YELLOW}${REGION}${NC}" # 4. Load Balancer Type echo "" echo -e "Select Load Balancer Type:" echo "1) Regional External (Internet facing)" echo "2) Regional Internal (VPN / Interconnect)" + echo "3) None (Gemini Enterprise App Only)" read -p "Enter selection [1]: " LB_SEL - if [[ "$LB_SEL" == "2" ]]; then + + CERT_MANAGEMENT_CHOICE="self_managed" + CUSTOM_DOMAIN="" + + if [[ "$LB_SEL" == "3" ]]; then + DEPLOYMENT_TYPE="none" + elif [[ "$LB_SEL" == "2" ]]; then DEPLOYMENT_TYPE="internal" else DEPLOYMENT_TYPE="external" + + if [[ "$COMPLIANCE_REGIME" != "IL4" && "$COMPLIANCE_REGIME" != "IL5" ]]; then + echo "" + echo -e "Select Certificate Management:" + echo "1) Regional Google-managed SSL Certificate" + echo " - Benefits: Automatically provisions and renews SSL certificate, less operational overhead" + echo "2) Regional Self-Managed Certificate (Default)" + echo " - Benefits: Full control over certificate lifecycle, allows use of custom/existing CA" + read -p "Enter selection [2]: " CERT_SEL + + if [[ "$CERT_SEL" == "1" ]]; then + CERT_MANAGEMENT_CHOICE="google_managed" + read -p "Enter Gemini Enterprise FQDN for the Certificate (e.g., gemini.example.com): " CUSTOM_DOMAIN + # Pre-set DOMAIN if empty to match CUSTOM_DOMAIN base + if [[ -z "$DOMAIN" ]]; then + DOMAIN=$(echo "$CUSTOM_DOMAIN" | awk -F. '{print $(NF-1)"."$NF}') + fi + fi + else + echo -e "${GREEN}${COMPLIANCE_REGIME} Regime active. Automatically enforcing Self-Managed Certificate.${NC}" + fi fi # 5. Domain @@ -1159,7 +1594,7 @@ configure_stage_0() { echo -e "${BLUE}--- Identity and Access ---${NC}" echo "Select Gemini Enterprise Identity Provider:" echo "----------------------------------------------------------------" - echo "1) GSUITE (Default)" + echo "1) GOOGLE CLOUD IDENTITY (Default)" echo " - Best for users with Google Workspace accounts." echo " - Uses standard Google Groups (e.g., gcp-gemini-enterprise-admins@${DOMAIN})." echo " - Simple setup, requires Cloud Identity or Google Workspace." @@ -1172,7 +1607,7 @@ configure_stage_0() { echo "----------------------------------------------------------------" read -p "Enter selection [1]: " ACL_SELECTION - ACL_IDP_TYPE="GSUITE" + ACL_IDP_TYPE="GOOGLE_CLOUD_IDENTITY" ACL_POOL_NAME="" ACL_PROVIDER_ID="" @@ -1241,11 +1676,11 @@ configure_stage_0() { echo -e "5. Ensure that the attribute ${YELLOW}google.email${NC} is mapped from your identity provider's email attribute." echo -e " (Example mapping: ${YELLOW}assertion.email${NC} or ${YELLOW}assertion.sub${NC})" echo "" - read -p "Press Enter after you have confirmed the attribute mapping is correct..." + read -p "Press Enter to acknowledge and continue..." fi # 7. Groups - if [[ "$ACL_IDP_TYPE" == "GSUITE" ]]; then + if [[ "$ACL_IDP_TYPE" == "GOOGLE_CLOUD_IDENTITY" ]]; then DEFAULT_ADMIN="gcp-gemini-enterprise-admins@${DOMAIN}" DEFAULT_USER="gcp-gemini-enterprise-users@${DOMAIN}" read -p "Enter Admin Group [${DEFAULT_ADMIN}]: " ADMIN_GROUP @@ -1256,6 +1691,25 @@ configure_stage_0() { # Add group: prefix [[ "$ADMIN_GROUP" != *":"* ]] && ADMIN_GROUP="group:${ADMIN_GROUP}" [[ "$USER_GROUP" != *":"* ]] && USER_GROUP="group:${USER_GROUP}" + + # Validate Groups + echo "Validating group existence and directory access..." + ADMIN_EMAIL="${ADMIN_GROUP#group:}" + USER_EMAIL="${USER_GROUP#group:}" + + if ! gcloud --quiet identity groups describe "$ADMIN_EMAIL" &>/dev/null; then + echo -e "${RED}WARNING: Cannot access or find Admin Group: ${ADMIN_EMAIL}${NC}" + echo -e "${YELLOW}Ensure the group exists and your account has directory read access.${NC}" + else + echo -e "${GREEN}Validated Admin Group access.${NC}" + fi + + if ! gcloud --quiet identity groups describe "$USER_EMAIL" &>/dev/null; then + echo -e "${RED}WARNING: Cannot access or find User Group: ${USER_EMAIL}${NC}" + echo -e "${YELLOW}Ensure the group exists and your account has directory read access.${NC}" + else + echo -e "${GREEN}Validated User Group access.${NC}" + fi else echo "" echo -e "${YELLOW}For Workforce Identity, please enter the full Principal / Principal Set.${NC}" @@ -1278,14 +1732,39 @@ configure_stage_0() { read -p "Enter Admin Principal/Principal Set: " ADMIN_GROUP read -p "Enter User Principal/Principal Set: " USER_GROUP fi + # 8. Implicit Model Data Caching + echo "" + echo -e "${BLUE}--- Vertex AI Configuration ---${NC}" + echo "Disabling Implicit Model Data Caching for project: ${PROJECT_ID}..." + local ACCESS_TOKEN + if ACCESS_TOKEN=$(gcloud auth print-access-token 2>/dev/null); then + local CACHE_CONFIG_URL="https://us-central1-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/cacheConfig" + local CACHE_PAYLOAD=$(jq -n --arg pid "$PROJECT_ID" '{name: "projects/\($pid)/cacheConfig", disableCache: true}') + + local CACHE_RESPONSE + CACHE_RESPONSE=$(curl -s -w "\n%{http_code}" -X PATCH \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "${CACHE_PAYLOAD}" \ + "${CACHE_CONFIG_URL}") + + local CACHE_HTTP_CODE=$(echo "$CACHE_RESPONSE" | tail -n 1) + if [[ "$CACHE_HTTP_CODE" == "200" || "$CACHE_HTTP_CODE" == "204" ]]; then + echo -e "${GREEN}Successfully disabled Implicit Model Data Caching.${NC}" + else + echo -e "${YELLOW}Failed to disable Implicit Model Data Caching (HTTP ${CACHE_HTTP_CODE}). Please verify Vertex AI permissions.${NC}" + fi + else + echo -e "${RED}WARNING: Could not get gcloud access token. Skipping caching check.${NC}" + fi - # 8. Access Policy + # 9. Access Policy echo "" echo -e "${BLUE}--- Access Policies ---${NC}" echo "Discovering Access Policy..." ACCESS_POLICY_NUMBER=$(gcloud access-context-manager policies list --organization "${ORG_ID}" --format="value(name)" --quiet 2>/dev/null | head -n 1) if [ -z "$ACCESS_POLICY_NUMBER" ]; then - echo -e "${YELLOW}Warning: Could not auto-discover Access Policy Number.${NC}" + echo -e "${RED}WARNING: Could not auto-discover Access Policy Number.${NC}" read -p "Enter Access Policy Number: " ACCESS_POLICY_NUMBER else ACCESS_POLICY_NUMBER=$(basename "${ACCESS_POLICY_NUMBER}") @@ -1326,7 +1805,7 @@ configure_stage_0() { echo -e "${GREEN}Found managed Access Levels in state.${NC}" fi else - echo -e "${YELLOW}Warning: Could not initialize Terraform state check. Proceeding as fresh deployment.${NC}" + echo -e "${RED}WARNING: Could not initialize Terraform state check. Proceeding as fresh deployment.${NC}" fi else echo "State bucket not determined. Skipping managed resource check." @@ -1352,26 +1831,204 @@ configure_stage_0() { echo -e "${YELLOW}--- NOTE: Data Stores can be created and associated with a Gemini Enterprise application at a later time. ---${NC}" read -p "Create Data Stores? (y/N): " DS_CHOICE CREATE_DS_BOOL="false" - GCS_DATA_STORES="[]" - BQ_DATA_STORES="[]" + ENABLE_DS_CMEK="true" # Default to true even if not creating, though irrelevant + GCS_DATA_STORES="{}" + BQ_DATA_STORES="{}" if [[ "$DS_CHOICE" == "y" || "$DS_CHOICE" == "Y" ]]; then CREATE_DS_BOOL="true" + # Ask for CMEK preference for Data Stores + ENABLE_DS_CMEK="true" + if [[ "$COMPLIANCE_REGIME" == "IL4" || "$COMPLIANCE_REGIME" == "IL5" ]]; then + echo -e "${GREEN}${COMPLIANCE_REGIME} Regime active. Automatically enforcing CMEK for Data Stores.${NC}" + else + read -p "Encrypt these Data Stores with Customer Managed Encryption Keys (CMEK)? (Y/n): " CMEK_CHOICE + if [[ "$CMEK_CHOICE" == "n" || "$CMEK_CHOICE" == "N" ]]; then + ENABLE_DS_CMEK="false" + echo -e "${YELLOW}Data Stores will use Google-managed encryption keys.${NC}" + else + echo -e "${GREEN}Data Stores will use CMEK.${NC}" + fi + fi + + if [[ "$ENABLE_DS_CMEK" == "true" ]]; then + echo -e "${YELLOW}CMEK for Data Stores requested. Ensuring key exists...${NC}" + + # We need standard variables. discover_infrastructure should have set them. + if [[ -z "$CAP_ENV" && -n "$ENVIRONMENT" ]]; then + CAP_ENV=$(echo "$ENVIRONMENT" | awk '{print toupper(substr($0,1,1)) substr($0,2)}') + fi + CAP_ENV=${CAP_ENV:-"Prod"} + TENANT=${TENANT:-"g4g"} + + _KEYRING_NAME="${CAP_ENV}-${TENANT}-keyring" + _KEY_NAME="gemini-enterprise" + _LOCATION="us" + + # Identify Target Project + _TARGET_KMS_PROJECT="${CMEK_PROJECT_ID}" + if [[ -z "$_TARGET_KMS_PROJECT" ]]; then + _TARGET_KMS_PROJECT="${TENANT_IAC_PROJECT}" + fi + if [[ -z "$_TARGET_KMS_PROJECT" ]]; then + _TARGET_KMS_PROJECT="${PROJECT_ID}" + fi + + _CMEK_US_KEYRING="projects/${_TARGET_KMS_PROJECT}/locations/${_LOCATION}/keyRings/${_KEYRING_NAME}" + _FULL_KEY_NAME="${_CMEK_US_KEYRING}/cryptoKeys/${_KEY_NAME}" + + if [[ -z "$CMEK_US_RESOURCES_KEY" ]]; then + echo -e "Target Project: ${YELLOW}${_TARGET_KMS_PROJECT}${NC}" + echo -e "Keyring: ${YELLOW}${_KEYRING_NAME}${NC}" + + if ! gcloud kms keys describe "${_FULL_KEY_NAME}" &>/dev/null; then + echo "Creating Key '${_KEY_NAME}'..." + gcloud kms keys create "${_KEY_NAME}" \ + --keyring="${_KEYRING_NAME}" \ + --location="${_LOCATION}" \ + --project="${_TARGET_KMS_PROJECT}" \ + --purpose="encryption" \ + --protection-level="hsm" \ + --rotation-period="7776000s" \ + --next-rotation-time="$(date -v+90d -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -d '+90 days' +%Y-%m-%dT%H:%M:%SZ)" + else + echo "Key '${_KEY_NAME}' already exists." + fi + CMEK_US_RESOURCES_KEY="${_FULL_KEY_NAME}" + else + echo -e "Using Existing CMEK Gemini Key: ${GREEN}${CMEK_US_RESOURCES_KEY}${NC}" + fi + + # Register the key BEFORE Terraform + echo -e "${YELLOW}Registering CMEK key for Gemini Enterprise in the US multi-region...${NC}" + + # --- Prerequisite: Grant IAM permissions to Discovery Engine service account --- + echo "Checking if Discovery Engine service account has access to the key..." + _PROJECT_NUMBER=$(gcloud projects describe "${PROJECT_ID}" --format="value(projectNumber)" 2>/dev/null) + if [[ -n "$_PROJECT_NUMBER" ]]; then + _SERVICES_SA="service-${_PROJECT_NUMBER}@gcp-sa-discoveryengine.iam.gserviceaccount.com" + echo "Granting roles/cloudkms.cryptoKeyEncrypterDecrypter to ${_SERVICES_SA} on key ${_KEY_NAME}..." + if ! gcloud kms keys add-iam-policy-binding "${_KEY_NAME}" \ + --location="${_LOCATION}" \ + --keyring="${_KEYRING_NAME}" \ + --project="${_TARGET_KMS_PROJECT}" \ + --member="serviceAccount:${_SERVICES_SA}" \ + --role="roles/cloudkms.cryptoKeyEncrypterDecrypter" 2>/dev/null; then + echo -e "${RED}WARNING: Failed to grant IAM binding to Discovery Engine service account.${NC}" + echo -e "${YELLOW}You might need 'roles/cloudkms.admin' on the key project.${NC}" + fi + else + echo -e "${RED}WARNING: Could not determine project number. Skipping IAM grant for Discovery Engine.${NC}" + fi + + _ACCESS_TOKEN=$(gcloud auth print-access-token) + + # --- Check if already registered --- + echo "Checking if CMEK key is already registered..." + _CONFIG_RESPONSE=$(curl -s -w "\n%{http_code}" -H "Authorization: Bearer ${_ACCESS_TOKEN}" \ + -H "x-goog-user-project: ${PROJECT_ID}" \ + "https://us-discoveryengine.googleapis.com/v1/projects/${PROJECT_ID}/locations/us/cmekConfigs/default_cmek_config") + + _OP_HTTP_CODE=$(echo "$_CONFIG_RESPONSE" | tail -n1) + _OP_BODY=$(echo "$_CONFIG_RESPONSE" | sed '$d') + + _CURRENT_KEY=$(echo "$_OP_BODY" | jq -r .kmsKey 2>/dev/null || echo "") + + _PROCEED_WITH_PATCH=true + + if [[ "$_OP_HTTP_CODE" -eq 200 ]]; then + if [[ "$_CURRENT_KEY" == "${CMEK_US_RESOURCES_KEY}" ]]; then + echo -e "${GREEN}CMEK key is already registered and matches.${NC}" + _PROCEED_WITH_PATCH=false + else + echo -e "${YELLOW}CMEK key is already registered with a different key: ${_CURRENT_KEY}${NC}" + echo -e "${YELLOW}Adopting the already registered key for infrastructure alignment.${NC}" + CMEK_US_RESOURCES_KEY="$_CURRENT_KEY" + _PROCEED_WITH_PATCH=false + fi + else + echo "CMEK config not found or error. Proceeding with registration..." + fi + + if [[ "$_PROCEED_WITH_PATCH" == "true" ]]; then + echo "Sending registration request..." + _API_RESPONSE=$(curl -s -w "\n%{http_code}" -X PATCH \ + -H "Authorization: Bearer ${_ACCESS_TOKEN}" \ + -H "Content-Type: application/json" \ + -H "x-goog-user-project: ${PROJECT_ID}" \ + "https://us-discoveryengine.googleapis.com/v1/projects/${PROJECT_ID}/locations/us/cmekConfigs/default_cmek_config?set_default=true" \ + -d "{\"kmsKey\": \"${CMEK_US_RESOURCES_KEY}\"}") + + _PATCH_HTTP_CODE=$(echo "$_API_RESPONSE" | tail -n1) + _PATCH_BODY=$(echo "$_API_RESPONSE" | sed '$d') + + if [[ "$_PATCH_HTTP_CODE" -eq 200 || "$_PATCH_HTTP_CODE" -eq 409 ]]; then + echo -e "${GREEN}Successfully initiated CMEK key registration.${NC}" + + # --- Polling for Long Running Operation (LRO) --- + _OPERATION_ID=$(echo "$_PATCH_BODY" | jq -r .name 2>/dev/null || echo "") + if [[ -n "$_OPERATION_ID" && "$_OPERATION_ID" != "null" ]]; then + echo -e "${YELLOW}Long Running Operation ID: ${_OPERATION_ID}${NC}" + echo -e "${YELLOW}Polling for completion (this may take a few minutes)...${NC}" + + while true; do + _OP_RESPONSE=$(curl -s -H "Authorization: Bearer ${_ACCESS_TOKEN}" \ + -H "x-goog-user-project: ${PROJECT_ID}" \ + "https://us-discoveryengine.googleapis.com/v1/${_OPERATION_ID}") + + _IS_DONE=$(echo "$_OP_RESPONSE" | jq -r .done 2>/dev/null || echo "false") + _HAS_ERROR=$(echo "$_OP_RESPONSE" | jq -r .error 2>/dev/null || echo "") + + if [[ "$_IS_DONE" == "true" ]]; then + if [[ -n "$_HAS_ERROR" && "$_HAS_ERROR" != "null" ]]; then + echo -e "${RED}CMEK registration failed in operation.${NC}" + echo -e "Error: $_HAS_ERROR" + break + fi + echo -e "\n${GREEN}CMEK registration completed successfully.${NC}" + break + fi + + echo -n "." + sleep 10 + done + echo "" + else + echo -e "${RED}Warning: Could not extract operation name from response.${NC}" + echo -e "Response: $_PATCH_BODY" + fi + else + echo -e "${RED}Failed to register CMEK key. HTTP Status: ${_PATCH_HTTP_CODE}${NC}" + echo -e "Response: $_PATCH_BODY" + echo -e "You may need to manually register the key." + fi + fi + fi GCS_LIST=() BQ_LIST=() configure_data_stores if [[ ${#GCS_LIST[@]} -gt 0 ]]; then - GCS_DATA_STORES="[$(IFS=,; echo "${GCS_LIST[*]}")]" + GCS_DATA_STORES="{ $(IFS=,; echo "${GCS_LIST[*]}") }" fi if [[ ${#BQ_LIST[@]} -gt 0 ]]; then - BQ_DATA_STORES="[$(IFS=,; echo "${BQ_LIST[*]}")]" + BQ_DATA_STORES="{ $(IFS=,; echo "${BQ_LIST[*]}") }" fi fi - # 10. Organization Policy Check + # 10. Analytics (Discovery Engine Audit Logs) + echo "" + echo -e "${BLUE}--- Analytics (Discovery Engine Audit Logs) ---${NC}" + read -p "Would you like to enable analytics for Gemini Enterprise (via Discovery Engine Audit Logs)? [y/N]: " ENABLE_ANALYTICS + if [[ "$ENABLE_ANALYTICS" =~ ^[Yy]$ ]]; then + ENABLE_ANALYTICS_FLAG="true" + else + ENABLE_ANALYTICS_FLAG="false" + fi + + # 11. Organization Policy Check echo "" echo -e "${BLUE}--- Organization Policies (Project-Level) ---${NC}" check_org_policies @@ -1429,14 +2086,6 @@ configure_stage_0() { echo -e "Using KMS Key: ${YELLOW}${KMS_KEY_ID}${NC}" fi - # Determine create_resource_keys - # If Brownfield or Custom, and we have a KMS Key ID, we assume we are using existing keys - # and do not need to create new ones. - CREATE_RESOURCE_KEYS_BOOL="true" - if [[ ("$IS_BROWNFIELD" == "true" || "$IS_CUSTOM" == "true") && -n "$KMS_KEY_ID" ]]; then - CREATE_RESOURCE_KEYS_BOOL="false" - fi - # Initialize Terraform early to check state echo "" echo -e "${BLUE}--- Existing Terraform State Check ---${NC}" @@ -1445,17 +2094,19 @@ configure_stage_0() { if [[ -z "$BUCKET_NAME" && -n "$STATE_BUCKET" ]]; then BUCKET_NAME=$(echo "$STATE_BUCKET" | sed 's/gs:\/\/ //' | sed 's/\/$//') fi - terraform init -migrate-state -backend-config="bucket=${BUCKET_NAME}" -backend-config="prefix=terraform/state/stage-0" || echo "Warning: Init failed during state check." + rm -rf .terraform + terraform init -migrate-state -backend-config="bucket=${BUCKET_NAME}" -backend-config="prefix=terraform/state/stage-0" || echo -e "${RED}WARNING: Init failed during state check.${NC}" # Check if KeyRing is in state if terraform state list | grep -q "google_kms_key_ring.created"; then - echo -e "${YELLOW}CMEK Keyring found in Terraform State. Will use managed resource instead of data source.${NC}" - CMEK_US_KEYRING="" + echo -e "${YELLOW}CMEK Keyring found in Terraform State. Removing it from Terraform management...${NC}" + terraform state rm google_kms_key_ring.created || true fi - # Check if Key is in state (only if not explicitly provided by user) - if [[ -z "$CMEK_US_RESOURCES_KEY" ]] && terraform state list | grep -q "google_kms_crypto_key.gemini_enterprise"; then - echo -e "${YELLOW}gemini-enterprise Crypto Key found in Terraform State. Will use managed resource instead of data source.${NC}" + # Check if Key is in state + if terraform state list | grep -q "google_kms_crypto_key.gemini_enterprise"; then + echo -e "${YELLOW}gemini-enterprise Crypto Key found in Terraform State. Removing it from Terraform management...${NC}" + terraform state rm google_kms_crypto_key.gemini_enterprise || true fi cd .. @@ -1464,17 +2115,20 @@ configure_stage_0() { main_project_id = "${PROJECT_ID}" environment = "${ENVIRONMENT}" tenant = "${TENANT}" +compliance_regime = "${COMPLIANCE_REGIME:-NONE}" kms_project_id = "${CMEK_PROJECT_ID}" us_keyring_name = "${CMEK_US_KEYRING}" kms_key_id = "${CMEK_US_RESOURCES_KEY}" -terraform_state_bucket = "${STATE_BUCKET}" +terraform_state_bucket = "${BUCKET_NAME}" region = "${REGION}" domain = "${DOMAIN}" prefix = "${PREFIX}" deployment_type = "${DEPLOYMENT_TYPE}" +cert_management_choice = "${CERT_MANAGEMENT_CHOICE:-self_managed}" +custom_domain = "${CUSTOM_DOMAIN:-}" access_policy_number = ${ACCESS_POLICY_NUMBER} admin_group = "${ADMIN_GROUP}" -user_group = "${USER_GROUP}" +user_groups = ["${USER_GROUP}"] acl_idp_type = "${ACL_IDP_TYPE}" acl_workforce_pool_name = "${ACL_POOL_NAME}" acl_workforce_provider_id = "${ACL_PROVIDER_ID}" @@ -1484,13 +2138,15 @@ shared_vpc_network_name = "${SHARED_VPC_NETWORK}" shared_vpc_subnet_name = "${SHARED_VPC_SUBNET}" shared_vpc_proxy_subnet_name = "${SHARED_VPC_PROXY_SUBNET}" create_data_stores = ${CREATE_DS_BOOL} +enable_analytics = ${ENABLE_ANALYTICS_FLAG} EOF # Add example data stores if [[ "$CREATE_DS_BOOL" == "true" ]]; then cat >> gemini-stage-0/terraform.tfvars </dev/null || echo "N/A") + CMEK_KEY_ID=$(terraform output -raw cmek_key_id 2>/dev/null || echo "") cd .. echo -e "${GREEN}Stage 0 Deployment Complete!${NC}" + # Optionally register the CMEK Key for US Multi-Region in Discovery Engine + if [[ "$ENABLE_DS_CMEK" == "true" || "$COMPLIANCE_REGIME" == "IL4" || "$COMPLIANCE_REGIME" == "IL5" ]]; then + # Check if key is managed by TF (Safety Check) + if terraform -chdir=gemini-stage-0 state list | grep -q "google_kms_crypto_key.gemini_enterprise" 2>/dev/null; then + echo -e "${YELLOW}CMEK Key is managed by Terraform. Registering it after creation...${NC}" + + CMEK_KEY_ID=$(terraform -chdir=gemini-stage-0 output -raw cmek_key_id 2>/dev/null || echo "") + + if [[ -n "$CMEK_KEY_ID" && "$CMEK_KEY_ID" != "null" ]]; then + ACCESS_TOKEN=$(gcloud auth print-access-token) + + API_RESPONSE=$(curl -s -w "\n%{http_code}" -X PATCH \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "Content-Type: application/json" \ + -H "x-goog-user-project: ${PROJECT_ID}" \ + "https://us-discoveryengine.googleapis.com/v1/projects/${PROJECT_ID}/locations/us/cmekConfigs/default_cmek_config?set_default=true" \ + -d "{\"kmsKey\": \"${CMEK_KEY_ID}\"}") + + HTTP_CODE=$(echo "$API_RESPONSE" | tail -n1) + BODY=$(echo "$API_RESPONSE" | sed '$d') + + if [[ "$HTTP_CODE" -eq 200 || "$HTTP_CODE" -eq 409 ]]; then + echo -e "${GREEN}Successfully registered CMEK key for Gemini Enterprise.${NC}" + else + echo -e "${RED}Failed to register CMEK key. HTTP Status: ${HTTP_CODE}${NC}" + echo -e "Response: $BODY" + echo -e "You may need to manually register the key." + fi + else + echo -e "${RED}Warning: CMEK key was enabled but no key ID was found in terraform output. Skipping registration.${NC}" + fi + fi + fi + if [[ "$CREATE_DS_BOOL" == "true" ]]; then echo "" echo -e "${YELLOW}ACTION REQUIRED: Populate the created Data Stores with data.${NC}" echo "" - echo -e "${BLUE}GCS${NC}: Upload your documents to the GCS bucket(s) created by Terraform (see output above \`gcs_data_store_to_bucket\`)." - echo -e "${BLUE}BigQuery${NC}: Populate the BigQuery table(s) created by Terraform (see output above \`bq_data_store_to_dataset_table\`)" + echo -e "${BLUE}GCS${NC}: Upload your documents to the GCS bucket(s) created by Terraform (see output above \`gcs_data_stores\`)." + echo -e "${BLUE}BigQuery${NC}: Populate the BigQuery dataset(s) created by Terraform (see output above \`bq_data_stores\`)" echo "" - echo -e "After uploading documents into the bucket / table, navigate to ${YELLOW}Helper Functions${NC} > ${YELLOW}Populate Data Stores${NC}" + echo -e "After uploading documents into the bucket / dataset, navigate to:" + echo -e "${YELLOW}Helper Functions${NC} > ${YELLOW}3. Import Documents to Gemini Enterprise Data Store (Cloud Storage / BigQuery)${NC}" echo -e "to import the data into the Gemini Enterprise Data Stores and begin the indexing process." - read -p "Press Enter to continue..." + echo "" + read -p "Press Enter to acknowledge and continue..." fi + CERT_CHOICE=$(grep "cert_management_choice" gemini-stage-0/terraform.tfvars | awk -F'=' '{print $2}' | tr -d ' "') + DEPLOY_TYPE=$(grep "deployment_type" gemini-stage-0/terraform.tfvars | awk -F'=' '{print $2}' | tr -d ' "') + echo "" echo -e "${YELLOW}IMPORTANT NEXT STEPS:${NC}" echo -e "1. From the Main Menu select ${BLUE}Step 2 - Create Gemini Enterprise App (gem4gov-cli)${NC}." - echo -e "2. Setup DNS A Record that points the desired Gemini Enterprise subdomain (i.e. gemini.yourdomain.com) to the provisioned Load Balancer IP address (${GEMINI_IP})." - echo -e "3. Provision an SSL Certificate and upload it to Google Cloud Certificate Manager (${YELLOW}Helper Functions > Upload SSL Certificate${NC})." - echo -e "4. From the Main Menu select ${BLUE}Step 3 - Configure & Deploy Load Balancer / Access Policies (gemini-stage-1)${NC}." + + if [[ "$DEPLOY_TYPE" != "none" ]]; then + echo -e "2. Setup DNS A Record that points the desired Gemini Enterprise subdomain (i.e. gemini.yourdomain.com) to the provisioned Load Balancer IP address (${GEMINI_IP})." + if [[ "$CERT_CHOICE" == "google_managed" ]]; then + DNS_RECORDS=$(terraform -chdir=gemini-stage-0 output -json dns_auth_records 2>/dev/null) + DNS_NAME=$(echo "$DNS_RECORDS" | jq -r '.[0].name // empty') + DNS_TYPE=$(echo "$DNS_RECORDS" | jq -r '.[0].type // empty') + DNS_DATA=$(echo "$DNS_RECORDS" | jq -r '.[0].data // empty') + echo -e "3. ${YELLOW}ACTION REQUIRED: Add the following CNAME record to your DNS configuration for the Google-managed certificate authorization!${NC}" + echo -e " - ${BLUE}Name:${NC} ${DNS_NAME}" + echo -e " - ${BLUE}Type:${NC} ${DNS_TYPE}" + echo -e " - ${BLUE}Data:${NC} ${DNS_DATA}" + echo -e " The certificate will not provision until this CNAME is resolvable." + else + echo -e "3. Provision an SSL Certificate and upload it to Google Cloud Region (${YELLOW}Helper Functions > Upload SSL Certificate${NC})." + echo -e " - Requirements: The certificate must be valid for the domain you intend to use and include the full certificate chain." + fi + echo -e "4. From the Main Menu select ${BLUE}Step 3 - Configure & Deploy Load Balancer / Access Policies (gemini-stage-1)${NC}." + fi pause } @@ -1650,21 +2363,132 @@ configure_gem4gov() { PROJECT_ID_STATE=$(echo "$STATE_CONTENT" | jq -r '.outputs.main_project_id.value // empty') PROJECT_ID=${PROJECT_ID_STATE:-$PROJECT_ID} + # Parse Compliance Regime + COMPLIANCE_REGIME_STATE=$(echo "$STATE_CONTENT" | jq -r '.outputs.compliance_regime.value // empty') + COMPLIANCE_REGIME=${COMPLIANCE_REGIME_STATE:-$COMPLIANCE_REGIME} + # Parse Load Balancer IP for display GEMINI_IP=$(echo "$STATE_CONTENT" | jq -r '.outputs.gemini_enterprise_ip.value // "N/A"') - # Construct command - CMD="gem4gov app create --project-id ${PROJECT_ID} --compliance-regime FEDRAMP_HIGH" + # Parse Data Stores + GCS_JSON_RAW=$(echo "$STATE_CONTENT" | jq -c '.outputs.gcs_data_stores.value // {} | to_entries | map(select(.value.data_store_id != null)) | map(.value)' 2>/dev/null) + BQ_JSON_RAW=$(echo "$STATE_CONTENT" | jq -c '.outputs.bq_data_stores.value // {} | to_entries | map(select(.value.data_store_id != null)) | map(.value)' 2>/dev/null) - # 1. Extract Data Store IDs - # Retrieve both GCS and BQ Data Store IDs from outputs and concatenate them - ALL_IDS_LIST=$(echo "$STATE_CONTENT" | jq -r '.outputs.gcs_data_store_ids.value[], .outputs.bq_data_store_ids.value[]' 2>/dev/null) + if [[ "$GCS_JSON_RAW" == "[]" || -z "$GCS_JSON_RAW" ]]; then GCS_JSON_RAW=""; fi + if [[ "$BQ_JSON_RAW" == "[]" || -z "$BQ_JSON_RAW" ]]; then BQ_JSON_RAW=""; fi + + DS_ID_ARRAY=() + DS_DISPLAY_ARRAY=() - # Join with commas for the CLI argument - ALL_IDS=$(echo "$ALL_IDS_LIST" | tr '\n' ',' | sed 's/,$//') + if [[ -n "$GCS_JSON_RAW" ]]; then + while IFS= read -r id; do [[ -n "$id" ]] && DS_ID_ARRAY+=("$id"); done < <(echo "$GCS_JSON_RAW" | jq -r '.[].data_store_id') + while IFS= read -r disp; do [[ -n "$disp" ]] && DS_DISPLAY_ARRAY+=("$disp"); done < <(echo "$GCS_JSON_RAW" | jq -r '.[].display_name') + fi + if [[ -n "$BQ_JSON_RAW" ]]; then + while IFS= read -r id; do [[ -n "$id" ]] && DS_ID_ARRAY+=("$id"); done < <(echo "$BQ_JSON_RAW" | jq -r '.[].data_store_id') + while IFS= read -r disp; do [[ -n "$disp" ]] && DS_DISPLAY_ARRAY+=("$disp"); done < <(echo "$BQ_JSON_RAW" | jq -r '.[].display_name') + fi + + echo "" + echo -e "${BLUE}--- Application Details ---${NC}" + echo -e "${YELLOW}Please provide details for the Gemini Enterprise Application.${NC}" + APP_LIST=() + + while true; do + APP_DISPLAY="" + while [[ -z "$APP_DISPLAY" ]]; do + read -p "Please enter a Display Name for the Application: " APP_DISPLAY + done + + APP_COMPANY="" + while [[ -z "$APP_COMPANY" ]]; do + read -p "Please enter the Agency / Department Name (no abbreviations): " APP_COMPANY + done + + echo "" + echo -e "${YELLOW}WARNING: Enabling Gemini Enterprise Usage Audit logs will write user queries, model thinking, and model responses to Cloud Logging.${NC}" + echo -e "${YELLOW}You must ensure that logging permissions are set to allow only necessary principals to access.${NC}" + read -p "Would you like to enable Gemini Enterprise Usage Audit logs (conversation logging) for this application? [y/N]: " ENABLE_AUDIT_LOGS + if [[ "$ENABLE_AUDIT_LOGS" =~ ^[Yy]$ ]]; then + ENABLE_AUDIT_LOGS_FLAG="true" + else + ENABLE_AUDIT_LOGS_FLAG="false" + fi + + echo "" + echo -e "${YELLOW}Agent Sharing Feature:${NC}" + echo -e "${YELLOW}When enabled, users can share agents with other users using the Gemini Enterprise app.${NC}" + read -p "Would you like to enable the 'Agent Sharing' feature? [y/N]: " ENABLE_AGENT_SHARING + if [[ "$ENABLE_AGENT_SHARING" =~ ^[Yy]$ ]]; then + ENABLE_AGENT_SHARING_FLAG="true" + sed -i '' 's/disable-agent-sharing:.*/disable-agent-sharing: "FEATURE_STATE_OFF"/' gem4gov-cli/engine_features.yaml 2>/dev/null || sed -i 's/disable-agent-sharing:.*/disable-agent-sharing: "FEATURE_STATE_OFF"/' gem4gov-cli/engine_features.yaml + else + ENABLE_AGENT_SHARING_FLAG="false" + sed -i '' 's/disable-agent-sharing:.*/disable-agent-sharing: "FEATURE_STATE_ON"/' gem4gov-cli/engine_features.yaml 2>/dev/null || sed -i 's/disable-agent-sharing:.*/disable-agent-sharing: "FEATURE_STATE_ON"/' gem4gov-cli/engine_features.yaml + fi + + echo "" + echo -e "${YELLOW}Agent Sharing without Admin Approval Feature:${NC}" + echo -e "${YELLOW}When enabled, users on your team can share and use agents without admin approval when using the Gemini Enterprise app.${NC}" + read -p "Would you like to enable 'Agent Sharing without Admin Approval'? [y/N]: " ENABLE_AGENT_SHARING_NO_APPROVAL + if [[ "$ENABLE_AGENT_SHARING_NO_APPROVAL" =~ ^[Yy]$ ]]; then + ENABLE_AGENT_SHARING_NO_APPROVAL_FLAG="true" + sed -i '' 's/agent-sharing-without-admin-approval:.*/agent-sharing-without-admin-approval: "FEATURE_STATE_ON"/' gem4gov-cli/engine_features.yaml 2>/dev/null || sed -i 's/agent-sharing-without-admin-approval:.*/agent-sharing-without-admin-approval: "FEATURE_STATE_ON"/' gem4gov-cli/engine_features.yaml + else + ENABLE_AGENT_SHARING_NO_APPROVAL_FLAG="false" + sed -i '' 's/agent-sharing-without-admin-approval:.*/agent-sharing-without-admin-approval: "FEATURE_STATE_OFF"/' gem4gov-cli/engine_features.yaml 2>/dev/null || sed -i 's/agent-sharing-without-admin-approval:.*/agent-sharing-without-admin-approval: "FEATURE_STATE_OFF"/' gem4gov-cli/engine_features.yaml + fi + + echo "" + # Determine App Key + APP_SUFFIX=$(python3 -c "import random, string; print(''.join(random.choices(string.ascii_lowercase + string.digits, k=4)))") + ENG_ID="g4g-gem-ent-app-${APP_SUFFIX}" + + SELECTED_IDS="" + if [[ ${#DS_ID_ARRAY[@]} -gt 0 ]]; then + echo -e "${YELLOW}Available Data Stores for association:${NC}" + i=1 + for idx in "${!DS_ID_ARRAY[@]}"; do + echo "$i. ${DS_DISPLAY_ARRAY[$idx]} (${DS_ID_ARRAY[$idx]})" + ((i++)) + done + read -p "Select Data Stores to associate (comma-separated numbers, e.g. 1,3) [Enter to skip]: " APP_DS_SEL + + if [[ -n "$APP_DS_SEL" ]]; then + IFS=',' read -ra SELECTED_INDICES <<< "$APP_DS_SEL" + SELECTED_DS_LIST=() + for index in "${SELECTED_INDICES[@]}"; do + index=$(echo "$index" | xargs) + if [[ "$index" =~ ^[0-9]+$ ]] && (( index >= 1 && index <= ${#DS_ID_ARRAY[@]} )); then + SELECTED_DS_LIST+=("${DS_ID_ARRAY[$((index-1))]}") + fi + done + if [[ ${#SELECTED_DS_LIST[@]} -gt 0 ]]; then + SELECTED_IDS=$(IFS=,; echo "${SELECTED_DS_LIST[*]}") + fi + fi + fi + + APP_JSON=$(jq -n \ + --arg id "$ENG_ID" \ + --arg display "$APP_DISPLAY" \ + --arg company "$APP_COMPANY" \ + --arg ds "$SELECTED_IDS" \ + --arg audit_logs "$ENABLE_AUDIT_LOGS_FLAG" \ + '{engine_id: $id, display_name: $display, company_name: $company, data_stores: $ds, enable_audit_logs: $audit_logs}') + APP_LIST+=("$APP_JSON") + + echo "" + read -p "[PREVIEW] Do you want to create another Gemini Enterprise Application? [y/N]: " CREATE_APP + if [[ ! "$CREATE_APP" =~ ^[Yy]$ ]]; then + break + fi + done - if [[ -n "$ALL_IDS" ]]; then - CMD="$CMD --data-stores $ALL_IDS" + if [[ ${#APP_LIST[@]} -eq 0 ]]; then + echo "No applications generated." + pause + return 0 fi # 2. Extract Workforce Identity Details @@ -1681,20 +2505,57 @@ configure_gem4gov() { fi fi + WIF_ARGS="" if [[ -n "$POOL_NAME" && -n "$PROVIDER_ID" ]]; then # Extract Pool ID from full name (locations/global/workforcePools/POOL_ID) POOL_ID=$(basename "$POOL_NAME") - CMD="$CMD --workforce-pool-id $POOL_ID --workforce-provider-id $PROVIDER_ID" + WIF_ARGS="--workforce-pool-id $POOL_ID --workforce-provider-id $PROVIDER_ID" fi - - echo "Running: $CMD" - echo "" export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}" export GOOGLE_CLOUD_QUOTA_PROJECT="${PROJECT_ID}" - $CMD + + echo "" + echo "Executing Application Configurations..." - echo -e "${GREEN}Gemini Enterprise Application configured.${NC}" + # Iterate apps + for APP_JSON in "${APP_LIST[@]}"; do + + ENG_ID=$(echo "$APP_JSON" | jq -r '.engine_id') + DISP_NAME=$(echo "$APP_JSON" | jq -r '.display_name') + COMP_NAME=$(echo "$APP_JSON" | jq -r '.company_name') + DS_KEYS=$(echo "$APP_JSON" | jq -r '.data_stores // empty') + + CMD="gem4gov app create --project-id \"${PROJECT_ID}\" --engine-id \"${ENG_ID}\" --display-name \"${DISP_NAME}\" --company-name \"${COMP_NAME}\"" + + ENABLE_AUDIT_LOGS=$(echo "$APP_JSON" | jq -r '.enable_audit_logs // "false"') + if [[ "$ENABLE_AUDIT_LOGS" == "true" ]]; then + CMD="$CMD --enable-audit-logs" + fi + + if [[ -n "$COMPLIANCE_REGIME" && "$COMPLIANCE_REGIME" != "NONE" ]]; then + CMD="$CMD --compliance-regime \"${COMPLIANCE_REGIME}\"" + fi + + if [[ -n "$DS_KEYS" && "$DS_KEYS" != "null" && "$DS_KEYS" != "\"\"" ]]; then + CMD="$CMD --data-stores \"${DS_KEYS}\"" + fi + + if [[ -n "$WIF_ARGS" ]]; then + CMD="$CMD $WIF_ARGS" + fi + + echo -e "${BLUE}Creating Application: ${DISP_NAME} (${ENG_ID})...${NC}" + echo "Running: $CMD" + if ! eval "$CMD"; then + echo -e "${RED}Error: Failed to create Application ${DISP_NAME}. Aborting.${NC}" + pause + return 1 + fi + echo "" + done + + echo -e "${GREEN}Gemini Enterprise Applications configured.${NC}" echo "" echo -e "${YELLOW}IMPORTANT NEXT STEPS:${NC}" @@ -1744,12 +2605,15 @@ update_app_compliance() { echo "Running: $CMD" export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}" export GOOGLE_CLOUD_QUOTA_PROJECT="${PROJECT_ID}" - $CMD + if ! $CMD; then + echo -e "${RED}ERROR: Failed to update compliance regime.${NC}" + return 1 + fi pause } -# --- Helper Functions --- +# --- Helper Functions Menu --- upload_ssl_certificate() { echo -e "${BLUE}--- Upload SSL Certificate ---${NC}" @@ -1777,7 +2641,7 @@ upload_ssl_certificate() { # Default region DEFAULT_REGION=${REGION:-"us-east4"} - read -p "Enter Region [${DEFAULT_REGION}]: " INPUT_REGION + read -p "Enter Network Region [${DEFAULT_REGION}]: " INPUT_REGION CERT_REGION=${INPUT_REGION:-$DEFAULT_REGION} while true; do @@ -1833,7 +2697,7 @@ upload_ssl_certificate() { replace_gemini_app() { echo -e "${BLUE}--- Replace Gemini Enterprise Application / Load Balancer Routing ---${NC}" - echo -e "${YELLOW}WARNING: This will create a NEW Gemini Enterprise Application and update the Load Balancer to route traffic to it.${NC}" + echo -e "${RED}WARNING: This will create a NEW Gemini Enterprise Application and update the Load Balancer to route traffic to it.${NC}" echo -e "${YELLOW}The old application will NOT be deleted automatically.${NC}" echo "" read -p "Are you sure you want to proceed? (y/N): " CONFIRM @@ -1842,7 +2706,7 @@ replace_gemini_app() { fi # 1. Create new App - configure_gem4gov + configure_gem4gov || return 1 # 2. Update Networking (Stage 1) echo "" @@ -1869,22 +2733,18 @@ import_documents_helper() { return 1 fi - # Ensure BUCKET_NAME is set from STATE_BUCKET if not already - # This covers the case where the user navigates directly to this helper function - if [[ -z "$BUCKET_NAME" && -n "$STATE_BUCKET" ]]; then - BUCKET_NAME=$(echo "$STATE_BUCKET" | sed 's/gs:\/\/ //' | sed 's/\/$//') - fi - - echo "Retrieving state from gs://${BUCKET_NAME}/terraform/state/stage-0/default.tfstate..." - STATE_CONTENT=$(gcloud storage cat "gs://${BUCKET_NAME}/terraform/state/stage-0/default.tfstate" 2>/dev/null || echo "{}") + # Hydrate state and populate STATE_CONTENT + hydrate_from_state # Parse GCS Data Stores - # Output: gcs_data_store_to_bucket = { "ds-id": "bucket-name" } - GCS_DS_MAP=$(echo "$STATE_CONTENT" | jq -r '.outputs.gcs_data_store_to_bucket.value // {}') + GCS_DS_MAP=$(echo "$STATE_CONTENT" | jq -c ' + .outputs.gcs_data_stores.value // {} | to_entries | map(select(.value.data_store_id != null)) | map(.value) + ' 2>/dev/null) # Parse BigQuery Data Stores - # Output: bq_data_store_to_dataset_table = { "ds-id": { "dataset_id": "...", "table_id": "..." } } - BQ_DS_MAP=$(echo "$STATE_CONTENT" | jq -r '.outputs.bq_data_store_to_dataset_table.value // {}') + BQ_DS_MAP=$(echo "$STATE_CONTENT" | jq -c ' + .outputs.bq_data_stores.value // {} | to_entries | map(select(.value.data_store_id != null)) | map(.value) + ' 2>/dev/null) echo "" echo "Available Data Stores:" @@ -1897,25 +2757,31 @@ import_documents_helper() { COUNT=0 # List GCS Data Stores - for key in $(echo "$GCS_DS_MAP" | jq -r 'keys[]'); do - BUCKET=$(echo "$GCS_DS_MAP" | jq -r --arg k "$key" '.[$k]') - COUNT=$((COUNT+1)) - echo "${COUNT}. [GCS] ${key} (Bucket: ${BUCKET})" - DS_IDS+=("$key") - DS_TYPES+=("gcs") - DS_SOURCES+=("$BUCKET") # Store bucket name for display/verification if needed - done + if [[ "$GCS_DS_MAP" != "[]" && -n "$GCS_DS_MAP" ]]; then + for i in $(jq -r 'keys[]' <<< "$GCS_DS_MAP"); do + DS_ID=$(jq -r ".[$i].data_store_id" <<< "$GCS_DS_MAP") + BUCKET=$(jq -r ".[$i].bucket_name" <<< "$GCS_DS_MAP") + COUNT=$((COUNT+1)) + echo "${COUNT}. [GCS] ${DS_ID} (Bucket: ${BUCKET})" + DS_IDS+=("$DS_ID") + DS_TYPES+=("gcs") + DS_SOURCES+=("$BUCKET") + done + fi # List BigQuery Data Stores - for key in $(echo "$BQ_DS_MAP" | jq -r 'keys[]'); do - DATASET=$(echo "$BQ_DS_MAP" | jq -r --arg k "$key" '.[$k].dataset_id') - TABLE=$(echo "$BQ_DS_MAP" | jq -r --arg k "$key" '.[$k].table_id') - COUNT=$((COUNT+1)) - echo "${COUNT}. [BigQuery] ${key} (Table: ${DATASET}.${TABLE})" - DS_IDS+=("$key") - DS_TYPES+=("bigquery") - DS_SOURCES+=("${DATASET}.${TABLE}") - done + if [[ "$BQ_DS_MAP" != "[]" && -n "$BQ_DS_MAP" ]]; then + for i in $(jq -r 'keys[]' <<< "$BQ_DS_MAP"); do + DS_ID=$(jq -r ".[$i].data_store_id" <<< "$BQ_DS_MAP") + DATASET=$(jq -r ".[$i].dataset_id" <<< "$BQ_DS_MAP") + TABLE=$(jq -r ".[$i].table_id" <<< "$BQ_DS_MAP") + COUNT=$((COUNT+1)) + echo "${COUNT}. [BigQuery] ${DS_ID} (Table: ${DATASET}.${TABLE})" + DS_IDS+=("$DS_ID") + DS_TYPES+=("bigquery") + DS_SOURCES+=("${DATASET}.${TABLE}") + done + fi if [[ "$COUNT" -eq 0 ]]; then echo -e "${YELLOW}No data stores found in Stage 0 state.${NC}" @@ -1924,7 +2790,11 @@ import_documents_helper() { fi echo "" - read -p "Select a Data Store to import into [1-${COUNT}]: " SELECTION + RANGE_STR="[1]" + if [[ "$COUNT" -gt 1 ]]; then + RANGE_STR="[1-${COUNT}]" + fi + read -p "Select a Data Store to import into ${RANGE_STR}: " SELECTION if [[ ! "$SELECTION" =~ ^[0-9]+$ ]] || [[ "$SELECTION" -lt 1 ]] || [[ "$SELECTION" -gt "$COUNT" ]]; then echo -e "${RED}Invalid selection.${NC}" @@ -1936,32 +2806,400 @@ import_documents_helper() { INDEX=$((SELECTION-1)) SELECTED_ID="${DS_IDS[$INDEX]}" SELECTED_TYPE="${DS_TYPES[$INDEX]}" - + SELECTED_SOURCE="${DS_SOURCES[$INDEX]}" echo -e "${GREEN}Selected: ${SELECTED_ID} (${SELECTED_TYPE})${NC}" echo "" - CMD="gem4gov datastore import --project-id ${PROJECT_ID} --data-store-id ${SELECTED_ID} --source-type ${SELECTED_TYPE}" - - echo "Running: $CMD" - export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}" - export GOOGLE_CLOUD_QUOTA_PROJECT="${PROJECT_ID}" - $CMD + if [[ "$SELECTED_TYPE" == "gcs" ]]; then + CMD_ARRAY=(gem4gov datastore import --project-id "${PROJECT_ID}" --data-store-id "${SELECTED_ID}" --source-type "${SELECTED_TYPE}" --gcs-bucket "${SELECTED_SOURCE}") + if ! "${CMD_ARRAY[@]}"; then + echo -e "${RED}Error: Failed to import documents from GCS.${NC}" + return 1 + fi + export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}" + export GOOGLE_CLOUD_QUOTA_PROJECT="${PROJECT_ID}" + elif [[ "$SELECTED_TYPE" == "bigquery" ]]; then + echo -e "${YELLOW}--- BigQuery Document Import ---${NC}" + + # BQ_DS_MAP gives us dataset.table directly in SELECTED_SOURCE variable + # Use tr to remove any lingering carriage returns from jq parsing + CLEAN_SOURCE=$(echo "$SELECTED_SOURCE" | tr -d '\r\n ') + SOURCE_PARTS=(${CLEAN_SOURCE//./ }) + DATASET=${SOURCE_PARTS[0]} + TABLE=${SOURCE_PARTS[1]} + + USE_EXISTING="n" + CUSTOM_SCHEMA_FILE="" + + echo "Detecting BigQuery Table Schema for ${PROJECT_ID}:${DATASET}.${TABLE}..." + + # Capture schema. We evaluate BQ_EXIT locally to prevent $? from being overwritten by echos. + BQ_SCHEMA_JSON=$(PYTHONPATH="" bq show --schema --format=prettyjson "${PROJECT_ID}:${DATASET}.${TABLE}") + BQ_EXIT=$? + + echo "## DEBUG bq exit code: $BQ_EXIT" + + if [ $BQ_EXIT -eq 0 ] && [ -n "$BQ_SCHEMA_JSON" ] && [ "$BQ_SCHEMA_JSON" != "null" ]; then + echo "" + echo "Successfully retrieved BigQuery Table schema:" + echo "$BQ_SCHEMA_JSON" | jq '.' + echo "" + read -p "Would you like to use this schema for the bigquery data store? (y/N): " USE_EXISTING + else + echo -e "${RED}Failed to automatically detect BigQuery schema or table is empty.${NC}" + fi + + if [[ "$USE_EXISTING" != "y" && "$USE_EXISTING" != "Y" ]]; then + echo "" + read -p "Enter the path to your custom JSON schema file: " CUSTOM_SCHEMA_FILE + + # 1. Check if the file exists and is a regular file + if [[ ! -f "$CUSTOM_SCHEMA_FILE" ]]; then + echo -e "${RED}Error: File '$CUSTOM_SCHEMA_FILE' not found or is not a regular file.${NC}" + pause + return 1 + fi + + # 2. Prevent explicit path traversal strings + if [[ "$CUSTOM_SCHEMA_FILE" == *"../"* || "$CUSTOM_SCHEMA_FILE" == *".." ]]; then + echo -e "${RED}Error: Invalid file path. Path traversal sequences ('../') are not allowed.${NC}" + pause + return 1 + fi + + # 3. Restrict execution to only files inside the project working directory + BASE_DIR=$(pwd) + ABS_PATH=$(realpath "$CUSTOM_SCHEMA_FILE" 2>/dev/null || echo "") + + if [[ -z "$ABS_PATH" || "$ABS_PATH" != "$BASE_DIR"* ]]; then + echo -e "${RED}Error: Custom schema file must be located within the project folder directory (${BASE_DIR}).${NC}" + pause + return 1 + fi + fi + + echo "" + echo -e "${YELLOW}--- Schema Property Mapping ---${NC}" + echo "A Document ID field is required." + echo "Optional Semantic Key Properties: title, description, category, uri" + echo "" + + # Extract fields to show the user + if [[ "$USE_EXISTING" == "y" || "$USE_EXISTING" == "Y" ]]; then + # Show BQ fields + FIELDS=$(echo "$BQ_SCHEMA_JSON" | jq -r '[.[].name] | join(", ")') + else + # Show fields from Custom JSON schema (assumes top level properties) + FIELDS=$(cat "$CUSTOM_SCHEMA_FILE" | jq -r '[.properties | keys[]] | join(", ")') + fi + + echo "Available Fields: $FIELDS" + echo "" + + read -p "Enter the field you would like to use as the unique identifier (leave blank for Auto): " ID_FIELD + read -p "Enter the field you would like to use for 'title' key property (leave blank for None): " TITLE_FIELD + read -p "Enter the field you would like to use for 'description' key property (leave blank for None): " DESC_FIELD + read -p "Enter the field you would like to use for 'category' key property (leave blank for None): " CAT_FIELD + read -p "Enter the field you would like to use for 'uri' key property (leave blank for None): " URI_FIELD + + # Generate Discovery Engine JSON Schema using inline Python + echo "" + echo "Generating Discovery Engine Schema..." + + export PYTHONPATH="" + export BQ_SCHEMA_JSON + export CUSTOM_SCHEMA_FILE + export TITLE_FIELD DESC_FIELD CAT_FIELD URI_FIELD + DE_SCHEMA_JSON=$(python3 << 'EOF' +import json +import os + +key_mappings = { + "title": os.environ.get("TITLE_FIELD", ""), + "description": os.environ.get("DESC_FIELD", ""), + "category": os.environ.get("CAT_FIELD", ""), + "uri": os.environ.get("URI_FIELD", "") +} +key_properties = {k: v for k, v in key_mappings.items() if v} + +custom_file = os.environ.get("CUSTOM_SCHEMA_FILE", "") +if custom_file: + with open(custom_file, "r") as f: + schema = json.load(f) + for key, val in key_properties.items(): + if val in schema.get("properties", {}): + schema["properties"][val]["keyPropertyMapping"] = key +else: + bq_schema = json.loads(os.environ.get("BQ_SCHEMA_JSON", "[]")) + + def get_json_type(bq_type): + if bq_type in ["INTEGER", "INT64"]: return "integer" + elif bq_type in ["FLOAT", "FLOAT64", "NUMERIC", "BIGNUMERIC"]: return "number" + elif bq_type in ["BOOLEAN", "BOOL"]: return "boolean" + return "string" + + def transform(fields): + props = {} + for f in fields: + fname = f["name"] + ftype = f.get("type", "STRING") + if ftype in ["RECORD", "STRUCT"]: + pdef = {"type": "object", "properties": transform(f.get("fields", []))} + else: + jtype = get_json_type(ftype) + is_matched_key = fname in key_properties.values() + + pdef = {"type": jtype} + if is_matched_key: + matched_key = [k for k,v in key_properties.items() if v == fname][0] + pdef["keyPropertyMapping"] = matched_key + pdef["retrievable"] = True if jtype in ["number", "string", "boolean", "integer", "datetime", "geolocation"] else False + else: + pdef["searchable"] = True if jtype == "string" else False + pdef["indexable"] = True if jtype in ["number", "string", "boolean", "integer", "datetime", "geolocation"] else False + pdef["retrievable"] = True if jtype in ["number", "string", "boolean", "integer", "datetime", "geolocation"] else False + + if f.get("mode") == "REPEATED": + props[fname] = {"type": "array", "items": pdef} + else: + props[fname] = pdef + return props + + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": transform(bq_schema) + } + +print(json.dumps(schema)) +EOF +) + + echo "Retrieving access token..." + ACCESS_TOKEN=$(gcloud auth print-access-token) + + # Patch Default Schema + echo "Patching Data Store Default Schema..." + SCHEMA_URL="https://us-discoveryengine.googleapis.com/v1alpha/projects/${PROJECT_ID}/locations/us/collections/default_collection/dataStores/${SELECTED_ID}/schemas/default_schema" + + PATCH_BODY="{\"structSchema\": $DE_SCHEMA_JSON}" + + SCHEMA_RESPONSE=$(curl -s -X PATCH \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "x-goog-user-project: ${PROJECT_ID}" \ + -H "Content-Type: application/json" \ + -d "$PATCH_BODY" \ + "${SCHEMA_URL}") + + if echo "$SCHEMA_RESPONSE" | grep -q "\"error\""; then + echo -e "${RED}Error patching schema:${NC}" + echo "$SCHEMA_RESPONSE" | jq '.' + echo "Aborting import." + pause + return 1 + fi + + echo -e "${GREEN}Default Schema patched successfully.${NC}" + + # Start Document Import + echo "Starting BigQuery Document Import..." + IMPORT_URL="https://us-discoveryengine.googleapis.com/v1alpha/projects/${PROJECT_ID}/locations/us/collections/default_collection/dataStores/${SELECTED_ID}/branches/default_branch/documents:import" + + IMPORT_BODY="{\"reconciliationMode\": \"FULL\", \"bigquerySource\": {\"projectId\": \"${PROJECT_ID}\", \"datasetId\": \"${DATASET}\", \"tableId\": \"${TABLE}\", \"dataSchema\": \"custom\"}" + if [[ -z "$ID_FIELD" ]]; then + IMPORT_BODY="${IMPORT_BODY}, \"autoGenerateIds\": true}" + else + IMPORT_BODY="${IMPORT_BODY}, \"idField\": \"${ID_FIELD}\"}" + fi + + IMPORT_RESPONSE=$(curl -s -X POST \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "x-goog-user-project: ${PROJECT_ID}" \ + -H "Content-Type: application/json" \ + -d "$IMPORT_BODY" \ + "${IMPORT_URL}") + + if echo "$IMPORT_RESPONSE" | grep -q "\"error\""; then + echo -e "${RED}Error starting import:${NC}" + echo "$IMPORT_RESPONSE" | jq '.' + pause + return 1 + fi + + echo -e "${GREEN}Document import operation started successfully!${NC}" + echo "Operation Details:" + echo "$IMPORT_RESPONSE" | jq '.' + + fi pause } +distribute_gemini_licenses() { + echo -e "${BLUE}--- Distribute Gemini for Government Licenses ---${NC}" + + echo -e "${YELLOW}Prerequisites:${NC}" + echo "1. You must have the 'Billing Account Administrator' role on the Billing Account." + echo "2. You must have the 'Service Usage Consumer' role on the project used for API calls." + echo "" + read -p "Have you confirmed these prerequisites? (y/N): " PRE_CONFIRM + if [[ "$PRE_CONFIRM" != "y" && "$PRE_CONFIRM" != "Y" ]]; then + return 0 + fi + + # Ensure Project ID is set for API quota + if [[ -z "$PROJECT_ID" ]]; then + echo -e "${RED}Project ID is required for API quota. Please select a project first.${NC}" + return 1 + fi + + # Setup gem4gov CLI path + GEM4GOV_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/gem4gov-cli/gem4gov.py" + + while true; do + read -p "Enter Billing Account ID (e.g., 012345-6789AB-CDEFGH): " BILLING_ACCOUNT_ID + if [[ -z "$BILLING_ACCOUNT_ID" ]]; then + echo -e "${RED}Billing Account ID is required.${NC}" + continue + fi + break + done + + while true; do + echo -e "${BLUE}Fetching available Gemini Enterprise subscriptions...${NC}" + + # Use new gem4gov command + CONFIGS_JSON=$(python3 "$GEM4GOV_PATH" license list --billing-account "$BILLING_ACCOUNT_ID" --quota-project "$PROJECT_ID" --format json) + + if [[ $? -ne 0 ]] || [[ -z "$CONFIGS_JSON" ]] || [[ "$CONFIGS_JSON" == "[]" ]]; then + echo -e "${RED}Failed to fetch license configurations or no configurations found.${NC}" + echo "$CONFIGS_JSON" + pause + return 1 + fi + + COUNT=$(echo "$CONFIGS_JSON" | jq '. | length') + + echo "Available Subscriptions:" + echo "-----------------------------------" + for i in $(seq 0 $((COUNT-1))); do + CONFIG=$(echo "$CONFIGS_JSON" | jq -c ".[$i]") + NAME=$(echo "$CONFIG" | jq -r '.subscriptionDisplayName // .name') + TOTAL=$(echo "$CONFIG" | jq -r '.licenseCount') + CONFIG_ID=$(echo "$CONFIG" | jq -r '.name' | awk -F'/' '{print $NF}') + + # Calculate distributed licenses + DISTRIBUTED=$(echo "$CONFIG" | jq -r '.licenseConfigDistributions | values | map(tonumber) | add // 0') + AVAILABLE=$((TOTAL - DISTRIBUTED)) + + echo "$((i+1)). ${NAME}" + echo " ID: ${CONFIG_ID}" + echo " Total Licenses: ${TOTAL}" + echo " Distributed: ${DISTRIBUTED}" + echo " Available: ${AVAILABLE}" + echo "-----------------------------------" + done + + read -p "Select a subscription to distribute from [1-${COUNT}, or 'q' to quit]: " SEL + if [[ "$SEL" == "q" ]]; then + return 0 + fi + + if [[ ! "$SEL" =~ ^[0-9]+$ ]] || [[ "$SEL" -lt 1 ]] || [[ "$SEL" -gt "$COUNT" ]]; then + echo -e "${RED}Invalid selection.${NC}" + continue + fi + + SELECTED_CONFIG=$(echo "$CONFIGS_JSON" | jq -c ".[$((SEL-1))]") + SELECTED_CONFIG_ID=$(echo "$SELECTED_CONFIG" | jq -r '.name' | awk -F'/' '{print $NF}') + + read -p "Enter Target Project ID (where licenses will be allocated): " TARGET_PROJECT_ID + if [[ -z "$TARGET_PROJECT_ID" ]]; then + echo -e "${RED}Target Project ID is required.${NC}" + continue + fi + + TARGET_PROJECT_NUMBER=$(gcloud projects describe "${TARGET_PROJECT_ID}" --format="value(projectNumber)" 2>/dev/null) + if [[ -z "$TARGET_PROJECT_NUMBER" ]]; then + echo -e "${RED}Could not find project: ${TARGET_PROJECT_ID}${NC}" + continue + fi + + echo "Select Location:" + echo "1. global" + echo "2. us" + echo "3. eu" + read -p "Select an option [1-3]: " LOC_SEL + case "$LOC_SEL" in + 1) LOCATION="global" ;; + 2) LOCATION="us" ;; + 3) LOCATION="eu" ;; + *) echo -e "${RED}Invalid location selection.${NC}"; continue ;; + esac + + read -p "Number of licenses to distribute (Incremental): " LICENSE_COUNT + if [[ ! "$LICENSE_COUNT" =~ ^[0-9]+$ ]]; then + echo -e "${RED}Invalid license count.${NC}" + continue + fi + + # Check for existing config + EXISTING_LICENSE_CONFIG_ID=$(echo "$SELECTED_CONFIG" | jq -r --arg pn "$TARGET_PROJECT_NUMBER" --arg loc "$LOCATION" ' + .licenseConfigDistributions // {} | keys[] | select(contains("projects/\($pn)/locations/\($loc)")) | split("/") | last + ' | head -n 1) + + echo "" + echo "Distribution Summary:" + echo "Subscription: ${SELECTED_CONFIG_ID}" + echo "Target Project: ${TARGET_PROJECT_ID} (${TARGET_PROJECT_NUMBER})" + echo "Location: ${LOCATION}" + echo "Count: ${LICENSE_COUNT}" + if [[ -n "$EXISTING_LICENSE_CONFIG_ID" ]]; then + echo "Existing License Config ID Found: ${EXISTING_LICENSE_CONFIG_ID}" + fi + echo "" + read -p "Confirm distribution? (y/N): " CONFIRM + if [[ "$CONFIRM" != "y" && "$CONFIRM" != "Y" ]]; then + continue + fi + + echo -e "${BLUE}Running API call via gem4gov CLI...${NC}" + + # Build command as an array to prevent injection + CMD_ARRAY=(python3 "$GEM4GOV_PATH" license distribute --billing-account "$BILLING_ACCOUNT_ID" --config-id "$SELECTED_CONFIG_ID" --target-project-number "$TARGET_PROJECT_NUMBER" --location "$LOCATION" --count "$LICENSE_COUNT" --quota-project "$PROJECT_ID") + if [[ -n "$EXISTING_LICENSE_CONFIG_ID" ]]; then + CMD_ARRAY+=("--license-config-id" "$EXISTING_LICENSE_CONFIG_ID") + fi + + "${CMD_ARRAY[@]}" + + if [[ $? -eq 0 ]]; then + echo -e "${GREEN}Licenses distributed successfully!${NC}" + else + echo -e "${RED}Distribution failed.${NC}" + fi + + echo "" + read -p "Would you like to perform another distribution? (y/N): " ANOTHER + if [[ "$ANOTHER" != "y" && "$ANOTHER" != "Y" ]]; then + break + fi + done +} + helper_menu() { while true; do clear print_header echo -e "${BLUE}--- Helper Functions ---${NC}" - echo "1. Update Gemini Enterprise App Compliance" + echo "1. Update Gemini for Government Compliance" echo "2. Replace Gemini Enterprise Application / Load Balancer Routing" echo "3. Import Documents to Gemini Enterprise Data Store (Cloud Storage / BigQuery)" - echo "4. Upload SSL Certificate" - echo "5. Back to Main Menu" + echo "4. Distribute Gemini for Government Licenses" + echo "5. Upload SSL Certificate" + echo "6. Back to Main Menu" echo "-----------------------------------" - read -p "Select an option [1-5]: " OPTION + read -p "Select an option [1-6]: " OPTION case $OPTION in 1) @@ -1974,9 +3212,12 @@ helper_menu() { import_documents_helper ;; 4) - upload_ssl_certificate + distribute_gemini_licenses ;; 5) + upload_ssl_certificate + ;; + 6) return 0 ;; *) @@ -1995,8 +3236,13 @@ configure_stage_1() { mkdir -p gemini-stage-1 if [[ -f "gemini-stage-1/terraform.tfvars" ]]; then + echo -e "${RED}WARNING: Answering 'n' will OVERWRITE existing gemini-stage-1/terraform.tfvars${NC}" read -p "Reuse existing configuration? (Y/n): " REUSE_CONFIG if [[ "$REUSE_CONFIG" != "n" && "$REUSE_CONFIG" != "N" ]]; then + # Ensure stage_0_state_bucket is updated with sanitary BUCKET_NAME + if [[ -n "$BUCKET_NAME" ]]; then + sed -i '' "s/stage_0_state_bucket *= *\".*\"/stage_0_state_bucket = \"${BUCKET_NAME}\"/" gemini-stage-1/terraform.tfvars 2>/dev/null || sed -i "s/stage_0_state_bucket *= *\".*\"/stage_0_state_bucket = \"${BUCKET_NAME}\"/" gemini-stage-1/terraform.tfvars + fi return 0 fi fi @@ -2005,20 +3251,12 @@ configure_stage_1() { if [[ -z "$REGION" ]]; then echo "Retrieving region from state..." - # Ensure BUCKET_NAME is set from STATE_BUCKET if not already - if [[ -z "$BUCKET_NAME" && -n "$STATE_BUCKET" ]]; then - BUCKET_NAME=$(echo "$STATE_BUCKET" | sed 's/gs:\/\/ //' | sed 's/\/$//') - fi - - STATE_CONTENT=$(gcloud storage cat "gs://${BUCKET_NAME}/terraform/state/stage-0/default.tfstate" 2>/dev/null || echo "{}") - REGION=$(echo "$STATE_CONTENT" | jq -r '.outputs.region.value // empty') + hydrate_from_state if [[ -z "$REGION" ]]; then # Try to get it from the bucket location or default REGION="us-central1" - echo -e "${YELLOW}Warning: Could not retrieve region from state. Using default: ${REGION}${NC}" - else - echo -e "Region retrieved: ${YELLOW}${REGION}${NC}" + echo -e "${RED}WARNING: Could not retrieve region from state. Using default: ${REGION}${NC}" fi fi @@ -2027,7 +3265,7 @@ configure_stage_1() { # Validate DNS echo "Validating DNS for ${GEMINI_DOMAIN}..." if [[ -z "$STATE_CONTENT" ]]; then - STATE_CONTENT=$(gcloud storage cat "gs://${BUCKET_NAME}/terraform/state/stage-0/default.tfstate" 2>/dev/null || echo "{}") + hydrate_from_state fi LB_IP=$(echo "$STATE_CONTENT" | jq -r '.outputs.gemini_enterprise_ip.value // empty') @@ -2038,7 +3276,7 @@ configure_stage_1() { echo -e "${GREEN}DNS Validation Successful: ${GEMINI_DOMAIN} resolves to ${LB_IP}${NC}" else RESOLVED_IPS=$(dig +short "$GEMINI_DOMAIN" | tr '\n' ' ') - echo -e "${YELLOW}WARNING: DNS Validation Failed!${NC}" + echo -e "${RED}WARNING: DNS Validation Failed!${NC}" echo -e "Expected IP: ${LB_IP}" echo -e "Resolved IPs: ${RESOLVED_IPS:-None}" echo -e "${YELLOW}Please ensure your DNS A record is correctly pointing to ${LB_IP}.${NC}" @@ -2048,26 +3286,40 @@ configure_stage_1() { fi fi else - echo -e "${YELLOW}Warning: Could not retrieve Load Balancer IP from state. Skipping DNS validation.${NC}" + echo -e "${RED}WARNING: Could not retrieve Load Balancer IP from state. Skipping DNS validation.${NC}" fi - # Auto-discover SSL Certificates - echo "" - echo "Discovering SSL Certificates in Region ${REGION}..." - CERTS_JSON=$(gcloud compute ssl-certificates list --filter="region:(${REGION})" --format="json" 2>/dev/null) - - if [[ -n "$CERTS_JSON" && "$CERTS_JSON" != "[]" ]]; then - echo "Available SSL Certificates:" - echo "$CERTS_JSON" | jq -r '.[] | "\(.name) (\(.type))"' | nl -w2 -s") " - - read -p "Select an SSL Certificate [1]: " CERT_SEL - CERT_SEL=${CERT_SEL:-1} - - SSL_CERT_NAME=$(echo "$CERTS_JSON" | jq -r ".[$((CERT_SEL-1))].name") - echo -e "Selected Certificate: ${YELLOW}${SSL_CERT_NAME}${NC}" + if [[ -f "gemini-stage-0/terraform.tfvars" ]]; then + CERT_MANAGEMENT_CHOICE=$(grep "cert_management_choice" gemini-stage-0/terraform.tfvars | awk -F'=' '{print $2}' | tr -d ' "') + CUSTOM_DOMAIN=$(grep "custom_domain" gemini-stage-0/terraform.tfvars | awk -F'=' '{print $2}' | tr -d ' "') + else + CERT_MANAGEMENT_CHOICE="self_managed" + CUSTOM_DOMAIN="" + fi + + if [[ "$CERT_MANAGEMENT_CHOICE" == "google_managed" ]]; then + echo "" + echo -e "${YELLOW}Google-managed certificate selected in Stage 0. Skipping manual SSL certificate selection.${NC}" + SSL_CERT_NAME="" else - echo -e "${YELLOW}No SSL Certificates found in region ${REGION}.${NC}" - read -p "Enter SSL Certificate Name (must exist in GCP): " SSL_CERT_NAME + # Auto-discover SSL Certificates + echo "" + echo "Discovering SSL Certificates in Region ${REGION}..." + CERTS_JSON=$(gcloud compute ssl-certificates list --filter="region:(${REGION})" --format="json" 2>/dev/null) + + if [[ -n "$CERTS_JSON" && "$CERTS_JSON" != "[]" ]]; then + echo "Available SSL Certificates:" + echo "$CERTS_JSON" | jq -r '.[] | "\(.name) (\(.type))"' | nl -w2 -s") " + + read -p "Select an SSL Certificate [1]: " CERT_SEL + CERT_SEL=${CERT_SEL:-1} + + SSL_CERT_NAME=$(echo "$CERTS_JSON" | jq -r ".[$((CERT_SEL-1))].name") + echo -e "Selected Certificate: ${YELLOW}${SSL_CERT_NAME}${NC}" + else + echo -e "${YELLOW}No SSL Certificates found in region ${REGION}.${NC}" + read -p "Enter SSL Certificate Name (must exist in GCP): " SSL_CERT_NAME + fi fi read -p "Enter Gemini Widget Config ID (from Step 2 output): " GEMINI_CONFIG_ID @@ -2077,6 +3329,8 @@ stage_0_state_bucket = "${BUCKET_NAME}" gemini_enterprise_domain = "${GEMINI_DOMAIN}" ssl_certificate_name = "${SSL_CERT_NAME}" gemini_config_id = "${GEMINI_CONFIG_ID}" +cert_management_choice = "${CERT_MANAGEMENT_CHOICE}" +custom_domain = "${CUSTOM_DOMAIN}" EOF # Add Shared VPC vars if needed (simple check) @@ -2095,6 +3349,7 @@ deploy_stage_1() { cd gemini-stage-1 rm -f backend.tf + rm -rf .terraform echo "Initializing Terraform..." if ! terraform init -migrate-state -backend-config="bucket=${BUCKET_NAME}" -backend-config="prefix=terraform/state/stage-1"; then @@ -2141,7 +3396,8 @@ deploy_stage_1() { echo "5. Click 'Create'. (Do not add redirect URIs yet)." echo "6. Copy the 'Client ID' and 'Client Secret'." echo -e "${NC}" - read -p "Press Enter after you have created the client..." + echo "" + read -p "Press Enter to acknowledge and continue..." echo "" echo -e "${YELLOW}Step 2: Update Redirect URI${NC}" @@ -2149,7 +3405,8 @@ deploy_stage_1() { echo -e "2. Add the following Authorized redirect URI (replace [CLIENT_ID] with the actual ID you just copied): ${BLUE}https://iap.googleapis.com/v1/oauth/clientIds/[CLIENT_ID]:handleRedirect${NC}" echo "3. Save the changes." echo -e "${NC}" - read -p "Press Enter after you have updated the redirect URI..." + echo "" + read -p "Press Enter to acknowledge and continue..." echo "" echo -e "${YELLOW}Step 3: Configure IAP for Workforce Identity${NC}" @@ -2161,7 +3418,8 @@ deploy_stage_1() { echo " - OAuth client secret: (Paste from Step 1)" echo "6. Click 'Save'." echo -e "${NC}" - read -p "Press Enter after you have configured IAP..." + echo "" + read -p "Press Enter to acknowledge and continue..." echo "" echo -e "${GREEN}OAuth and IAP Manual Configuration marked as complete.${NC}" fi @@ -2176,6 +3434,9 @@ deploy_stage_1() { main_menu() { while true; do clear + # Attempt to hydrate state to populate variables for menu display + hydrate_from_state + print_header echo -e "Current Project: ${YELLOW}${PROJECT_ID:-None}${NC}" echo -e "Deployment Topology: ${YELLOW}${DEPLOYMENT_TYPE_TEXT:-None}${NC}" @@ -2236,7 +3497,6 @@ main_menu() { esac done } - # --- Entry Point --- check_dependencies diff --git a/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/auth.py b/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/auth.py index de28cc15d..7c10684f1 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/auth.py +++ b/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/auth.py @@ -1,3 +1,17 @@ +# Copyright 2026 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import click import google.auth from googleapiclient.discovery import build diff --git a/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/data_stores.py b/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/data_stores.py index 8f5ac4353..2739848c6 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/data_stores.py +++ b/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/data_stores.py @@ -1,3 +1,17 @@ +# Copyright 2026 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import click import json from googleapiclient.errors import HttpError diff --git a/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/engine_features.yaml b/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/engine_features.yaml index f24061eae..f934db9a8 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/engine_features.yaml +++ b/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/engine_features.yaml @@ -11,10 +11,13 @@ features: bi-directional-audio: "FEATURE_STATE_OFF" feedback: "FEATURE_STATE_OFF" session-sharing: "FEATURE_STATE_OFF" - disable-agent-sharing: "FEATURE_STATE_ON" personalization-memory: "FEATURE_STATE_OFF" + personalization-suggested-highlights: "FEATURE_STATE_OFF" + disable-agent-sharing: "FEATURE_STATE_OFF" + agent-sharing-without-admin-approval: "FEATURE_STATE_OFF" disable-image-generation: "FEATURE_STATE_ON" disable-video-generation: "FEATURE_STATE_ON" disable-onedrive-upload: "FEATURE_STATE_ON" - disable-talk-to-content: "FEATURE_STATE_ON" + disable-talk-to-content: "FEATURE_STATE_OFF" disable-google-drive-upload: "FEATURE_STATE_ON" + disable-welcome-emails: "FEATURE_STATE_OFF" diff --git a/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/gem4gov.py b/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/gem4gov.py index 01ac63497..0b1e0aa55 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/gem4gov.py +++ b/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/gem4gov.py @@ -1,3 +1,18 @@ +# Copyright 2026 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys import click import google.auth from googleapiclient.discovery import build @@ -29,7 +44,7 @@ ) # Global Varibles used in prompts -supported_aw_boundaries = "FedRAMP High, IL4" +supported_aw_boundaries = "FedRAMP High, IL4, IL5" required_apis = "Vertex AI, Discovery Engine, Cloud Resource Manager, Cloud Key Management Service (KMS), Identity and Access Management (IAM), Service Usage, Cloud Storage, BigQuery" supported_data_stores = "Cloud Storage, BigQuery" @@ -62,7 +77,7 @@ def init(): except (subprocess.CalledProcessError, FileNotFoundError): click.echo("Could not set the gcloud project configuration. Please ensure gcloud is installed and configured correctly.") click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) credentials = get_credentials() click.echo(f"Successfully set project ID to: {project_id}") @@ -81,8 +96,9 @@ def onboard(): click.echo("What compliance regime will Gemini for Government be deployed in?") click.echo("1) FedRAMP High") click.echo("2) IL4") - click.echo("3) None") - compliance_regime_id = click.prompt('Please enter the number for your response', type=click.Choice(['1', '2', '3']), default = '1', show_default = False) + click.echo("3) IL5") + click.echo("4) None") + compliance_regime_id = click.prompt('Please enter the number for your response', type=click.Choice(['1', '2', '3', '4']), default = '1', show_default = False) click.echo(nl=True) click.echo(nl=True) @@ -93,7 +109,7 @@ def onboard(): else: click.echo(click.style("Please create an Assured Workloads folder and a GCP Project within that folder before continuing.", fg='red')) click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) try: subprocess.run(['gcloud', 'config', 'set', 'project', project_id], check=True, capture_output=True) @@ -102,7 +118,7 @@ def onboard(): except (subprocess.CalledProcessError, FileNotFoundError): click.echo("Could not set the gcloud project configuration. Please ensure gcloud is installed and configured correctly.") click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) # Set the quota project on the credentials credentials = credentials.with_quota_project(project_id) @@ -116,7 +132,7 @@ def onboard(): if not click.confirm('Would you like to continue the Onboarding process anyway?'): click.echo(click.style("Please grant your user the list of IAM roles found in the README or have another user with those roles run the Onboarding process.", fg='red')) click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) click.echo(nl=True) click.echo(nl=True) @@ -147,7 +163,7 @@ def onboard(): else: click.echo(click.style("Please configure a Workforce Identity Pool and Provider before continuing the Onboarding process", fg='red')) click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) idp_type = configure_identity_provider(credentials, project_id, idp_select, workforce_pool_id) click.echo(nl=True) click.echo(nl=True) @@ -175,11 +191,11 @@ def onboard(): click.echo("Failed to grant KMS permissions. Please check the error and try again.") if not click.confirm('Would you like to try again?'): click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) else: if not click.confirm('The KMS key is invalid. Would you like to try again?'): click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) elif cmek_action == '2': click.echo('Please navigate to https://console.cloud.google.com/security/kms/keyrings and create a Cloud KMS Key Ring in the "us" multi-region.') @@ -199,11 +215,11 @@ def onboard(): click.echo("Failed to grant KMS permissions. Please check the error and try again.") if not click.confirm('Would you like to try again?'): click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) else: if not click.confirm('The KMS key is invalid. Would you like to try again?'): click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) else: click.echo('You can always setup the Gemini Enterprise CMEK configuration at a later time.') click.echo('NOTE: Ensure that Gemini Enterprise CMEK configuration is setup before adding any data stores to your Gemini Enterprise application.') @@ -238,7 +254,7 @@ def onboard(): if not click.confirm('Would you like to continue without setting up Gemini Enterprise CMEK configuration?'): click.echo(click.style('Please run `gem4gov onboard` again to setup Gemini Enterprise CMEK configuration.', fg="red")) click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) click.echo(click.style(f"Gemini Enterprise data stores allow end-users to search and ask questions based on a variety of first and third-party datasets. Currently, the only data stores that are available in Gemini for Governement customers are: {supported_data_stores}", fg='yellow')) while True: if click.confirm('Do you have an existing data store(s) already created and loaded with data?'): @@ -361,7 +377,7 @@ def onboard(): if len(data_store_list) == 0: if not click.confirm('Would you like to continue the Onboarding process for a Default Gemini Enterpise application?'): click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) else: app_type = '1' break @@ -441,7 +457,6 @@ def onboard(): click.echo(click.style("- NotebookLM Enterprise", fg="yellow")) click.echo(click.style("- Prompt Gallery", fg="yellow")) click.echo(click.style("- Session Sharing", fg="yellow")) - click.echo(click.style("- Talk to Content", fg="yellow")) click.echo(click.style("- User Event Collection", fg="yellow")) click.echo(click.style("- User Feedback", fg="yellow")) configure_gemini_enterprise_for_fedramp_high(credentials, project_id, engine_id) @@ -458,10 +473,25 @@ def onboard(): click.echo(click.style("- NotebookLM Enterprise", fg="yellow")) click.echo(click.style("- Prompt Gallery", fg="yellow")) click.echo(click.style("- Session Sharing", fg="yellow")) - click.echo(click.style("- Talk to Content", fg="yellow")) click.echo(click.style("- User Event Collection", fg="yellow")) click.echo(click.style("- User Feedback", fg="yellow")) configure_gemini_enterprise_for_il4(credentials, project_id, engine_id) + elif compliance_regime_id == '3': + click.echo(click.style("Gemini Enterprise contains default features that are not yet authorized for IL5 and must be disabled. These features are currently:", fg="yellow")) + click.echo(click.style("- Grounding with OneDrive / Google Drive File Uploads", fg="yellow")) + click.echo(click.style("- Grounding with Google Search", fg="yellow")) + click.echo(click.style("- Image / Video Generation", fg="yellow")) + click.echo(click.style("- Implicit Model Data Caching", fg="yellow")) + click.echo(click.style("- Knowledge Graph / People Connectors", fg="yellow")) + click.echo(click.style("- Location Context", fg="yellow")) + click.echo(click.style("- Memory and Customization", fg="yellow")) + click.echo(click.style("- Model Armor", fg="yellow")) + click.echo(click.style("- NotebookLM Enterprise", fg="yellow")) + click.echo(click.style("- Prompt Gallery", fg="yellow")) + click.echo(click.style("- Session Sharing", fg="yellow")) + click.echo(click.style("- User Event Collection", fg="yellow")) + click.echo(click.style("- User Feedback", fg="yellow")) + configure_gemini_enterprise_for_il5(credentials, project_id, engine_id) click.echo(nl=True) click.echo(nl=True) @@ -504,11 +534,15 @@ def app(): @app.command("create") @click.option('--project-id', required=True, help='GCP Project ID') +@click.option('--engine-id', default=None, help='Gemini Enterprise Engine ID') +@click.option('--display-name', default=None, help='Display Name for the Gemini Enterprise application') +@click.option('--company-name', default=None, help='Agency / Department Name') @click.option('--data-stores', default="", help='Comma-separated list of Data Store IDs') @click.option('--workforce-pool-id', default=None, help='Workforce Identity Pool ID') @click.option('--workforce-provider-id', default=None, help='Workforce Identity Provider ID') -@click.option('--compliance-regime', type=click.Choice(['FEDRAMP_HIGH', 'IL4', 'NONE']), default=None, help='Compliance Regime') -def create_application(project_id, data_stores, workforce_pool_id, workforce_provider_id, compliance_regime): +@click.option('--compliance-regime', type=click.Choice(['FEDRAMP_HIGH', 'IL4', 'IL5', 'NONE']), default=None, help='Compliance Regime') +@click.option('--enable-audit-logs', is_flag=True, default=False, help='Enable Gemini Enterprise Usage Audit logs') +def create_application(project_id, engine_id, display_name, company_name, data_stores, workforce_pool_id, workforce_provider_id, compliance_regime, enable_audit_logs): """Creates a Gemini Enterprise application.""" credentials = get_credentials() # split comma separated string into list @@ -520,16 +554,18 @@ def create_application(project_id, data_stores, workforce_pool_id, workforce_pro compliance_regime_id = '1' elif compliance_regime == 'IL4': compliance_regime_id = '2' - elif compliance_regime == 'NONE': + elif compliance_regime == 'IL5': compliance_regime_id = '3' + elif compliance_regime == 'NONE': + compliance_regime_id = '4' - create_application_logic(credentials, project_id, data_store_list, workforce_pool_id, workforce_provider_id, compliance_regime_id) + create_application_logic(credentials, project_id, data_store_list, workforce_pool_id, workforce_provider_id, compliance_regime_id, engine_id, display_name, company_name, enable_audit_logs) @app.command("update-compliance") @click.option('--project-id', required=True, help='GCP Project ID') @click.option('--engine-id', required=True, help='Gemini Enterprise Engine ID') -@click.option('--compliance-regime', required=True, type=click.Choice(['FEDRAMP_HIGH', 'IL4']), help='Compliance Regime') +@click.option('--compliance-regime', required=True, type=click.Choice(['FEDRAMP_HIGH', 'IL4', 'IL5']), help='Compliance Regime') def update_compliance(project_id, engine_id, compliance_regime): """Configures a Gemini Enterprise application for a specific compliance regime.""" credentials = get_credentials() @@ -547,7 +583,6 @@ def update_compliance(project_id, engine_id, compliance_regime): click.echo(click.style("- NotebookLM Enterprise", fg="yellow")) click.echo(click.style("- Prompt Gallery", fg="yellow")) click.echo(click.style("- Session Sharing", fg="yellow")) - click.echo(click.style("- Talk to Content", fg="yellow")) click.echo(click.style("- User Event Collection", fg="yellow")) click.echo(click.style("- User Feedback", fg="yellow")) configure_gemini_enterprise_for_fedramp_high(credentials, project_id, engine_id) @@ -564,10 +599,25 @@ def update_compliance(project_id, engine_id, compliance_regime): click.echo(click.style("- NotebookLM Enterprise", fg="yellow")) click.echo(click.style("- Prompt Gallery", fg="yellow")) click.echo(click.style("- Session Sharing", fg="yellow")) - click.echo(click.style("- Talk to Content", fg="yellow")) click.echo(click.style("- User Event Collection", fg="yellow")) click.echo(click.style("- User Feedback", fg="yellow")) configure_gemini_enterprise_for_il4(credentials, project_id, engine_id) + elif compliance_regime == 'IL5': + click.echo(click.style("Gemini Enterprise contains default features that are not yet authorized for IL5 and must be disabled. These features are currently:", fg="yellow")) + click.echo(click.style("- Grounding with OneDrive / Google Drive File Uploads", fg="yellow")) + click.echo(click.style("- Grounding with Google Search", fg="yellow")) + click.echo(click.style("- Image / Video Generation", fg="yellow")) + click.echo(click.style("- Implicit Model Data Caching", fg="yellow")) + click.echo(click.style("- Knowledge Graph / People Connectors", fg="yellow")) + click.echo(click.style("- Location Context", fg="yellow")) + click.echo(click.style("- Memory and Customization", fg="yellow")) + click.echo(click.style("- Model Armor", fg="yellow")) + click.echo(click.style("- NotebookLM Enterprise", fg="yellow")) + click.echo(click.style("- Prompt Gallery", fg="yellow")) + click.echo(click.style("- Session Sharing", fg="yellow")) + click.echo(click.style("- User Event Collection", fg="yellow")) + click.echo(click.style("- User Feedback", fg="yellow")) + configure_gemini_enterprise_for_il5(credentials, project_id, engine_id) click.echo(click.style("Compliance configuration complete!", fg='green')) @@ -601,17 +651,238 @@ def datastore(): @click.option('--project-id', required=True, help='GCP Project ID') @click.option('--source-type', required=True, type=click.Choice(['gcs', 'bigquery']), help='Source of the documents to import') @click.option('--data-store-id', required=False, help='Gemini Enterprise Data Store ID') -def import_documents(project_id, source_type, data_store_id): +@click.option('--gcs-bucket', required=False, help='Optional GCS Bucket name to simplify the prompt') +def import_documents(project_id, source_type, data_store_id, gcs_bucket): """Import documents into a Gemini Enterprise data store.""" credentials = get_credentials() # Set quota project credentials = credentials.with_quota_project(project_id) - import_documents_helper(credentials, project_id, source_type, data_store_id) + import_documents_helper(credentials, project_id, source_type, data_store_id, gcs_bucket) + +############################################################## +################ gem4gov license ################ +############################################################## + +@cli.group() +def license(): + """Manages Gemini for Government licenses.""" + pass + +@license.command(name='list') +@click.option('--billing-account', required=True, help='The billing account ID.') +@click.option('--quota-project', required=False, help='The project ID to use for API quota.') +@click.option('--format', type=click.Choice(['text', 'json']), default='text', help='The output format.') +def list_licenses(billing_account, quota_project, format): + """Lists available Gemini for Government license configurations for a billing account.""" + credentials = get_credentials() + if quota_project: + credentials = credentials.with_quota_project(quota_project) + + # Use discoveryengine v1alpha as per PDF + client_options = ClientOptions(api_endpoint="https://us-discoveryengine.googleapis.com") + service = build('discoveryengine', 'v1alpha', credentials=credentials, client_options=client_options) + + try: + request = service.billingAccounts().billingAccountLicenseConfigs().list( + parent=f'billingAccounts/{billing_account}' + ) + response = request.execute() + + configs = response.get('billingAccountLicenseConfigs', []) + + if format == 'json': + click.echo(json.dumps(configs, indent=2)) + return + + if not configs: + click.echo(f"No license configurations found for billing account {billing_account}.") + return + + for config in configs: + name = config.get('subscriptionDisplayName', config.get('name')) + total = config.get('licenseCount', 0) + distributions = config.get('licenseConfigDistributions', {}) + distributed = sum(int(v) for v in distributions.values()) + available = int(total) - distributed + + # Extract ID from name: billingAccounts/ID/billingAccountLicenseConfigs/CONFIG_ID + config_id = config.get('name').split('/')[-1] + + click.echo(f"Subscription: {name}") + click.echo(f" ID: {config_id}") + click.echo(f" Total Licenses: {total}") + click.echo(f" Distributed: {distributed}") + click.echo(f" Available: {available}") + click.echo("---") + + except HttpError as e: + click.echo(f"An error occurred: {e}") + except Exception as e: + click.echo(f"An unexpected error occurred: {e}") + +@license.command(name='distribute') +@click.option('--billing-account', required=True, help='The billing account ID.') +@click.option('--config-id', required=True, help='The billing account license config ID.') +@click.option('--target-project-number', required=True, help='The target project number.') +@click.option('--location', default='global', type=click.Choice(['global', 'us', 'eu']), help='The location.') +@click.option('--count', required=True, type=int, help='The number of licenses to distribute (incremental).') +@click.option('--license-config-id', help='The existing project-level license config ID (optional).') +@click.option('--quota-project', required=False, help='The project ID to use for API quota.') +def distribute_licenses(billing_account, config_id, target_project_number, location, count, license_config_id, quota_project): + """Distributes Gemini for Government licenses to a project.""" + credentials = get_credentials() + if quota_project: + credentials = credentials.with_quota_project(quota_project) + + endpoint = "https://discoveryengine.googleapis.com" + if location == 'us': + endpoint = "https://us-discoveryengine.googleapis.com" + elif location == 'eu': + endpoint = "https://eu-discoveryengine.googleapis.com" + + client_options = ClientOptions(api_endpoint=endpoint) + # v1alpha is needed for billingAccountLicenseConfigs + service = build('discoveryengine', 'v1alpha', credentials=credentials, client_options=client_options) + + name = f'billingAccounts/{billing_account}/billingAccountLicenseConfigs/{config_id}' + + body = { + "projectNumber": target_project_number, + "location": location, + "licenseCount": count + } + if license_config_id: + body["licenseConfigId"] = license_config_id + + try: + request = service.billingAccounts().billingAccountLicenseConfigs().distributeLicenseConfig( + name=name, + body=body + ) + response = request.execute() + click.echo("Licenses distributed successfully!") + click.echo(json.dumps(response, indent=2)) + + except HttpError as e: + click.echo(f"An error occurred: {e}") + except Exception as e: + click.echo(f"An unexpected error occurred: {e}") +############################################################## +################ gem4gov license ################ +############################################################## + +@cli.group() +def license(): + """Manages Gemini for Government licenses.""" + pass + +@license.command(name='list') +@click.option('--billing-account', required=True, help='The billing account ID.') +@click.option('--quota-project', required=False, help='The project ID to use for API quota.') +@click.option('--format', type=click.Choice(['text', 'json']), default='text', help='The output format.') +def list_licenses(billing_account, quota_project, format): + """Lists available Gemini for Government license configurations for a billing account.""" + credentials = get_credentials() + if quota_project: + credentials = credentials.with_quota_project(quota_project) + + # Use discoveryengine v1alpha as per PDF + client_options = ClientOptions(api_endpoint="https://us-discoveryengine.googleapis.com") + service = build('discoveryengine', 'v1alpha', credentials=credentials, client_options=client_options) + + try: + request = service.billingAccounts().billingAccountLicenseConfigs().list( + parent=f'billingAccounts/{billing_account}' + ) + response = request.execute() + + configs = response.get('billingAccountLicenseConfigs', []) + + if format == 'json': + click.echo(json.dumps(configs, indent=2)) + return + + if not configs: + click.echo(f"No license configurations found for billing account {billing_account}.") + return -def import_documents_helper(credentials, project_id, source_type, data_store_id=None): + for config in configs: + name = config.get('subscriptionDisplayName', config.get('name')) + total = config.get('licenseCount', 0) + distributions = config.get('licenseConfigDistributions', {}) + distributed = sum(int(v) for v in distributions.values()) + available = int(total) - distributed + + # Extract ID from name: billingAccounts/ID/billingAccountLicenseConfigs/CONFIG_ID + config_id = config.get('name').split('/')[-1] + + click.echo(f"Subscription: {name}") + click.echo(f" ID: {config_id}") + click.echo(f" Total Licenses: {total}") + click.echo(f" Distributed: {distributed}") + click.echo(f" Available: {available}") + click.echo("---") + + except HttpError as e: + click.echo(f"An error occurred: {e}") + except Exception as e: + click.echo(f"An unexpected error occurred: {e}") + +@license.command(name='distribute') +@click.option('--billing-account', required=True, help='The billing account ID.') +@click.option('--config-id', required=True, help='The billing account license config ID.') +@click.option('--target-project-number', required=True, help='The target project number.') +@click.option('--location', default='global', type=click.Choice(['global', 'us', 'eu']), help='The location.') +@click.option('--count', required=True, type=int, help='The number of licenses to distribute (incremental).') +@click.option('--license-config-id', help='The existing project-level license config ID (optional).') +@click.option('--quota-project', required=False, help='The project ID to use for API quota.') +def distribute_licenses(billing_account, config_id, target_project_number, location, count, license_config_id, quota_project): + """Distributes Gemini for Government licenses to a project.""" + credentials = get_credentials() + if quota_project: + credentials = credentials.with_quota_project(quota_project) + + endpoint = "https://discoveryengine.googleapis.com" + if location == 'us': + endpoint = "https://us-discoveryengine.googleapis.com" + elif location == 'eu': + endpoint = "https://eu-discoveryengine.googleapis.com" + + client_options = ClientOptions(api_endpoint=endpoint) + # v1alpha is needed for billingAccountLicenseConfigs + service = build('discoveryengine', 'v1alpha', credentials=credentials, client_options=client_options) + + name = f'billingAccounts/{billing_account}/billingAccountLicenseConfigs/{config_id}' + + body = { + "projectNumber": target_project_number, + "location": location, + "licenseCount": count + } + if license_config_id: + body["licenseConfigId"] = license_config_id + + try: + request = service.billingAccounts().billingAccountLicenseConfigs().distributeLicenseConfig( + name=name, + body=body + ) + response = request.execute() + click.echo("Licenses distributed successfully!") + click.echo(json.dumps(response, indent=2)) + + except HttpError as e: + click.echo(f"An error occurred: {e}") + except Exception as e: + click.echo(f"An unexpected error occurred: {e}") + + + + +def import_documents_helper(credentials, project_id, source_type, data_store_id=None, gcs_bucket=None): """Helper to import documents into a selected data store.""" if not data_store_id: click.echo(nl=True) @@ -637,17 +908,23 @@ def import_documents_helper(credentials, project_id, source_type, data_store_id= if source_type == 'gcs': # GCS Data Store click.echo(click.style("Importing from Google Cloud Storage.", fg='green')) - gcs_uri = click.prompt('Please enter the GCS URI to the documents (e.g., gs://my-bucket/path/to/docs)', type=str).strip() - if not gcs_uri.startswith("gs://"): - click.echo(click.style("Invalid URI. Must start with 'gs://'.", fg='red')) - return - - # Parse bucket and prefix - # gs://bucket/prefix... - parts = gcs_uri[5:].split('/', 1) - bucket_name = parts[0] - prefix = parts[1] if len(parts) > 1 else "" + if gcs_bucket: + relative_path = click.prompt(f'Please enter the path to the documents relative to the root of gs://{gcs_bucket}/ (leave blank for root)', type=str, default="").strip() + bucket_name = gcs_bucket + prefix = relative_path + else: + gcs_uri = click.prompt('Please enter the GCS URI to the documents (e.g., gs://my-bucket/path/to/docs)', type=str).strip() + + if not gcs_uri.startswith("gs://"): + click.echo(click.style("Invalid URI. Must start with 'gs://'.", fg='red')) + return + + # Parse bucket and prefix + # gs://bucket/prefix... + parts = gcs_uri[5:].split('/', 1) + bucket_name = parts[0] + prefix = parts[1] if len(parts) > 1 else "" # Let's clean the prefix prefix = prefix.strip('/') @@ -667,47 +944,45 @@ def import_documents_helper(credentials, project_id, source_type, data_store_id= click.echo("Please use the 'onboard' command for BigQuery data store creation and initial import.") -def create_application_logic(credentials, project_id, data_store_list, workforce_pool_id, workforce_provider_id, compliance_regime=None): +def create_application_logic(credentials, project_id, data_store_list, workforce_pool_id, workforce_provider_id, compliance_regime=None, engine_id=None, engine_display_name=None, company_name=None, enable_audit_logs=False): """Shared logic for creating a Gemini Enterprise application.""" - engine_display_name = click.prompt('Please enter a Display Name for the Gemini Enterprise application').strip() - company_name = click.prompt('Please enter the Agency / Department Name (no abbreviations)').strip() - click.echo(nl=True) - engine_id = generate_id('g4g-gem-ent-app-' + ''.join(random.choices(string.ascii_lowercase + string.digits, k=4))) + if not engine_display_name: + engine_display_name = click.prompt('Please enter a Display Name for the Gemini Enterprise application').strip() + if not company_name: + company_name = click.prompt('Please enter the Agency / Department Name (no abbreviations)').strip() - create_engine(credentials, project_id, engine_id, engine_display_name, company_name, data_store_list) + if not engine_id: + click.echo(nl=True) + import random + import string + engine_id = 'g4g-gem-ent-app-' + ''.join(random.choices(string.ascii_lowercase + string.digits, k=4)) + + create_engine(credentials, project_id, engine_id, engine_display_name, company_name, data_store_list, enable_audit_logs) if workforce_pool_id and workforce_provider_id: configure_idp_for_widget(credentials, project_id, engine_id, workforce_pool_id, workforce_provider_id) - # Check if we need to prompt for workforce identity if it wasn't provided but might be needed - # The CLI command argument is optional, so if not provided, we check IDP type like onboard does. - # However, 'onboard' does a specific check. For the 'create' command, we assume arguments provided are final. - # But since we are sharing logic, we should handle the 'onboard' flow's dynamic prompting if needed, - # OR 'onboard' should gather everything before calling this. - # Let's assume 'onboard' gathers everything. - - # The 'onboard' command had logic to check regulatory boundary and configure FedRAMP/IL4. - # The 'create' command should also do this? - # The user request says "configure_gemini_enterprise_for_fedramp" should be run. - # We can ask the user here or assume default. Since 'onboard' asks, let's ask here if not passed? - # But the refactor request didn't specify a 'boundary' argument. - # Let's ask the user for the boundary as part of the application creation process if it's not contextually available. - # Re-using the prompt from onboard for consistency if not compliance_regime: click.echo(nl=True) click.echo("What compliance regime will this application be deployed in?") click.echo("1) FedRAMP High") click.echo("2) IL4") - click.echo("3) None") - compliance_regime = click.prompt('Please enter the number for your response', type=click.Choice(['1', '2', '3']), default = '1', show_default = False) + click.echo("3) IL5") + click.echo("4) None") + compliance_regime = click.prompt('Please enter the number for your response', type=click.Choice(['1', '2', '3', '4']), default = '1', show_default = False) - if compliance_regime == '1': + if compliance_regime in ['1', 'FEDRAMP_HIGH']: click.echo(click.style("Configuring for FedRAMP High...", fg="yellow")) configure_gemini_enterprise_for_fedramp_high(credentials, project_id, engine_id) - elif compliance_regime == '2': + elif compliance_regime in ['2', 'IL4']: click.echo(click.style("Configuring for IL4...", fg="yellow")) configure_gemini_enterprise_for_il4(credentials, project_id, engine_id) + elif compliance_regime in ['3', 'IL5']: + click.echo(click.style("Configuring for IL5...", fg="yellow")) + configure_gemini_enterprise_for_il5(credentials, project_id, engine_id) + elif compliance_regime in ['4', 'NONE']: + click.echo(click.style("Skipping compliance-specific app configuration...", fg="yellow")) click.echo(nl=True) @@ -793,7 +1068,7 @@ def check_apis(credentials, project_id): else: click.echo("Exiting. Please enable the missing APIs and re-run the script.") click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) click.echo("All required APIs are enabled.") @@ -816,7 +1091,7 @@ def enable_apis(credentials, project_id, apis_to_enable): click.echo(f"An error occurred while enabling {api}: {e}") click.echo("Please try enabling the APIs manually and re-run the script.") click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) def check_identity_provider(credentials, project_id): @@ -884,7 +1159,7 @@ def configure_identity_provider(credentials, project_id, idp_type, workforce_poo except Exception as e: click.echo(f"An error occurred while configuring the identity provider: {e}") click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) def check_cmek(credentials, project_id): @@ -1015,10 +1290,10 @@ def configure_cmek(credentials, project_id, kms_key_name): except Exception as e: click.echo(f"An error occurred while configuring CMEK: {e}") click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) -def create_engine(credentials, project_id, engine_id, display_name, company_name, data_store_list): +def create_engine(credentials, project_id, engine_id, display_name, company_name, data_store_list, enable_audit_logs=False): """Creates a new engine.""" client_options = ClientOptions(api_endpoint="https://us-discoveryengine.googleapis.com") service = build('discoveryengine', 'v1alpha', credentials=credentials, client_options=client_options) @@ -1035,7 +1310,6 @@ def create_engine(credentials, project_id, engine_id, display_name, company_name engine = { "displayName": display_name, "appType": "APP_TYPE_INTRANET", - "disableAnalytics": True, "solutionType": "SOLUTION_TYPE_SEARCH", "searchEngineConfig": { "searchTier": "SEARCH_TIER_ENTERPRISE", @@ -1044,12 +1318,24 @@ def create_engine(credentials, project_id, engine_id, display_name, company_name }, "features": engine_features.get('features'), "industryVertical": "GENERIC", + "disableAnalytics": True, "commonConfig": { "companyName": company_name }, + "sessionConfig": { + "sessionManagementPolicy": "VERTEX_AI_MANAGED" + }, + "disableCmekChanges": True, + "dataStores": [], "dataStoreIds": [] } + if enable_audit_logs: + engine["observabilityConfig"] = { + "observabilityEnabled": True, + "sensitiveLoggingEnabled": True + } + if data_store_list: engine['dataStoreIds'] = data_store_list else: @@ -1092,9 +1378,23 @@ def create_engine(credentials, project_id, engine_id, display_name, company_name click.echo(f"Engine created successfully!") except Exception as e: - click.echo(f"An error occurred while creating the engine: {e}") - click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + click.echo("Received an API error during creation. Checking if engine was created asynchronously despite the error...") + engine_full_name = f"projects/{project_id}/locations/us/collections/default_collection/engines/{engine_id}" + max_retries = 12 + for attempt in range(max_retries): + try: + eng_request = service.projects().locations().collections().engines().get(name=engine_full_name) + eng_response = eng_request.execute() + click.echo("Engine verified successfully! Proceeding with configuration.") + return + except Exception as inner_e: + if attempt < max_retries - 1: + click.echo(".", nl=False) + time.sleep(5) + else: + click.echo(f"\nAn error occurred while creating the engine: {e}") + click.echo(click.style("Exiting Onboarding process...", fg="red")) + sys.exit(1) def configure_idp_for_widget(credentials, project_id, engine_id, workforce_pool_id, workforce_provider_id): @@ -1269,27 +1569,26 @@ def configure_gemini_enterprise_for_fedramp_high(credentials, project_id, engine access_token = token_process.stdout.strip() except subprocess.CalledProcessError as e: click.echo(f"Error getting access token not critical, but noted: {e}") - # We might not be able to proceed with curl if token fails, but let's try to continue or just return - # If we can't get a token, we can't do the rest. - # But user said "gracefully log... but continue". - # Continuing without a token will just fail the next step. - # I'll let it fail naturally or just return from this function logic? - # Actually proper "continue" means try the next steps. - # If token fails, curl calls WILL fail. pass access_token = "" if access_token: - url = f"https://us-discoveryengine.googleapis.com/v1alpha/{assistant_name}?updateMask=generationConfig.defaultLanguage,webGroundingType,defaultWebGroundingToggleOff,enableEndUserAgentCreation,disableLocationContext" + url = f"https://us-discoveryengine.googleapis.com/v1alpha/{assistant_name}?updateMask=customerPolicy,agentConfigs,generationConfig,disableLocationContext,webGroundingType,defaultWebGroundingToggleOff" assistant_patch_body = { - "generationConfig": { - "defaultLanguage": "en" - }, - "webGroundingType": "WEB_GROUNDING_TYPE_ENTERPRISE_WEB_SEARCH", - "defaultWebGroundingToggleOff": False, - "enableEndUserAgentCreation": False, - "disableLocationContext": True + "displayName":"Default Assistant", + "googleSearchGroundingEnabled": False, + "webGroundingType":"WEB_GROUNDING_TYPE_ENTERPRISE_WEB_SEARCH", + "customerPolicy":{ + "bannedPhrases":[] + }, + "generationConfig":{ + "systemInstruction":{ + "additionalSystemInstruction":"" + } + }, + "defaultWebGroundingToggleOff": False, + "disableLocationContext": True } # Use subprocess to run the curl command @@ -1369,11 +1668,11 @@ def configure_gemini_enterprise_for_il4(credentials, project_id, engine_id): try: engine_response = engine_request.execute() - click.echo(f"Engine {engine_id} configured for FedRAMP High.") + click.echo(f"Engine {engine_id} configured for IL4.") except Exception as e: - click.echo(f"An error occurred while configuring the engine for FedRAMP High: {e}") + click.echo(f"An error occurred while configuring the engine for IL4: {e}") click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) # Default Search Widget: Disable User Event Collection disable_user_event_collection(credentials, project_id, engine_id) @@ -1388,7 +1687,7 @@ def configure_gemini_enterprise_for_il4(credentials, project_id, engine_id): except subprocess.CalledProcessError as e: click.echo(f"Error getting access token: {e}") click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) url = f"https://us-discoveryengine.googleapis.com/v1alpha/{assistant_name}?updateMask=generationConfig.defaultLanguage,webGroundingType,defaultWebGroundingToggleOff,enableEndUserAgentCreation,disableLocationContext" @@ -1422,12 +1721,124 @@ def configure_gemini_enterprise_for_il4(credentials, project_id, engine_id): click.echo(result.stderr) click.echo(result.stdout) click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) except Exception as e: click.echo(f"An error occurred while configuring the default assistant for IL4: {e}") click.echo(click.style("Exiting Onboarding process...", fg="red")) - exit() + sys.exit(1) + + # Project: Disable Implicit Model Caching + try: + aiplatform_client_options = ClientOptions(api_endpoint="https://us-central1-aiplatform.googleapis.com") + aiplatform_service = build('aiplatform', 'v1', credentials=credentials, client_options=aiplatform_client_options) + + cache_config_name = f"projects/{project_id}/cacheConfig" + cache_config_body = { + "name": cache_config_name, + "disableCache": True + } + + request = aiplatform_service.projects().updateCacheConfig( + name=cache_config_name, + body=cache_config_body + ) + request.execute() + click.echo("Successfully disabled Implicit Model Caching for the project.") + except Exception as e: + click.echo(f"An error occurred while disabling Implicit Model Caching: {e}") + # Do not exit, as this may not be a critical failure. + + +def configure_gemini_enterprise_for_il5(credentials, project_id, engine_id): + """Configures the Gemini Enterprise engine and default assistant for IL5.""" + client_options = ClientOptions(api_endpoint="https://us-discoveryengine.googleapis.com") + service = build('discoveryengine', 'v1alpha', credentials=credentials, client_options=client_options) + + # Get the absolute path to the directory containing the script + script_dir = os.path.dirname(os.path.abspath(__file__)) + # Construct the absolute path to the YAML file + yaml_path = os.path.join(script_dir, 'engine_features.yaml') + + # Load features from the YAML file + with open(yaml_path, 'r') as f: + engine_features = yaml.safe_load(f) + + # Engine: Update IL5 authorized features and disable Private Knowledge Graph (People Connectors are not yet authorized for IL5) + engine_name = f"projects/{project_id}/locations/us/collections/default_collection/engines/{engine_id}" + engine_patch_body = { + "features": engine_features.get('features'), + "disableAnalytics": True + } + engine_update_mask = "features" + + engine_request = service.projects().locations().collections().engines().patch( + name=engine_name, + body=engine_patch_body, + updateMask=engine_update_mask + ) + + try: + engine_response = engine_request.execute() + click.echo(f"Engine {engine_id} configured for IL5.") + except Exception as e: + click.echo(f"An error occurred while configuring the engine for IL5: {e}") + click.echo(click.style("Exiting Onboarding process...", fg="red")) + sys.exit(1) + + # Default Search Widget: Disable User Event Collection + disable_user_event_collection(credentials, project_id, engine_id) + + # Assistant: Disable Grounding with Google Search / Location Context + assistant_name = f"projects/{project_id}/locations/us/collections/default_collection/engines/{engine_id}/assistants/default_assistant" + + # Get access token + try: + token_process = subprocess.run(['gcloud', 'auth', 'print-access-token'], check=True, capture_output=True, text=True) + access_token = token_process.stdout.strip() + except subprocess.CalledProcessError as e: + click.echo(f"Error getting access token: {e}") + click.echo(click.style("Exiting Onboarding process...", fg="red")) + sys.exit(1) + + url = f"https://us-discoveryengine.googleapis.com/v1alpha/{assistant_name}?updateMask=generationConfig.defaultLanguage,webGroundingType,defaultWebGroundingToggleOff,enableEndUserAgentCreation,disableLocationContext" + + assistant_patch_body = { + "generationConfig": { + "defaultLanguage": "en" + }, + "webGroundingType": "WEB_GROUNDING_TYPE_ENTERPRISE_WEB_SEARCH", + "defaultWebGroundingToggleOff": False, + "enableEndUserAgentCreation": False, + "disableLocationContext": True + } + + # Use subprocess to run the curl command + curl_command = [ + 'curl', '-X', 'PATCH', + '-H', f"Authorization: Bearer {access_token}", + '-H', f"x-goog-user-project: {project_id}", + '-H', "Content-Type: application/json", + '-d', json.dumps(assistant_patch_body), + url + ] + + try: + result = subprocess.run(curl_command, capture_output=True, text=True) + + if result.returncode == 0 and "error" not in result.stdout.lower(): + click.echo(f"Default assistant for engine {engine_id} configured for IL5.") + else: + click.echo(f"An error occurred while configuring the default assistant for IL5:") + click.echo(result.stderr) + click.echo(result.stdout) + click.echo(click.style("Exiting Onboarding process...", fg="red")) + sys.exit(1) + + except Exception as e: + click.echo(f"An error occurred while configuring the default assistant for IL5: {e}") + click.echo(click.style("Exiting Onboarding process...", fg="red")) + sys.exit(1) # Project: Disable Implicit Model Caching try: diff --git a/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/requirements.txt b/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/requirements.txt index fb91f359d..50230ab2a 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/requirements.txt +++ b/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/requirements.txt @@ -1,3 +1,16 @@ +# Copyright 2026 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. click google-api-python-client google-auth diff --git a/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/setup.py b/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/setup.py index 04826f3e4..e82adeaec 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/setup.py +++ b/blueprints/fedramp-high/gemini-enterprise/gem4gov-cli/setup.py @@ -1,3 +1,17 @@ +# Copyright 2026 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from setuptools import setup, find_packages setup( diff --git a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/analytics.tf b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/analytics.tf new file mode 100644 index 000000000..363369ccc --- /dev/null +++ b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/analytics.tf @@ -0,0 +1,53 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "google_project_iam_audit_config" "discovery_engine_audit" { + count = var.enable_analytics ? 1 : 0 + project = var.main_project_id + service = "discoveryengine.googleapis.com" + audit_log_config { + log_type = "DATA_READ" + } + audit_log_config { + log_type = "DATA_WRITE" + } + audit_log_config { + log_type = "ADMIN_READ" + } +} + +resource "google_bigquery_dataset" "analytics_dataset" { + count = var.enable_analytics ? 1 : 0 + dataset_id = "${replace(var.prefix, "-", "_")}_gemini_analytics" + project = var.main_project_id + location = var.geolocation + description = "Dataset for Gemini Enterprise Discovery Engine audit logs" +} + +resource "google_logging_project_sink" "discovery_engine_sink" { + count = var.enable_analytics ? 1 : 0 + name = "${var.prefix}-discovery-engine-analytics-sink" + project = var.main_project_id + destination = "bigquery.googleapis.com/${google_bigquery_dataset.analytics_dataset[0].id}" + filter = "protoPayload.serviceName=\"discoveryengine.googleapis.com\"" + unique_writer_identity = true +} + +resource "google_bigquery_dataset_iam_member" "sink_bq_editor" { + count = var.enable_analytics ? 1 : 0 + dataset_id = google_bigquery_dataset.analytics_dataset[0].dataset_id + project = var.main_project_id + role = "roles/bigquery.dataEditor" + member = google_logging_project_sink.discovery_engine_sink[0].writer_identity +} diff --git a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/cloudarmor.tf b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/cloudarmor.tf index 4266b603a..4bd820156 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/cloudarmor.tf +++ b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/cloudarmor.tf @@ -17,6 +17,7 @@ locals { } resource "google_compute_region_security_policy" "gemini_enterprise_policy" { + count = var.deployment_type != "none" ? 1 : 0 provider = google-beta project = var.main_project_id region = var.region diff --git a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/cmek.tf b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/cmek.tf index 30739cabd..ceba27071 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/cmek.tf +++ b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/cmek.tf @@ -13,73 +13,17 @@ # limitations under the License. locals { - kms_rotation_period = "7776000s" # 90 days - - # Determine if we have a US Keyring - has_us_keyring = var.us_keyring_name != "" - - # ID of the KeyRing to use (Created or Existing) - keyring_id = local.has_us_keyring ? data.google_kms_key_ring.existing[0].id : google_kms_key_ring.created[0].id - - # Determine if we need to create a new Key - create_key = var.kms_key_id == "" - - # Final CMEK Key ID - cmek_key_id = local.create_key ? google_kms_crypto_key.gemini_enterprise[0].id : var.kms_key_id -} - -# ---------------------------------------------------------------------------- # -# 1. KeyRing (US Multi-Region) -# ---------------------------------------------------------------------------- # - -# Create KeyRing if name not provided -resource "google_kms_key_ring" "created" { - count = local.has_us_keyring ? 0 : 1 - name = "${title(var.environment)}-${var.tenant}-keyring" - location = "us" - project = var.kms_project_id -} - -# Read KeyRing if name provided -data "google_kms_key_ring" "existing" { - count = local.has_us_keyring ? 1 : 0 - name = basename(var.us_keyring_name) - location = "us" - project = var.kms_project_id -} - -# ---------------------------------------------------------------------------- # -# 2. Crypto Key (Gemini Enterprise) -# ---------------------------------------------------------------------------- # - -# Create Key if kms_key_id is not provided -resource "google_kms_crypto_key" "gemini_enterprise" { - count = local.create_key ? 1 : 0 - name = "gemini-enterprise" - key_ring = local.keyring_id - purpose = "ENCRYPT_DECRYPT" - rotation_period = local.kms_rotation_period - - version_template { - algorithm = "GOOGLE_SYMMETRIC_ENCRYPTION" - protection_level = "HSM" - } -} - -# If NOT creating keys (kms_key_id IS provided) -# Usage for IAM binding validation/lookup if needed: -data "google_kms_crypto_key" "provided" { - count = local.create_key ? 0 : 1 - name = basename(var.kms_key_id) - key_ring = element(split("/cryptoKeys/", var.kms_key_id), 0) + # Final CMEK Key ID - Only explicitly set if we are enabling Data Store CMEK + cmek_key_id = var.enable_data_store_cmek ? var.kms_key_id : null } # ---------------------------------------------------------------------------- # -# 3. IAM Bindings +# 1. IAM Bindings # ---------------------------------------------------------------------------- # # Grant Discovery Engine Service Agent access resource "google_kms_crypto_key_iam_member" "discoveryengine_sa_kms_access" { + count = var.enable_data_store_cmek ? 1 : 0 crypto_key_id = local.cmek_key_id role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-discoveryengine.iam.gserviceaccount.com" @@ -92,6 +36,7 @@ resource "google_kms_crypto_key_iam_member" "discoveryengine_sa_kms_access" { # Grant Cloud Storage Service Agent access resource "google_kms_crypto_key_iam_member" "gcs_sa_kms_access" { + count = var.enable_data_store_cmek ? 1 : 0 crypto_key_id = local.cmek_key_id role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" member = "serviceAccount:service-${data.google_project.project.number}@gs-project-accounts.iam.gserviceaccount.com" @@ -104,6 +49,7 @@ resource "google_kms_crypto_key_iam_member" "gcs_sa_kms_access" { # Grant BigQuery Service Agent access resource "google_kms_crypto_key_iam_member" "bq_sa_kms_access" { + count = var.enable_data_store_cmek ? 1 : 0 crypto_key_id = local.cmek_key_id role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" member = "serviceAccount:bq-${data.google_project.project.number}@bigquery-encryption.iam.gserviceaccount.com" @@ -112,4 +58,4 @@ resource "google_kms_crypto_key_iam_member" "bq_sa_kms_access" { google_project_service_identity.bigquery, time_sleep.wait_for_services ] -} \ No newline at end of file +} diff --git a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/discovery-engine.tf b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/discovery-engine.tf index 64b06fe3c..21dc2fa0a 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/discovery-engine.tf +++ b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/discovery-engine.tf @@ -21,50 +21,190 @@ locals { discovery_engine_industry_vertical = "GENERIC" discovery_engine_solution_types = ["SOLUTION_TYPE_SEARCH"] discovery_engine_content_config = "CONTENT_REQUIRED" - + # Document Processing (digital_parsing_config or ocr_parsing_config) - discovery_engine_parsing_mode = "digital_parsing_config" + discovery_engine_parsing_mode = "digital_parsing_config" } # ---------------------------------------------------------------------------- # -# Discovery Engine CMEK Config # +# Gemini Enterprise - Datastore CMEK Config # # ---------------------------------------------------------------------------- # # CMEK Configuration for Discovery Engine (Conditional) -resource "google_discovery_engine_cmek_config" "default" { - count = var.create_data_stores ? 1 : 0 +# resource "google_discovery_engine_cmek_config" "default" { +# count = var.create_data_stores && var.enable_data_store_cmek ? 1 : 0 + +# project = var.main_project_id +# location = var.geolocation # should be "US" +# cmek_config_id = "default_cmek_config" +# kms_key = local.cmek_key_id +# set_default = true +# provider = google-beta + +# depends_on = [ +# google_kms_crypto_key_iam_member.discoveryengine_sa_kms_access, +# google_kms_crypto_key_iam_member.gcs_sa_kms_access, +# google_project_service.services, +# time_sleep.wait_for_services, +# ] +# } - project = var.main_project_id - location = var.geolocation # should be "US" - cmek_config_id = "default_cmek_config" - kms_key = local.cmek_key_id - set_default = true - provider = google-beta +# ---------------------------------------------------------------------------- # +# Gemini Enterprise - Identity Config # +# ---------------------------------------------------------------------------- # + +# Discovery Engine ACL Config (Google Identity / Workforce Identity Federation) +resource "google_discovery_engine_acl_config" "gemini_enterprise_acl_config" { + project = var.main_project_id + location = var.geolocation # Must match the connector location + idp_config { + idp_type = var.acl_idp_type == "GOOGLE_CLOUD_IDENTITY" ? "GSUITE" : var.acl_idp_type + dynamic "external_idp_config" { + for_each = var.acl_idp_type == "THIRD_PARTY" ? [1] : [] + content { + workforce_pool_name = var.acl_workforce_pool_name + } + } + } + provider = google-beta depends_on = [ - google_kms_crypto_key_iam_member.discoveryengine_sa_kms_access, - google_kms_crypto_key_iam_member.gcs_sa_kms_access, - google_project_service.services, - time_sleep.wait_for_services, + time_sleep.wait_for_services ] } # ---------------------------------------------------------------------------- # -# Google Cloud Storage Data Stores # +# Gemini Enterprise - Application # +# ---------------------------------------------------------------------------- # + +# import { +# for_each = var.gemini_apps +# id = "projects/${var.main_project_id}/locations/${var.geolocation}/collections/default_collection/engines/${each.key}" +# to = google_discovery_engine_search_engine.gemini_enterprise_search_engine[each.key] +# } + +# resource "google_discovery_engine_search_engine" "gemini_enterprise_search_engine" { +# for_each = var.gemini_apps +# project = var.main_project_id +# engine_id = each.key +# collection_id = "default_collection" +# location = var.geolocation +# display_name = each.value.display_name +# data_store_ids = each.value.data_store_id != "" ? [ +# try( +# google_discovery_engine_data_store.gemini_enterprise_gcs_data_store[each.value.data_store_id].data_store_id, +# google_discovery_engine_data_store.gemini_enterprise_bq_data_store[each.value.data_store_id].data_store_id, +# each.value.data_store_id +# ) +# ] : [] +# industry_vertical = "GENERIC" +# app_type = "APP_TYPE_INTRANET" +# disable_analytics = true +# kms_key_name = var.enable_data_store_cmek ? local.cmek_key_id : null +# search_engine_config { +# search_tier = "SEARCH_TIER_ENTERPRISE" +# search_add_ons = [ +# "SEARCH_ADD_ON_LLM" +# ] +# } +# common_config { +# company_name = each.value.company_name +# } +# knowledge_graph_config {} +# features = { +# agent-gallery = "FEATURE_STATE_ON" +# no-code-agent-builder = "FEATURE_STATE_ON" +# prompt-gallery = "FEATURE_STATE_OFF" +# model-selector = "FEATURE_STATE_ON" +# notebook-lm = "FEATURE_STATE_OFF" +# people-search = "FEATURE_STATE_OFF" +# people-search-org-chart = "FEATURE_STATE_OFF" +# bi-directional-audio = "FEATURE_STATE_OFF" +# feedback = "FEATURE_STATE_OFF" +# session-sharing = "FEATURE_STATE_OFF" +# personalization-memory = "FEATURE_STATE_OFF" +# personalization-suggested-highlights = "FEATURE_STATE_OFF" +# disable-agent-sharing = "FEATURE_STATE_ON" +# agent-sharing-without-admin-approval = "FEATURE_STATE_OFF" +# disable-image-generation = "FEATURE_STATE_ON" +# disable-video-generation = "FEATURE_STATE_ON" +# disable-onedrive-upload = "FEATURE_STATE_ON" +# disable-talk-to-content = "FEATURE_STATE_OFF" +# disable-google-drive-upload = "FEATURE_STATE_ON" +# disable-welcome-emails = "FEATURE_STATE_OFF" +# } +# } + +# ---------------------------------------------------------------------------- # +# Gemini Enterprise - Default Assistant # +# ---------------------------------------------------------------------------- # + +# import { +# for_each = var.gemini_apps +# id = "projects/${var.main_project_id}/locations/${var.geolocation}/collections/default_collection/engines/${google_discovery_engine_search_engine.gemini_enterprise_search_engine[each.key].engine_id}/assistants/default_assistant" +# to = google_discovery_engine_assistant.gemini_enterprise_default_assistant[each.key] +# } + +# resource "google_discovery_engine_assistant" "gemini_enterprise_default_assistant" { +# for_each = var.gemini_apps +# project = var.main_project_id +# location = var.geolocation +# collection_id = "default_collection" +# engine_id = google_discovery_engine_search_engine.gemini_enterprise_search_engine[each.key].engine_id +# assistant_id = "default_assistant" +# display_name = "Gemini Enterprise Default Assistant" +# generation_config { +# default_language = "en" +# } +# web_grounding_type = "WEB_GROUNDING_TYPE_ENTERPRISE_WEB_SEARCH" +# } + +# ---------------------------------------------------------------------------- # +# Gemini Enterprise - Widget Config # +# ---------------------------------------------------------------------------- # + +# resource "google_discovery_engine_widget_config" "gemini_enterprise_widget_config" { +# for_each = var.gemini_apps +# project = var.main_project_id +# location = var.geolocation +# engine_id = google_discovery_engine_search_engine.gemini_enterprise_search_engine[each.key].engine_id +# dynamic "access_settings" { +# for_each = var.acl_workforce_pool_name != "" && var.acl_workforce_provider_id != "" ? [1] : [] +# content { +# enable_web_app = true +# workforce_identity_pool_provider = "${var.acl_workforce_pool_name}/providers/${var.acl_workforce_provider_id}" +# } +# } +# ui_settings { +# generative_answer_config { +# language_code = "en" +# } +# enable_autocomplete = true +# enable_quality_feedback = false +# disable_user_events_collection = true +# enable_people_search = false +# } +# } + +# ---------------------------------------------------------------------------- # +# Gemini Enterprise - Google Cloud Storage Data Stores # # ---------------------------------------------------------------------------- # # GCS Buckets for Discovery Engine Data Sources resource "google_storage_bucket" "gemini_enterprise_gcs_bucket" { - for_each = var.create_data_stores ? toset(var.gcs_data_store_names) : [] + for_each = var.create_data_stores ? { for k, v in var.gcs_data_store_configs : k => v if v.create_bucket } : {} project = var.main_project_id - name = "${var.main_project_id}-${each.key}-data" + name = each.value.name location = var.geolocation uniform_bucket_level_access = true - force_destroy = false # Set to true only for non-production + force_destroy = true # Set to true only for non-production/demo - encryption { - default_kms_key_name = local.cmek_key_id + dynamic "encryption" { + for_each = local.cmek_key_id != null ? [1] : [] + content { + default_kms_key_name = local.cmek_key_id + } } lifecycle_rule { @@ -78,7 +218,7 @@ resource "google_storage_bucket" "gemini_enterprise_gcs_bucket" { labels = { environment = var.environment - service = "${var.prefix}-gcs" + service = "g4g-gcs-data-store" data_store = each.key } @@ -89,7 +229,7 @@ resource "google_storage_bucket" "gemini_enterprise_gcs_bucket" { # Random suffix for GCS Data Store IDs resource "random_string" "gcs_suffix" { - for_each = var.create_data_stores ? toset(var.gcs_data_store_names) : [] + for_each = var.create_data_stores ? var.gcs_data_store_configs : {} length = 6 special = false @@ -103,18 +243,18 @@ resource "random_string" "gcs_suffix" { } } -# Discovery Engine Data Stores for GCS +# Empty GCS Data Store resource "google_discovery_engine_data_store" "gemini_enterprise_gcs_data_store" { - for_each = var.create_data_stores ? toset(var.gcs_data_store_names) : [] + for_each = var.create_data_stores ? var.gcs_data_store_configs : {} project = var.main_project_id location = var.geolocation # Must match the Data Store and Engine location - data_store_id = "${each.key}-gcs-data-store-${random_string.gcs_suffix[each.key].result}" - display_name = each.key + data_store_id = "g4g-gcs-data-store-${random_string.gcs_suffix[each.key].result}" + display_name = each.value.display_name != null ? each.value.display_name : each.key industry_vertical = local.discovery_engine_industry_vertical content_config = local.discovery_engine_content_config solution_types = local.discovery_engine_solution_types - kms_key_name = local.cmek_key_id + kms_key_name = var.enable_data_store_cmek ? local.cmek_key_id : null provider = google-beta document_processing_config { @@ -127,33 +267,49 @@ resource "google_discovery_engine_data_store" "gemini_enterprise_gcs_data_store" } depends_on = [ - google_discovery_engine_cmek_config.default, + # google_discovery_engine_cmek_config.default, google_kms_crypto_key_iam_member.discoveryengine_sa_kms_access, google_kms_crypto_key_iam_member.gcs_sa_kms_access, google_project_service.services, time_sleep.wait_for_services, + time_sleep.wait_for_gcs_iam, ] } -# ---------------------------------------------------------------------------- # -# BigQuery Data Stores # -# ---------------------------------------------------------------------------- # +# Grant Storage Admin to Discovery Engine SA if GCS Data Stores are present +resource "google_project_iam_member" "discoveryengine_sa_gcs_admin" { + count = var.create_data_stores && length(var.gcs_data_store_configs) > 0 ? 1 : 0 + project = var.main_project_id + role = "roles/storage.admin" + member = "serviceAccount:${google_project_service_identity.discoveryengine.email}" +} -locals { - bq_configs = { for idx, config in var.bq_data_store_configs : idx => config } +# Wait for IAM propagation before creating Data store which triggers doc import +resource "time_sleep" "wait_for_gcs_iam" { + count = var.create_data_stores && length(var.gcs_data_store_configs) > 0 ? 1 : 0 + create_duration = "60s" + depends_on = [google_project_iam_member.discoveryengine_sa_gcs_admin] } +# ---------------------------------------------------------------------------- # +# Gemini Enterprise - BigQuery Data Stores # +# ---------------------------------------------------------------------------- # + +# BQ Dataset for Discovery Engine Data Sources resource "google_bigquery_dataset" "gemini_enterprise_bq_dataset" { - for_each = var.create_data_stores ? local.bq_configs : {} + for_each = var.create_data_stores ? { for k, v in var.bq_data_store_configs : k => v if v.create_dataset } : {} project = var.main_project_id dataset_id = each.value.dataset_id - friendly_name = "Gemini Enterprise Data - ${each.value.dataset_id}" - description = "Dataset for Discovery Engine connector - ${each.value.dataset_id}" + friendly_name = "Gemini Enterprise Data Store - ${each.value.display_name}" + description = "Dataset for Gemini Enterprise Data Store - ${each.value.display_name}" location = var.geolocation # Or a more specific region specific location if desired - default_encryption_configuration { - kms_key_name = local.cmek_key_id + dynamic "default_encryption_configuration" { + for_each = local.cmek_key_id != null ? [1] : [] + content { + kms_key_name = local.cmek_key_id + } } depends_on = [ @@ -163,51 +319,9 @@ resource "google_bigquery_dataset" "gemini_enterprise_bq_dataset" { ] } -resource "google_bigquery_table" "gemini_enterprise_bq_table" { - for_each = var.create_data_stores ? local.bq_configs : {} - - project = var.main_project_id - dataset_id = google_bigquery_dataset.gemini_enterprise_bq_dataset[each.key].dataset_id - table_id = each.value.table_id - deletion_protection = false - - encryption_configuration { - kms_key_name = local.cmek_key_id - } - - # Define a default schema, users can adapt this as needed - schema = < 0 ? 1 : 0 + project = var.main_project_id + role = "roles/bigquery.admin" + member = "serviceAccount:${google_project_service_identity.discoveryengine.email}" +} - depends_on = [ - time_sleep.wait_for_services - ] +# Wait for IAM propagation before creating Data store which triggers schema fetch/import +resource "time_sleep" "wait_for_bq_iam" { + count = var.create_data_stores && length(var.bq_data_store_configs) > 0 ? 1 : 0 + create_duration = "60s" + depends_on = [google_project_iam_member.discoveryengine_sa_bq_admin] } diff --git a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/iam.tf b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/iam.tf index d8b66de4c..fbb6989e3 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/iam.tf +++ b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/iam.tf @@ -46,13 +46,15 @@ resource "google_project_iam_member" "admins_logging_viewer" { # --- User Group Roles --- resource "google_project_iam_member" "users_discoveryengine_user" { - project = var.main_project_id - role = "roles/discoveryengine.user" - member = var.user_group + for_each = toset(var.user_groups) + project = var.main_project_id + role = "roles/discoveryengine.user" + member = each.value } resource "google_project_iam_member" "users_serviceusage_consumer" { - project = var.main_project_id - role = "roles/serviceusage.serviceUsageConsumer" - member = var.user_group + for_each = toset(var.user_groups) + project = var.main_project_id + role = "roles/serviceusage.serviceUsageConsumer" + member = each.value } diff --git a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/load_balancer.tf b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/load_balancer.tf index 1720270ff..8ff9923ea 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/load_balancer.tf +++ b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/load_balancer.tf @@ -18,6 +18,8 @@ locals { # Define the Backend Service on the Load Balancer and integrate all components. resource "google_compute_region_backend_service" "gemini_enterprise_backend" { + count = var.deployment_type != "none" ? 1 : 0 + provider = google-beta name = "${var.prefix}-backend-service" project = var.main_project_id protocol = "HTTPS" @@ -26,7 +28,7 @@ resource "google_compute_region_backend_service" "gemini_enterprise_backend" { # Attach the Internet NEG backend { - group = google_compute_region_network_endpoint_group.gemini_enterprise_neg.id + group = google_compute_region_network_endpoint_group.gemini_enterprise_neg[0].id capacity_scaler = 1.0 } @@ -39,7 +41,7 @@ resource "google_compute_region_backend_service" "gemini_enterprise_backend" { enable = true sample_rate = 1 } - security_policy = google_compute_region_security_policy.gemini_enterprise_policy.self_link + security_policy = google_compute_region_security_policy.gemini_enterprise_policy[0].self_link lifecycle { ignore_changes = [ @@ -51,6 +53,7 @@ resource "google_compute_region_backend_service" "gemini_enterprise_backend" { # This is an optional but recommended companion to the HTTPS setup, # creating an HTTP load balancer to redirect HTTP traffic to HTTPS. resource "google_compute_region_url_map" "gemini_enterprise_http_redirect_url_map" { + count = var.deployment_type != "none" ? 1 : 0 project = var.main_project_id name = "${var.prefix}-http-redirect-url-map" region = var.region @@ -64,13 +67,15 @@ resource "google_compute_region_url_map" "gemini_enterprise_http_redirect_url_ma } resource "google_compute_region_target_http_proxy" "gemini_enterprise_http_proxy" { + count = var.deployment_type != "none" ? 1 : 0 project = var.main_project_id name = "${var.prefix}-http-proxy" region = var.region - url_map = google_compute_region_url_map.gemini_enterprise_http_redirect_url_map.id + url_map = google_compute_region_url_map.gemini_enterprise_http_redirect_url_map[0].id } resource "google_compute_forwarding_rule" "gemini_enterprise_http_forwarding_rule" { + count = var.deployment_type != "none" ? 1 : 0 project = var.main_project_id name = "${var.prefix}-http-forwarding-rule" region = var.region @@ -79,6 +84,18 @@ resource "google_compute_forwarding_rule" "gemini_enterprise_http_forwarding_rul load_balancing_scheme = local.load_balancing_scheme network = local.vpc_network_id subnetwork = var.deployment_type == "internal" ? local.vpc_subnet_id : null - ip_address = google_compute_address.gemini_enterprise_ip.address - target = google_compute_region_target_http_proxy.gemini_enterprise_http_proxy.id + ip_address = google_compute_address.gemini_enterprise_ip[0].address + target = google_compute_region_target_http_proxy.gemini_enterprise_http_proxy[0].id +} + +# ----------------------------------------------------------------------------- +# Certificate Manager DNS Authorization +# ----------------------------------------------------------------------------- +resource "google_certificate_manager_dns_authorization" "gemini_enterprise_dns_auth" { + count = var.deployment_type != "none" && var.cert_management_choice == "google_managed" ? 1 : 0 + name = "${var.prefix}-dns-auth" + location = var.region + project = var.main_project_id + description = "DNS authorization for Gemini Enterprise Google-managed certificate" + domain = var.custom_domain } \ No newline at end of file diff --git a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/main.tf b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/main.tf index 3840f8b11..03200889d 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/main.tf +++ b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/main.tf @@ -19,25 +19,37 @@ data "google_project" "project" { project_id = var.main_project_id } -resource "google_project_service" "services" { - project = var.main_project_id - for_each = toset([ +locals { + base_services = [ + "aiplatform.googleapis.com", "discoveryengine.googleapis.com", "compute.googleapis.com", "cloudkms.googleapis.com", "bigquery.googleapis.com", - "aiplatform.googleapis.com", "storage.googleapis.com", "accesscontextmanager.googleapis.com", - "beyondcorp.googleapis.com", - "certificatemanager.googleapis.com", "iam.googleapis.com", "iap.googleapis.com", "orgpolicy.googleapis.com", "serviceusage.googleapis.com", - "secretmanager.googleapis.com" # Added Secret Manager API - ]) - service = each.value + "secretmanager.googleapis.com" + ] + + restricted_services = [ + "beyondcorp.googleapis.com", + "certificatemanager.googleapis.com" + ] + + enabled_services = concat( + local.base_services, + var.compliance_regime == "FEDRAMP_HIGH" || var.compliance_regime == "NONE" ? local.restricted_services : [] + ) +} + +resource "google_project_service" "services" { + project = var.main_project_id + for_each = toset(local.enabled_services) + service = each.value timeouts { create = "30m" update = "40m" @@ -96,9 +108,9 @@ resource "google_project_service_identity" "iap" { # service-projectid@gs-project-accounts.iam.gserviceaccount.com # service-projectid@gcp-sa-discoveryengine.iam.gserviceaccount.com -# This wait time is needed to give time to the API enablement, and the service-agents to create the google service-agents above, which are required to utilize the cloud KMS key. +# This wait time is needed to give time to the API enablement and to create the google service-agents above, which are required to utilize the cloud KMS key. resource "time_sleep" "wait_for_services" { - create_duration = "280s" #Wait for APIs, particularly to avoid the "Discovery Engine API has not been used in project" error. + create_duration = "180s" #Wait for APIs, particularly to avoid the "Discovery Engine API has not been used in project" error. depends_on = [ google_project_service.services diff --git a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/network.tf b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/network.tf index 19e737e09..22328d50c 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/network.tf +++ b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/network.tf @@ -16,27 +16,28 @@ # Networking Resources # # ---------------------------------------------------------------------------- # locals { - vpc_network_id = var.use_shared_vpc ? data.google_compute_network.shared_vpc[0].id : google_compute_network.gemini_enterprise_vpc[0].id - vpc_subnet_id = var.use_shared_vpc ? data.google_compute_subnetwork.shared_subnet[0].id : google_compute_subnetwork.gemini_enterprise_vpc_subnet[0].id - vpc_proxy_subnet_id = var.use_shared_vpc ? data.google_compute_subnetwork.shared_proxy_subnet[0].id : google_compute_subnetwork.gemini_enterprise_vpc_proxy_subnet[0].id + # Safe lookups with try handles when deployment_type == "none" + vpc_network_id = var.use_shared_vpc ? try(data.google_compute_network.shared_vpc[0].id, null) : try(google_compute_network.gemini_enterprise_vpc[0].id, null) + vpc_subnet_id = var.use_shared_vpc ? try(data.google_compute_subnetwork.shared_subnet[0].id, null) : try(google_compute_subnetwork.gemini_enterprise_vpc_subnet[0].id, null) + vpc_proxy_subnet_id = var.use_shared_vpc ? try(data.google_compute_subnetwork.shared_proxy_subnet[0].id, null) : try(google_compute_subnetwork.gemini_enterprise_vpc_proxy_subnet[0].id, null) ip_address_type = var.deployment_type == "internal" ? "INTERNAL" : "EXTERNAL" } resource "google_compute_network" "gemini_enterprise_vpc" { - count = var.use_shared_vpc ? 0 : 1 + count = (var.use_shared_vpc || var.deployment_type == "none") ? 0 : 1 project = var.main_project_id name = "${var.prefix}-vpc" auto_create_subnetworks = false } data "google_compute_network" "shared_vpc" { - count = var.use_shared_vpc ? 1 : 0 + count = (var.use_shared_vpc && var.deployment_type != "none") ? 1 : 0 project = var.network_project_id name = var.shared_vpc_network_name } resource "google_compute_subnetwork" "gemini_enterprise_vpc_subnet" { - count = var.use_shared_vpc ? 0 : 1 + count = (var.use_shared_vpc || var.deployment_type == "none") ? 0 : 1 project = var.main_project_id name = "${var.prefix}-vpc-subnet" ip_cidr_range = var.internal_lb_subnet_range @@ -46,7 +47,7 @@ resource "google_compute_subnetwork" "gemini_enterprise_vpc_subnet" { } resource "google_compute_subnetwork" "gemini_enterprise_vpc_proxy_subnet" { - count = var.use_shared_vpc ? 0 : 1 + count = (var.use_shared_vpc || var.deployment_type == "none") ? 0 : 1 project = var.main_project_id name = "${var.prefix}-vpc-proxy-subnet" ip_cidr_range = "10.10.11.0/24" @@ -57,20 +58,21 @@ resource "google_compute_subnetwork" "gemini_enterprise_vpc_proxy_subnet" { } data "google_compute_subnetwork" "shared_subnet" { - count = var.use_shared_vpc ? 1 : 0 + count = (var.use_shared_vpc && var.deployment_type != "none") ? 1 : 0 project = var.network_project_id region = var.region name = var.shared_vpc_subnet_name } data "google_compute_subnetwork" "shared_proxy_subnet" { - count = var.use_shared_vpc ? 1 : 0 + count = (var.use_shared_vpc && var.deployment_type != "none") ? 1 : 0 project = var.network_project_id region = var.region name = var.shared_vpc_proxy_subnet_name } resource "google_compute_address" "gemini_enterprise_ip" { + count = var.deployment_type != "none" ? 1 : 0 project = var.main_project_id name = "${var.prefix}-${var.deployment_type}-ip" region = var.region @@ -82,6 +84,7 @@ resource "google_compute_address" "gemini_enterprise_ip" { # Internet NEG for vertexaisearch.cloud.google.com FQDN # ----------------------------------------------------------------------------- resource "google_compute_region_network_endpoint_group" "gemini_enterprise_neg" { + count = var.deployment_type != "none" ? 1 : 0 name = "${var.prefix}-internet-neg" project = var.main_project_id network = local.vpc_network_id @@ -90,8 +93,9 @@ resource "google_compute_region_network_endpoint_group" "gemini_enterprise_neg" } resource "google_compute_region_network_endpoint" "gemini_enterprise_endpoint" { + count = var.deployment_type != "none" ? 1 : 0 project = var.main_project_id - region_network_endpoint_group = google_compute_region_network_endpoint_group.gemini_enterprise_neg.name + region_network_endpoint_group = google_compute_region_network_endpoint_group.gemini_enterprise_neg[0].name region = var.region fqdn = "vertexaisearch.cloud.google.com" port = 443 diff --git a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/outputs.tf b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/outputs.tf index 974f2e8e6..098261d9c 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/outputs.tf +++ b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/outputs.tf @@ -17,21 +17,31 @@ output "admin_group" { description = "The principal for the Gemini Enterprise administrators group." } -output "user_group" { - value = var.user_group - description = "The principal for the Gemini Enterprise users group." +output "user_groups" { + value = var.user_groups + description = "The principals for the Gemini Enterprise users groups." } output "gemini_enterprise_ip" { - value = google_compute_address.gemini_enterprise_ip.address + value = var.deployment_type != "none" ? google_compute_address.gemini_enterprise_ip[0].address : null description = "The reserved IP address for the load balancer." } +output "dns_auth_records" { + value = var.deployment_type != "none" && var.cert_management_choice == "google_managed" ? google_certificate_manager_dns_authorization.gemini_enterprise_dns_auth[0].dns_resource_record : null + description = "DNS Authorization resource records for Google-managed certificate." +} + output "deployment_type" { value = var.deployment_type description = "The deployment type of the load balancer (internal or external)." } +output "compliance_regime" { + value = var.compliance_regime + description = "The compliance regime selected during deployment." +} + output "tf_state_bucket_name" { value = data.google_storage_bucket.terraform_state.name description = "The name of the GCS bucket used for Terraform state." @@ -102,30 +112,26 @@ output "shared_vpc_proxy_subnet_name" { description = "The Shared VPC Proxy Subnet Name." } -output "gcs_data_store_ids" { - description = "A list of GCS Discovery Engine Data Store IDs." - value = [for v in google_discovery_engine_data_store.gemini_enterprise_gcs_data_store : v.data_store_id] -} - -output "gcs_data_store_to_bucket" { - description = "A mapping of GCS Data Store IDs to their corresponding GCS Bucket names." - value = { for k, v in google_discovery_engine_data_store.gemini_enterprise_gcs_data_store : v.data_store_id => google_storage_bucket.gemini_enterprise_gcs_bucket[k].name } -} - -output "bq_data_store_ids" { - description = "A list of BigQuery Discovery Engine Data Store IDs." - value = [for v in google_discovery_engine_data_store.gemini_enterprise_bq_data_store : v.data_store_id] +output "gcs_data_stores" { + description = "A mapping of formatted data store keys to their configuration, ID, and bucket details." + value = { for k, v in var.gcs_data_store_configs : k => { + display_name = v.display_name + data_store_id = can(google_discovery_engine_data_store.gemini_enterprise_gcs_data_store[k]) ? google_discovery_engine_data_store.gemini_enterprise_gcs_data_store[k].data_store_id : null + bucket_name = can(google_storage_bucket.gemini_enterprise_gcs_bucket[k]) ? google_storage_bucket.gemini_enterprise_gcs_bucket[k].name : v.name + } } } -output "bq_data_store_to_dataset_table" { - description = "A mapping of BigQuery Data Store IDs to their corresponding Dataset and Table." - value = { for k, v in google_discovery_engine_data_store.gemini_enterprise_bq_data_store : v.data_store_id => { - dataset_id = google_bigquery_dataset.gemini_enterprise_bq_dataset[k].dataset_id - table_id = google_bigquery_table.gemini_enterprise_bq_table[k].table_id +output "bq_data_stores" { + description = "A mapping of formatted data store keys to their configuration, ID, dataset, and table details." + value = { for k, v in var.bq_data_store_configs : k => { + display_name = v.display_name + data_store_id = can(google_discovery_engine_data_store.gemini_enterprise_bq_data_store[k]) ? google_discovery_engine_data_store.gemini_enterprise_bq_data_store[k].data_store_id : null + dataset_id = can(google_bigquery_dataset.gemini_enterprise_bq_dataset[k]) ? google_bigquery_dataset.gemini_enterprise_bq_dataset[k].dataset_id : v.dataset_id + table_id = v.table_id } } } -output "cmek_key_id" { - description = "The CMEK Key ID used for encryption." - value = local.cmek_key_id -} +# output "engine_ids" { +# value = { for k, v in google_discovery_engine_search_engine.gemini_enterprise_search_engine : k => v.engine_id } +# description = "A map of application keys to their corresponding Gemini Enterprise Search Engine IDs." +# } diff --git a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/terraform.tfvars.sample b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/terraform.tfvars.sample index 5a756c68d..c8e951a03 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/terraform.tfvars.sample +++ b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/terraform.tfvars.sample @@ -33,7 +33,7 @@ acl_idp_type = "GSUITE" # If GSUITE, provide the Google Group name using IAM syntax (i.e. "group:[GROUP_NAME]@[DOMAIN]") # If THIRD_PARTY, provide the Workforce Identity Federation principalSet (i.e. "principalSet://iam.googleapis.com/locations/global/workforcePools/[WORKFORCE_POOL_ID]/group/[GROUP_ID]") admin_group = "group:gcp-gemini-enterprise-admins@customer-domain.com" -user_group = "group:gcp-gemini-enterprise-users@customer-domain.com" +user_groups = ["group:gcp-gemini-enterprise-users@customer-domain.com"] # Security # Enable Chrome Enterprise Premium (Zero Trust) features diff --git a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/variables.tf b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/variables.tf index aae1f6216..1387753b8 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/variables.tf +++ b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-0/variables.tf @@ -24,6 +24,12 @@ variable "tenant" { type = string } +variable "compliance_regime" { + description = "Compliance regime this environment is deployed in (e.g. FEDRAMP_HIGH, IL4, IL5, NONE)." + type = string + default = "NONE" +} + variable "kms_project_id" { description = "The Project ID where CMEK keys are stored." type = string @@ -61,9 +67,9 @@ variable "admin_group" { type = string } -variable "user_group" { - description = "The email address of the Gemini Enterprise users group." - type = string +variable "user_groups" { + description = "A list of email addresses or principal sets for the Gemini Enterprise users group." + type = list(string) } variable "gemini_enterprise_gcs_bucket_name" { @@ -132,15 +138,31 @@ variable "kms_key_id" { } variable "deployment_type" { - description = "Type of deployment: 'internal' or 'external'" + description = "Type of deployment: 'internal', 'external', or 'none'" type = string default = "external" # Default to external as per original design validation { - condition = contains(["internal", "external"], var.deployment_type) - error_message = "Allowed values for deployment_type are 'internal' or 'external'." + condition = contains(["internal", "external", "none"], var.deployment_type) + error_message = "Allowed values for deployment_type are 'internal', 'external', or 'none'." } } +variable "cert_management_choice" { + description = "Certificate management choice for external deployments: 'google_managed' or 'self_managed'." + type = string + default = "self_managed" + validation { + condition = contains(["google_managed", "self_managed"], var.cert_management_choice) + error_message = "Allowed values for cert_management_choice are 'google_managed' or 'self_managed'." + } +} + +variable "custom_domain" { + description = "The fully qualified domain name (FQDN) to use for the Google-managed certificate." + type = string + default = "" +} + variable "create_ip_based_access" { description = "Whether to create the IP-based access level." type = bool @@ -190,12 +212,12 @@ variable "create_data_stores" { } variable "acl_idp_type" { - description = "The Identity Provider type for Discovery Engine ACLs. Options: 'GSUITE', 'THIRD_PARTY'." + description = "The Identity Provider type for Discovery Engine ACLs. Options: 'GSUITE', 'THIRD_PARTY', 'GOOGLE_CLOUD_IDENTITY'." type = string - default = "GSUITE" + default = "GOOGLE_CLOUD_IDENTITY" validation { - condition = contains(["GSUITE", "THIRD_PARTY"], var.acl_idp_type) - error_message = "The acl_idp_type value must be either 'GSUITE' or 'THIRD_PARTY'." + condition = contains(["GSUITE", "THIRD_PARTY", "GOOGLE_CLOUD_IDENTITY"], var.acl_idp_type) + error_message = "The acl_idp_type value must be either 'GSUITE', 'THIRD_PARTY', or 'GOOGLE_CLOUD_IDENTITY'." } } @@ -253,20 +275,26 @@ variable "shared_vpc_proxy_subnet_name" { default = "" } -variable "gcs_data_store_names" { - description = "A list of names to use for creating GCS buckets and associated Discovery Engine Data Stores." - type = list(string) - default = [] +variable "gcs_data_store_configs" { + description = "A map of configurations for Google Cloud Storage (GCS) Data Stores. If create_bucket is true, the script will create the bucket." + type = map(object({ + name = string + create_bucket = bool + display_name = optional(string) + })) + default = {} } variable "bq_data_store_configs" { - description = "A list of objects defining BigQuery datasets and tables to create and connect to Discovery Engine. Each object should have 'dataset_id' and 'table_id'." - type = list(object({ - dataset_id = string - table_id = string - + description = "A map of configurations for BigQuery Data Stores. If create_dataset is true, it creates the dataset. Schema provides the structure for the data." + type = map(object({ + dataset_id = string + table_id = string + create_dataset = bool + schema = optional(string) + display_name = optional(string) })) - default = [] + default = {} } @@ -287,3 +315,15 @@ variable "moderate_device_access_levels" { type = list(string) default = [] } + +variable "enable_data_store_cmek" { + description = "Whether to encrypt Data Stores with CMEK. If false, Google-managed keys will be used." + type = bool + default = true +} + +variable "enable_analytics" { + description = "Enable analytics for Gemini Enterprise via Discovery Engine Audit Logs." + type = bool + default = false +} diff --git a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-1/load_balancer.tf b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-1/load_balancer.tf index 0eee165d1..05075f763 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-1/load_balancer.tf +++ b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-1/load_balancer.tf @@ -23,6 +23,7 @@ data "terraform_remote_state" "stage_0" { # Data source to get the details of the customer's pre-uploaded SSL certificate data "google_compute_region_ssl_certificate" "gemini_enterprise_cert" { + count = data.terraform_remote_state.stage_0.outputs.deployment_type != "none" && var.cert_management_choice == "self_managed" ? 1 : 0 project = data.terraform_remote_state.stage_0.outputs.main_project_id name = var.ssl_certificate_name region = data.terraform_remote_state.stage_0.outputs.region @@ -30,6 +31,7 @@ data "google_compute_region_ssl_certificate" "gemini_enterprise_cert" { # Data source to get the backend service created in stage-0 data "google_compute_region_backend_service" "gemini_enterprise_backend" { + count = data.terraform_remote_state.stage_0.outputs.deployment_type != "none" ? 1 : 0 project = data.terraform_remote_state.stage_0.outputs.main_project_id name = "${data.terraform_remote_state.stage_0.outputs.prefix}-backend-service" region = data.terraform_remote_state.stage_0.outputs.region @@ -37,6 +39,7 @@ data "google_compute_region_backend_service" "gemini_enterprise_backend" { # Data source to get the network created in stage-0 or Shared VPC data "google_compute_network" "gemini_enterprise_vpc" { + count = data.terraform_remote_state.stage_0.outputs.deployment_type != "none" ? 1 : 0 project = var.host_project_id != "" ? var.host_project_id : ( try(data.terraform_remote_state.stage_0.outputs.use_shared_vpc, false) ? data.terraform_remote_state.stage_0.outputs.network_project_id : data.terraform_remote_state.stage_0.outputs.main_project_id ) @@ -47,6 +50,7 @@ data "google_compute_network" "gemini_enterprise_vpc" { # Data source to get the IP address created in stage-0 data "google_compute_address" "gemini_enterprise_ip" { + count = data.terraform_remote_state.stage_0.outputs.deployment_type != "none" ? 1 : 0 project = data.terraform_remote_state.stage_0.outputs.main_project_id name = "${data.terraform_remote_state.stage_0.outputs.prefix}-${data.terraform_remote_state.stage_0.outputs.deployment_type}-ip" region = data.terraform_remote_state.stage_0.outputs.region @@ -58,11 +62,12 @@ locals { # This resource defines the URL map with the specified routing rules. resource "google_compute_region_url_map" "gemini_enterprise_load_balancer" { + count = data.terraform_remote_state.stage_0.outputs.deployment_type != "none" ? 1 : 0 project = data.terraform_remote_state.stage_0.outputs.main_project_id name = "${data.terraform_remote_state.stage_0.outputs.prefix}-gemini-enterprise-url-map" region = data.terraform_remote_state.stage_0.outputs.region description = "URL map for ${data.terraform_remote_state.stage_0.outputs.prefix}-gemini-enterprise" - default_service = data.google_compute_region_backend_service.gemini_enterprise_backend.id + default_service = data.google_compute_region_backend_service.gemini_enterprise_backend[0].id host_rule { hosts = ["${var.gemini_enterprise_domain}"] @@ -73,14 +78,14 @@ resource "google_compute_region_url_map" "gemini_enterprise_load_balancer" { for_each = data.terraform_remote_state.stage_0.outputs.acl_idp_type == "GSUITE" ? ["gsuite"] : [] content { name = "path-matcher-1" - default_service = data.google_compute_region_backend_service.gemini_enterprise_backend.id + default_service = data.google_compute_region_backend_service.gemini_enterprise_backend[0].id route_rules { priority = 100 match_rules { prefix_match = "/" } - service = data.google_compute_region_backend_service.gemini_enterprise_backend.id + service = data.google_compute_region_backend_service.gemini_enterprise_backend[0].id route_action { url_rewrite { host_rewrite = "vertexaisearch.cloud.google.com" @@ -106,29 +111,46 @@ resource "google_compute_region_url_map" "gemini_enterprise_load_balancer" { } } +# Certificate Manager Certificate for Google Managed Option +resource "google_certificate_manager_certificate" "gemini_enterprise_managed_cert" { + count = data.terraform_remote_state.stage_0.outputs.deployment_type != "none" && var.cert_management_choice == "google_managed" ? 1 : 0 + name = "${data.terraform_remote_state.stage_0.outputs.prefix}-managed-cert" + description = "Google-managed certificate for Gemini Enterprise" + location = data.terraform_remote_state.stage_0.outputs.region + project = data.terraform_remote_state.stage_0.outputs.main_project_id + managed { + domains = [var.gemini_enterprise_domain] + dns_authorizations = [ + "projects/${data.terraform_remote_state.stage_0.outputs.main_project_id}/locations/${data.terraform_remote_state.stage_0.outputs.region}/dnsAuthorizations/${data.terraform_remote_state.stage_0.outputs.prefix}-dns-auth" + ] + } +} + # This resource creates the target HTTPS proxy for the load balancer. -# It now references the pre-existing SSL certificate via the data source. resource "google_compute_region_target_https_proxy" "gemini_enterprise_https_proxy" { + count = data.terraform_remote_state.stage_0.outputs.deployment_type != "none" ? 1 : 0 project = data.terraform_remote_state.stage_0.outputs.main_project_id name = "${data.terraform_remote_state.stage_0.outputs.prefix}-gemini-enterprise-https-proxy" region = data.terraform_remote_state.stage_0.outputs.region - url_map = google_compute_region_url_map.gemini_enterprise_load_balancer.id - ssl_certificates = [data.google_compute_region_ssl_certificate.gemini_enterprise_cert.self_link] + url_map = google_compute_region_url_map.gemini_enterprise_load_balancer[0].id + + ssl_certificates = var.cert_management_choice == "self_managed" ? [data.google_compute_region_ssl_certificate.gemini_enterprise_cert[0].self_link] : [] + certificate_manager_certificates = var.cert_management_choice == "google_managed" ? [google_certificate_manager_certificate.gemini_enterprise_managed_cert[0].id] : [] } # This resource creates the forwarding rule for the load balancer. -# This requires the SSL cert via the proxy to be uploaded, pending stage 00 and upload. resource "google_compute_forwarding_rule" "gemini_enterprise_forwarding_rule" { + count = data.terraform_remote_state.stage_0.outputs.deployment_type != "none" ? 1 : 0 project = data.terraform_remote_state.stage_0.outputs.main_project_id name = "${data.terraform_remote_state.stage_0.outputs.prefix}-gemini-enterprise-forwarding-rule" region = data.terraform_remote_state.stage_0.outputs.region ip_protocol = "TCP" port_range = "443" load_balancing_scheme = local.load_balancing_scheme - network = data.google_compute_network.gemini_enterprise_vpc.self_link + network = data.google_compute_network.gemini_enterprise_vpc[0].self_link subnetwork = data.terraform_remote_state.stage_0.outputs.deployment_type == "internal" ? data.google_compute_subnetwork.gemini_enterprise_vpc_subnet[0].self_link : null - ip_address = data.google_compute_address.gemini_enterprise_ip.address - target = google_compute_region_target_https_proxy.gemini_enterprise_https_proxy.id + ip_address = data.google_compute_address.gemini_enterprise_ip[0].address + target = google_compute_region_target_https_proxy.gemini_enterprise_https_proxy[0].id } # Data source to get the subnet created in stage-0 @@ -141,11 +163,11 @@ data "google_compute_subnetwork" "gemini_enterprise_vpc_subnet" { # --- IAP Access Roles --- # Grant a user or group access through IAP. -# --- IAP Access Roles --- (CORRECT REGIONAL RESOURCE TYPE) resource "google_iap_web_region_backend_service_iam_member" "iap_admin" { + count = data.terraform_remote_state.stage_0.outputs.deployment_type != "none" ? 1 : 0 project = data.terraform_remote_state.stage_0.outputs.main_project_id region = data.terraform_remote_state.stage_0.outputs.region - web_region_backend_service = data.google_compute_region_backend_service.gemini_enterprise_backend.name + web_region_backend_service = data.google_compute_region_backend_service.gemini_enterprise_backend[0].name role = "roles/iap.httpsResourceAccessor" member = data.terraform_remote_state.stage_0.outputs.admin_group condition { @@ -156,11 +178,12 @@ resource "google_iap_web_region_backend_service_iam_member" "iap_admin" { } resource "google_iap_web_region_backend_service_iam_member" "iap_user" { + for_each = data.terraform_remote_state.stage_0.outputs.deployment_type != "none" ? toset(data.terraform_remote_state.stage_0.outputs.user_groups) : toset([]) project = data.terraform_remote_state.stage_0.outputs.main_project_id region = data.terraform_remote_state.stage_0.outputs.region - web_region_backend_service = data.google_compute_region_backend_service.gemini_enterprise_backend.name + web_region_backend_service = data.google_compute_region_backend_service.gemini_enterprise_backend[0].name role = "roles/iap.httpsResourceAccessor" - member = data.terraform_remote_state.stage_0.outputs.user_group + member = each.value condition { title = "User Access" description = data.terraform_remote_state.stage_0.outputs.enable_chrome_enterprise_premium ? "Access for Users with Moderate Device Policy" : "Access for Users with Basic Policy" diff --git a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-1/variables.tf b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-1/variables.tf index 3359c00de..18098d670 100644 --- a/blueprints/fedramp-high/gemini-enterprise/gemini-stage-1/variables.tf +++ b/blueprints/fedramp-high/gemini-enterprise/gemini-stage-1/variables.tf @@ -26,6 +26,17 @@ variable "gemini_enterprise_domain" { variable "ssl_certificate_name" { description = "The name of the pre-uploaded SSL certificate in Google Cloud." type = string + default = "" +} + +variable "cert_management_choice" { + description = "Certificate management choice for external deployments: 'google_managed' or 'self_managed'." + type = string + default = "self_managed" + validation { + condition = contains(["google_managed", "self_managed"], var.cert_management_choice) + error_message = "Allowed values for cert_management_choice are 'google_managed' or 'self_managed'." + } } variable "gemini_config_id" {