diff --git a/.example.env b/.example.env new file mode 100644 index 0000000..2cb6bad --- /dev/null +++ b/.example.env @@ -0,0 +1,7 @@ +ENV=dev +AWS_REGION=us-west-2 +STATE_BUCKET_NAME=ghgc-features-tf-state-bucket +PROJECT_NAME=ghgc-features-api +REGISTRY_NAME=ghgc-features-api-registry +SERVICE_PORT=8080 +VPC_ID= diff --git a/.github/actions/terraform-deploy/action.yml b/.github/actions/terraform-deploy/action.yml new file mode 100644 index 0000000..0d436db --- /dev/null +++ b/.github/actions/terraform-deploy/action.yml @@ -0,0 +1,59 @@ +name: Deploy +inputs: + env_aws_secret_name: + required: true + type: string + env-file: + type: string + default: ".env" + dir: + required: false + type: string + default: "." + script_path: + type: string + +runs: + using: "composite" + + steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: Install python dependencies + shell: bash + working-directory: ${{ inputs.dir }} + run: | + python -m pip install --upgrade pip + python -m pip install boto3 + + - name: Get relevant environment configuration from aws secrets + shell: bash + working-directory: ${{ inputs.dir }} + run: | + if [[ -z "${{ inputs.script_path }}" ]]; then + ./scripts/sync-env.sh ${{ inputs.env_aws_secret_name }} + else + python ${{ inputs.script_path }} --secret-id ${{ inputs.env_aws_secret_name }} + source ${{ inputs.env-file }} + echo "PREFIX=feature-api-${STAGE}" >> ${{ inputs.env-file }} + echo "REGISTRY_NAME=feature-api-${STAGE}" >> ${{ inputs.env-file }} + echo "ENV=${STAGE}" >> ${{ inputs.env-file }} + echo "PROJECT_NAME=veda-${STAGE}" >> ${{ inputs.env-file }} + fi + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.3.9 + + - name: Deploy + shell: bash + working-directory: ${{ inputs.dir }} + run: | + bash ./scripts/deploy.sh .env <<< init + bash ./scripts/deploy.sh .env <<< deploy + diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml new file mode 100644 index 0000000..5a2f99f --- /dev/null +++ b/.github/workflows/cicd.yml @@ -0,0 +1,60 @@ +name: CICD 🚀 + +permissions: + id-token: write + contents: read + +on: + push: + branches: + - main + - dev + - production + - update-workflows + +jobs: + define-environment: + name: Set ✨ environment ✨ + runs-on: ubuntu-latest + steps: + - name: Set the environment based on the branch + id: define_environment + run: | + if [ "${{ github.ref }}" = "refs/heads/main" ]; then + echo "env_name=staging" >> $GITHUB_OUTPUT + elif [ "${{ github.ref }}" = "refs/heads/dev" ]; then + echo "env_name=development" >> $GITHUB_OUTPUT + elif [ "${{ github.ref }}" = "refs/heads/production" ]; then + echo "env_name=production" >> $GITHUB_OUTPUT + elif [ "${{ github.ref }}" = "refs/heads/update-workflows" ]; then + echo "env_name=development" >> $GITHUB_OUTPUT + fi + - name: Print the environment + run: echo "The environment is ${{ steps.define_environment.outputs.env_name }}" + + outputs: + env_name: ${{ steps.define_environment.outputs.env_name }} + + deploy: + name: Deploy to ${{ needs.define-environment.outputs.env_name }} 🚀 + runs-on: ubuntu-latest + needs: [define-environment] + if: ${{ needs.define-environment.outputs.env_name }} + environment: ${{ needs.define-environment.outputs.env_name }} + concurrency: ${{ needs.define-environment.outputs.env_name }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "ghgc-features-api-github-${{ needs.define-environment.outputs.env_name }}-deployment" + aws-region: "us-west-2" + + - name: Run deployment + uses: "./.github/actions/terraform-deploy" + with: + env_aws_secret_name: ${{ secrets.ENV_AWS_SECRET_NAME }} diff --git a/README.md b/README.md index f48cd6e..8add51e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# VEDA Features API +# GHGC Features API -Hosting and serving collections of vector data features for VEDA +Hosting and serving collections of vector data features for GHGC --- diff --git a/db/Dockerfile b/db/Dockerfile new file mode 100644 index 0000000..8cc4921 --- /dev/null +++ b/db/Dockerfile @@ -0,0 +1,13 @@ +FROM --platform=linux/amd64 public.ecr.aws/lambda/python:3.9 + +# WORKDIR /tmp + +RUN pip install boto3 requests "urllib3<2" psycopg["binary"] -t "${LAMBDA_TASK_ROOT}" + +COPY ./handler.py ${LAMBDA_TASK_ROOT} + +# https://stackoverflow.com/a/61746719 +# Turns out, asyncio is part of python +# RUN rm -rf /asset/asyncio* + +CMD ["handler.handler"] diff --git a/db/handler.py b/db/handler.py new file mode 100644 index 0000000..16b4702 --- /dev/null +++ b/db/handler.py @@ -0,0 +1,200 @@ +# """Bootstrap Postgres db.""" + +import asyncio +import json + +import boto3 +import psycopg +# import requests +from psycopg import sql +from psycopg.conninfo import make_conninfo +import os + +# def send( +# event, +# context, +# responseStatus, +# responseData, +# physicalResourceId=None, +# noEcho=False, +# ): +# """ +# Copyright 2016 Amazon Web Services, Inc. or its affiliates. All Rights Reserved. +# This file is licensed to you under the AWS Customer Agreement (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at http://aws.amazon.com/agreement/ . +# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. +# See the License for the specific language governing permissions and limitations under the License. + +# Send response from AWS Lambda. + +# Note: The cfnresponse module is available only when you use the ZipFile property to write your source code. +# It isn't available for source code that's stored in Amazon S3 buckets. +# For code in buckets, you must write your own functions to send responses. +# """ +# responseUrl = event["ResponseURL"] + +# print(responseUrl) + +# responseBody = {} +# responseBody["Status"] = responseStatus +# responseBody["Reason"] = ( +# "See the details in CloudWatch Log Stream: " + context.log_stream_name +# ) +# responseBody["PhysicalResourceId"] = physicalResourceId or context.log_stream_name +# responseBody["StackId"] = event["StackId"] +# responseBody["RequestId"] = event["RequestId"] +# responseBody["LogicalResourceId"] = event["LogicalResourceId"] +# responseBody["NoEcho"] = noEcho +# responseBody["Data"] = responseData + +# json_responseBody = json.dumps(responseBody) + +# print("Response body:\n" + json_responseBody) + +# headers = {"content-type": "", "content-length": str(len(json_responseBody))} + +# try: +# response = requests.put(responseUrl, data=json_responseBody, headers=headers) +# print("Status code: " + response.reason) +# except Exception as e: +# print("send(..) failed executing requests.put(..): " + str(e)) + + +def get_secret(secret_name): + """Get Secrets from secret manager.""" + print(f"Fetching {secret_name}...") + client = boto3.client(service_name="secretsmanager") + response = client.get_secret_value(SecretId=secret_name) + return json.loads(response["SecretString"]) + + +def create_db(cursor, db_name: str) -> None: + """Create DB.""" + cursor.execute( + sql.SQL("SELECT 1 FROM pg_catalog.pg_database " "WHERE datname = %s"), [db_name] + ) + if cursor.fetchone(): + print(f"database {db_name} exists, not creating DB") + else: + print(f"database {db_name} not found, creating...") + cursor.execute( + sql.SQL("CREATE DATABASE {db_name}").format(db_name=sql.Identifier(db_name)) + ) + + +def create_user(cursor, username: str, password: str) -> None: + """Create User.""" + cursor.execute( + sql.SQL( + "DO $$ " + "BEGIN " + " IF NOT EXISTS ( " + " SELECT 1 FROM pg_roles " + " WHERE rolname = {user}) " + " THEN " + " CREATE USER {username} " + " WITH PASSWORD {password}; " + " ELSE " + " ALTER USER {username} " + " WITH PASSWORD {password}; " + " END IF; " + "END " + "$$; " + ).format(username=sql.Identifier(username), password=password, user=username) + ) + + +def create_permissions(cursor, db_name: str, username: str) -> None: + """Add permissions.""" + cursor.execute( + sql.SQL( + "GRANT CONNECT ON DATABASE {db_name} TO {username};" + "GRANT CREATE ON DATABASE {db_name} TO {username};" # Allow schema creation + "GRANT USAGE ON SCHEMA public TO {username};" + "ALTER DEFAULT PRIVILEGES IN SCHEMA public " + "GRANT ALL PRIVILEGES ON TABLES TO {username};" + "ALTER DEFAULT PRIVILEGES IN SCHEMA public " + "GRANT ALL PRIVILEGES ON SEQUENCES TO {username};" + ).format( + db_name=sql.Identifier(db_name), + username=sql.Identifier(username), + ) + ) + + +def register_extensions(cursor) -> None: + """Add PostGIS extension.""" + cursor.execute(sql.SQL("CREATE EXTENSION IF NOT EXISTS postgis;")) + + +def handler(event, context): + """Lambda Handler.""" + print(f"Event: {event}") + + if event["tf"]["action"] not in ["create", "update"]: + print("failed") + return 0 + # return send(event, context, "SUCCESS", {"msg": "No action to be taken"}) + + try: + connection_params = get_secret(os.environ['CONN_SECRET_ARN']) + user_params = event["user_params"] + print("Connecting to DB...") + con_str = make_conninfo( + dbname=connection_params.get("dbname", "postgres"), + user=connection_params["username"], + password=connection_params["password"], + host=connection_params["host"], + port=connection_params["port"], + ) + with psycopg.connect(con_str, autocommit=True) as conn: + with conn.cursor() as cur: + print("Creating database...") + create_db( + cursor=cur, + db_name=user_params["dbname"], + ) + + print("Creating user...") + create_user( + cursor=cur, + username=user_params["username"], + password=user_params["password"], + ) + + print("Setting permissions...") + create_permissions( + cursor=cur, + db_name=user_params["dbname"], + username=user_params["username"], + ) + + # Install extensions on the user DB with + # superuser permissions, since they will + # otherwise fail to install when run as + # the non-superuser within the pgstac + # migrations. + print("Connecting to DB...") + con_str = make_conninfo( + dbname=user_params["dbname"], + user=connection_params["username"], + password=connection_params["password"], + host=connection_params["host"], + port=connection_params["port"], + ) + with psycopg.connect(con_str, autocommit=True) as conn: + with conn.cursor() as cur: + print("Registering PostGIS ...") + register_extensions(cursor=cur) + + except Exception as e: + print(e) + return { + 'message' : e + } + + print("Event Complete") + return { + 'message' : connection_params + } \ No newline at end of file diff --git a/docs/IACHOWTO.md b/docs/IACHOWTO.md index 82c6162..287a448 100644 --- a/docs/IACHOWTO.md +++ b/docs/IACHOWTO.md @@ -22,6 +22,8 @@ $ tfenv use 1.3.9 5. we also use Terraform "workspaces" so our infra state stays nicely separated in the same S3 bucket. Some quick samples of how to interact with that: ```bash +$ AWS_PROFILE= terraform workspace new west2-staging + $ AWS_PROFILE= terraform workspace list * default west2-staging diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100644 index 0000000..91f2016 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,25 @@ +#!/bin/sh +export TARGET_ENVIRONMENT=dev +export TARGET_PROJECT_NAME=ghgc-features-api + +cd wfs3-app/ + +aws ecr describe-repositories \ + | jq '.repositories | map(.repositoryUri)' \ + | grep $TARGET_PROJECT_NAME | grep $TARGET_ENVIRONMENT \ + | xargs -I {} bash -c "aws ecr get-login-password | docker login --username AWS --password-stdin {}" + +aws ecr describe-repositories \ + | jq '.repositories | map(.repositoryUri)' \ + | grep $TARGET_PROJECT_NAME | grep $TARGET_ENVIRONMENT \ + | sed -E 's/"|,//g' \ + | xargs -I {} docker build -t {}:latest ../wfs3-app/ + +aws ecr describe-repositories \ + | jq '.repositories | map(.repositoryUri)' \ + | grep $TARGET_PROJECT_NAME | grep $TARGET_ENVIRONMENT \ + | sed -E 's/"|,//g' \ + | xargs -I {} docker images --format "{{json . }}" {} \ + | grep '"Tag":"latest"' \ + | jq '"\(.Repository):\(.Tag)"' \ + | xargs -I{} docker push {} \ No newline at end of file diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100644 index 0000000..4b495c1 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,86 @@ +#! /bin/bash +# Check .env file + + +DOT_ENV=$1 + +if [ -f $DOT_ENV ] +then + set -a; source $DOT_ENV; set +a +else + echo "Run: ./scripts/deploy.sh <.env_file>" + echo "Please create $DOT_ENV file first and try again" + exit 1 +fi + +function create_state_bucket { + # $1 region + # $2 bucket_name + + aws s3 mb s3://$2 --region $1 + aws s3api put-bucket-versioning \ + --bucket $2 \ + --versioning-configuration Status=Enabled +} + +function generate_terraform_variables { + tf_vars=(tf tfvars) + for tf_var in "${tf_vars[@]}"; do + ( + echo "cat < terraform.${tf_var} + done + +} + +function check_create_remote_state { + # $1 aws_region + # $2 bucket name + # $3 dynamotable_name + AWS_REGION=$1 + STATE_BUCKET_NAME=$2 + + bucketstatus=$(aws s3api head-bucket --bucket $STATE_BUCKET_NAME 2>&1) + + if echo "${bucketstatus}" | grep 'Not Found'; + then + echo "Creating TF remote state" + create_state_bucket $AWS_REGION $STATE_BUCKET_NAME + elif echo "${bucketstatus}" | grep 'Forbidden'; + then + echo "Bucket $STATE_BUCKET_NAME exists but not owned" + exit 1 + elif echo "${bucketstatus}" | grep 'Bad Request'; + then + echo "Bucket $STATE_BUCKET_NAME specified is less than 3 or greater than 63 characters" + exit 1 + else + echo "State Bucket $STATE_BUCKET_NAME owned and exists. Continue..."; + fi +} + + +cd ./terraform/features-api +generate_terraform_variables +check_create_remote_state $AWS_REGION $STATE_BUCKET_NAME + +read -rp 'action [init|plan|deploy]: ' ACTION +case $ACTION in + init) + terraform init + ;; + plan) + terraform plan + ;; + + deploy) + terraform apply --auto-approve + ;; + *) + echo "Choose from 'init', 'plan' or 'deploy'" + exit 1 + ;; +esac + diff --git a/scripts/sync-env.sh b/scripts/sync-env.sh new file mode 100755 index 0000000..c13e14c --- /dev/null +++ b/scripts/sync-env.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Use this script to load environment variables for a deployment from AWS Secrets +echo Loading environment secrets from $1 +aws secretsmanager get-secret-value --secret-id $1 --query SecretString --output text | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" > .env diff --git a/terraform/veda-wfs3/.gitignore b/terraform/features-api/.gitignore similarity index 100% rename from terraform/veda-wfs3/.gitignore rename to terraform/features-api/.gitignore diff --git a/terraform/features-api/.terraform.lock.hcl b/terraform/features-api/.terraform.lock.hcl new file mode 100644 index 0000000..16eeaeb --- /dev/null +++ b/terraform/features-api/.terraform.lock.hcl @@ -0,0 +1,63 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.1.0" + constraints = ">= 3.29.0, ~> 5.0" + hashes = [ + "h1:iDyYmwv8q94Dvr4DRG1KBxTWPZRFkRmKGa3cjCEsPZU=", + "zh:0c48f157b804c1f392adb5c14b81e756c652755e358096300ea8dd1283021129", + "zh:1a50495a6c0e5665e51df57dac6e781ec71439b11ebf05f971b6f3a3eb4eb7b2", + "zh:2959ff472c05e56d59e012118dd8d55022f005534c0ae961ce81136de9f66a4d", + "zh:2dfda9133581b99ed6e709e89a453fd2974ce88c703d3e073ec31bf99d7508ce", + "zh:2f3d92cc7a6624da42cee2202f8fb23e6d38f156ab7851884d637282cb0dc709", + "zh:3bc2a34d09cbaf439a1815846904f070c782cd8dfd60b5e0116827cda25f7549", + "zh:4ef43f1a247aa8de8690ac3bbc2b00ebaf6b2872fc8d0f5130e4a8130c874b87", + "zh:5477cb272dcaeb0030091bcf23a9f0f33b5410e44e317e9d3d49446f545dbaa4", + "zh:734c8fb4c0b79c82dd757566761dda5b91ee1ef9a2b848a748ade11e0e1cc69f", + "zh:80346c051b677f4f018da7fe06318b87c5bd0f1ec67ce78ab33baed3bb8b031a", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a865b2f88dfee13df14116c5cf53d033d2c15855f4b59b9c65337309a928df2c", + "zh:c0345f266eedaece5612c1000722b302f895d1bc5af1d5a4265f0e7000ca48bb", + "zh:d59703c8e6a9d8b4fbd3b4583b945dfff9cb2844c762c0b3990e1cef18282279", + "zh:d8d04a6a6cd2dfcb23b57e551db7b15e647f6166310fb7d883d8ec67bdc9bdc8", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.1" + hashes = [ + "h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=", + "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", + "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", + "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", + "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", + "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", + "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", + "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", + "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", + "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", + "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.5.1" + hashes = [ + "h1:IL9mSatmwov+e0+++YX2V6uel+dV6bn+fC/cnGDK3Ck=", + "zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64", + "zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d", + "zh:4d2b219d09abf3b1bb4df93d399ed156cadd61f44ad3baf5cf2954df2fba0831", + "zh:6130bdde527587bbe2dcaa7150363e96dbc5250ea20154176d82bc69df5d4ce3", + "zh:6cc326cd4000f724d3086ee05587e7710f032f94fc9af35e96a386a1c6f2214f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:b6d88e1d28cf2dfa24e9fdcc3efc77adcdc1c3c3b5c7ce503a423efbdd6de57b", + "zh:ba74c592622ecbcef9dc2a4d81ed321c4e44cddf7da799faa324da9bf52a22b2", + "zh:c7c5cde98fe4ef1143bd1b3ec5dc04baf0d4cc3ca2c5c7d40d17c0e9b2076865", + "zh:dac4bad52c940cd0dfc27893507c1e92393846b024c5a9db159a93c534a3da03", + "zh:de8febe2a2acd9ac454b844a4106ed295ae9520ef54dc8ed2faf29f12716b602", + "zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014", + ] +} diff --git a/terraform/features-api/dns.tf b/terraform/features-api/dns.tf new file mode 100644 index 0000000..ebde0e6 --- /dev/null +++ b/terraform/features-api/dns.tf @@ -0,0 +1,34 @@ +# data "aws_route53_zone" "zone" { +# provider = aws.west2 +# name = var.dns_zone_name +# } + +# resource "aws_acm_certificate" "cert" { +# provider = aws.west2 +# domain_name = "*.${data.aws_route53_zone.zone.name}" +# validation_method = "DNS" +# tags = var.tags + +# lifecycle { +# create_before_destroy = true +# } +# } + +# resource "aws_route53_record" "subdomain_record" { +# provider = aws.west2 +# name = "${var.dns_subdomain}.${data.aws_route53_zone.zone.name}" +# zone_id = data.aws_route53_zone.zone.id +# type = "A" + +# alias { +# name = aws_alb.alb_ecs.dns_name +# zone_id = aws_alb.alb_ecs.zone_id +# evaluate_target_health = true +# } +# } + +# resource "aws_lb_listener_certificate" "cert" { +# provider = aws.west2 +# listener_arn = aws_alb_listener.alb_listener_ecs.arn +# certificate_arn = aws_acm_certificate.cert.arn +# } \ No newline at end of file diff --git a/terraform/features-api/ecr.tf b/terraform/features-api/ecr.tf new file mode 100644 index 0000000..64a239d --- /dev/null +++ b/terraform/features-api/ecr.tf @@ -0,0 +1,57 @@ +module "ecr_registry_wfs" { + source = "github.com/developmentseed/tf-seed/modules/aws_ecr" + environment = var.env + registry_name = var.project_name + enable_registry_scanning = true + mutable_image_tags = true + enable_deploy_user = false + iam_deploy_username = null + tags = var.tags +} + +module "ecr_registry_db" { + source = "github.com/developmentseed/tf-seed/modules/aws_ecr" + environment = var.env + registry_name = "${var.project_name}-db" + enable_registry_scanning = true + mutable_image_tags = true + enable_deploy_user = false + iam_deploy_username = null + tags = var.tags +} + +resource "null_resource" "build_ecr_image_wfs" { + triggers = { + folder_path = sha1(join("", [for f in fileset("../../wfs3-app", "**") : filesha1("../../wfs3-app/${f}")])) + # handler_file_path = filemd5("../../wfs3-app/fast_api_main.py") + # docker_file_path = filemd5("../../wfs3-app/Dockerfile") + } + + provisioner "local-exec" { + command = < 0 ? jsonencode(var.container_command) : "" - working_directory = var.container_working_directory - container_secrets = jsonencode(var.container_secrets) - container_environment = jsonencode(var.container_environment) - service_protocol = var.service_protocol - service_port = var.service_port - use_adot_as_sidecar = var.use_adot_as_sidecar ? "on" : "" - log_group = aws_cloudwatch_log_group.service.name - region = var.region - } -} - resource "aws_ecs_task_definition" "service" { family = "tf-${var.service_name}-${var.environment}" requires_compatibilities = ["FARGATE"] @@ -239,7 +220,22 @@ resource "aws_ecs_task_definition" "service" { tags = var.tags execution_role_arn = aws_iam_role.ecs_execution_role.arn task_role_arn = aws_iam_role.ecs_execution_role.arn - container_definitions = data.template_file.container_definition.rendered + container_definitions = templatefile("${path.module}/container_definition.tftpl", + { + service_name = var.service_name + environment = var.environment + image = var.image + container_command = length(var.container_command) > 0 ? jsonencode(var.container_command) : "" + working_directory = var.container_working_directory + container_secrets = jsonencode(var.container_secrets) + container_environment = jsonencode(var.container_environment) + service_protocol = var.service_protocol + service_port = var.service_port + use_adot_as_sidecar = var.use_adot_as_sidecar ? "on" : "" + log_group = aws_cloudwatch_log_group.service.name + region = var.region + } + ) } ####################################################################### diff --git a/terraform/modules/aws_ecs_service/variables.tf b/terraform/modules/aws_ecs_service/variables.tf index bba749f..ecf52ea 100755 --- a/terraform/modules/aws_ecs_service/variables.tf +++ b/terraform/modules/aws_ecs_service/variables.tf @@ -11,7 +11,9 @@ variable "tags" { } variable "service_name" {} -variable "service_port" {} +variable "service_port" { + type = number +} variable "service_protocol" { type = string @@ -114,6 +116,8 @@ variable "use_adot_as_sidecar" { } variable "ecr_repository_name" {} +variable "ecr_repository_arn" {} + variable "image" {} variable "load_balancer" { @@ -131,3 +135,4 @@ variable "lb_type" { variable "lb_target_group_arn" {} variable "lb_security_group_id" {} variable "lb_container_port" {} +variable "permissions_boundary_policy_name" {} \ No newline at end of file diff --git a/terraform/veda-wfs3/.terraform.lock.hcl b/terraform/veda-wfs3/.terraform.lock.hcl deleted file mode 100644 index a758db9..0000000 --- a/terraform/veda-wfs3/.terraform.lock.hcl +++ /dev/null @@ -1,80 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/archive" { - version = "2.3.0" - hashes = [ - "h1:pTPG9Kf1Qg2aPsZLXDa6OvLqsEXaMrKnp0Z4Q/TIBPA=", - "zh:0869128d13abe12b297b0cd13b8767f10d6bf047f5afc4215615aabc39c2eb4f", - "zh:481ed837d63ba3aa45dd8736da83e911e3509dee0e7961bf5c00ed2644f807b3", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:9f08fe2977e2166849be24fb9f394e4d2697414d463f7996fd0d7beb4e19a29c", - "zh:9fe566deeafd460d27999ca0bbfd85426a5fcfcb40007b23884deb76da127b6f", - "zh:a1bd9a60925d9769e0da322e4523330ee86af9dc2e770cba1d0247a999ef29cb", - "zh:bb4094c8149f74308b22a87e1ac19bcccca76e8ef021b571074d9bccf1c0c6f0", - "zh:c8984c9def239041ce41ec8e19bbd76a49e74ed2024ff736dad60429dee89bcc", - "zh:ea4bb5ae73db1de3a586e62f39106f5e56770804a55aa5e6b4f642df973e0e75", - "zh:f44a9d596ecc3a8c5653f56ba0cd202ad93b49f76767f4608daf7260b813289e", - "zh:f5c5e6cc9f7f070020ab7d95fcc9ed8e20d5cf219978295a71236e22cbb6d508", - "zh:fd2273f51dcc8f43403bf1e425ba9db08a57c3ddcba5ad7a51742ccde21ca611", - ] -} - -provider "registry.terraform.io/hashicorp/aws" { - version = "4.58.0" - constraints = "~> 4.0" - hashes = [ - "h1:znLROwEAINbYzAG5X7Ep04whM7KxkQGrvhFdhSvNKEk=", - "zh:14b2b2dfbc7ee705c412d762b1485ee08958c816a64ac74f5769e946e4a1d265", - "zh:17a37e6825e2023b18987d31c0cbb9336654ea146b68e6c90710ea4636af71ae", - "zh:273127c69fb244577e5c136c46164d34f77b0c956c18d27f63d1072dd558f924", - "zh:4b2b6416d34fb3e1051c99d2a84045b136976140e34381d5fbf90e32db15272e", - "zh:7e6a8571ff15d51f892776265642ee01004b8553fd4f6f2014b6f3f2834670c7", - "zh:847c76ab2381b66666d0f79cf1ac697b5bfd0d9c3009fd11bc6ad6545d1eb427", - "zh:9a52cae08ba8d27d0639a8d2b8c61591027883058bf0cc5a639cffe1e299f019", - "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:9df647e8322d6f94f1843366ba39d21c4b36c8e7dcdc03711d52e27f73b0e974", - "zh:9e52037e68409802ff913b166c30e3f2035af03865cbef0c1b03762bce853941", - "zh:a30288e7c3c904d6998d1709835d7c5800a739f8608f0837f960286a2b8b6e59", - "zh:a7f24e3bda3be566468e4ad62cef1016f68c6f5a94d2e3e979485bc05626281b", - "zh:ba326ba80f5e39829b67a6d1ce54ba52b171e5e13a0a91ef5f9170a9b0cc9ce4", - "zh:c4e3fe9f2be6e244a3dfce599f4b0be9e8fffaece64cbc65f3195f825f65489b", - "zh:f20a251af37039bb2c7612dbd2c5df3a25886b4cc78f902385a2850ea6e30d08", - ] -} - -provider "registry.terraform.io/hashicorp/random" { - version = "3.4.3" - hashes = [ - "h1:tL3katm68lX+4lAncjQA9AXL4GR/VM+RPwqYf4D2X8Q=", - "zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752", - "zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b", - "zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:84103eae7251384c0d995f5a257c72b0096605048f757b749b7b62107a5dccb3", - "zh:8ee974b110adb78c7cd18aae82b2729e5124d8f115d484215fd5199451053de5", - "zh:9dd4561e3c847e45de603f17fa0c01ae14cae8c4b7b4e6423c9ef3904b308dda", - "zh:bb07bb3c2c0296beba0beec629ebc6474c70732387477a65966483b5efabdbc6", - "zh:e891339e96c9e5a888727b45b2e1bb3fcbdfe0fd7c5b4396e4695459b38c8cb1", - "zh:ea4739860c24dfeaac6c100b2a2e357106a89d18751f7693f3c31ecf6a996f8d", - "zh:f0c76ac303fd0ab59146c39bc121c5d7d86f878e9a69294e29444d4c653786f8", - "zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93", - ] -} - -provider "registry.terraform.io/hashicorp/template" { - version = "2.2.0" - hashes = [ - "h1:0wlehNaxBX7GJQnPfQwTNvvAf38Jm0Nv7ssKGMaG6Og=", - "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", - "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", - "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", - "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", - "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", - "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", - "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", - "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", - "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", - "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", - ] -} diff --git a/terraform/veda-wfs3/dns.tf b/terraform/veda-wfs3/dns.tf deleted file mode 100644 index 9ad8883..0000000 --- a/terraform/veda-wfs3/dns.tf +++ /dev/null @@ -1,34 +0,0 @@ -data "aws_route53_zone" "zone" { - provider = aws.west2 - name = var.dns_zone_name -} - -resource "aws_acm_certificate" "cert" { - provider = aws.west2 - domain_name = "*.${data.aws_route53_zone.zone.name}" - validation_method = "DNS" - tags = var.tags - - lifecycle { - create_before_destroy = true - } -} - -resource "aws_route53_record" "subdomain_record" { - provider = aws.west2 - name = "${var.dns_subdomain}.${data.aws_route53_zone.zone.name}" - zone_id = data.aws_route53_zone.zone.id - type = "A" - - alias { - name = aws_alb.alb_ecs.dns_name - zone_id = aws_alb.alb_ecs.zone_id - evaluate_target_health = true - } -} - -resource "aws_lb_listener_certificate" "cert" { - provider = aws.west2 - listener_arn = aws_alb_listener.alb_listener_ecs.arn - certificate_arn = aws_acm_certificate.cert.arn -} \ No newline at end of file diff --git a/terraform/veda-wfs3/ecr.tf b/terraform/veda-wfs3/ecr.tf deleted file mode 100644 index 5d8d7ab..0000000 --- a/terraform/veda-wfs3/ecr.tf +++ /dev/null @@ -1,10 +0,0 @@ -module "ecr_registry" { - source = "github.com/developmentseed/tf-seed/modules/aws_ecr" - environment = var.env - registry_name = var.registry_name - enable_registry_scanning = true - mutable_image_tags = true - enable_deploy_user = true - iam_deploy_username = aws_iam_user.deploy_user.name - tags = var.tags -} \ No newline at end of file diff --git a/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/lambda_function.py b/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/lambda_function.py deleted file mode 100644 index d524d83..0000000 --- a/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/lambda_function.py +++ /dev/null @@ -1,57 +0,0 @@ -import boto3 -import http.client -import os -import base64 -import ast -import json -mwaa_env_name = 'veda-pipeline-staging-mwaa' -dag_name = 'veda_discover' -mwaa_cli_command = 'dags trigger' -client = boto3.client('mwaa') - - -def lambda_handler(event, context): - for record in event['Records']: - print(f"[ RECORD ]: {record}") - s3_event_key = record['s3']['object']['key'] - print(f"[ S3 EVENT KEY ]: {s3_event_key}") - s3_filename_target = os.path.split(s3_event_key)[-1] - print(f"[ S3 FILENAME TARGET ]: {s3_filename_target}") - s3_filename_no_ext = os.path.splitext(s3_filename_target)[0] - print(f"[ S3 FILENAME NO EXT ]: {s3_filename_no_ext}") - - bucket_key_prefix = "EIS/FEDSoutput/Snapshot/" - if s3_filename_no_ext.startswith("lf_"): - bucket_key_prefix = "EIS/FEDSoutput/LFArchive/" - - # get web token - mwaa_cli_token = client.create_cli_token( - Name=mwaa_env_name - ) - print(f"[ CLI TOKEN ]: {mwaa_cli_token}") - serialized_args = json.dumps({ - "discovery": "s3", - "collection": s3_filename_no_ext, - "prefix": bucket_key_prefix, - "bucket": "veda-data-store-staging", - "filename_regex": f"^(.*){s3_filename_target}$", - "vector": True - }) - conn = http.client.HTTPSConnection(mwaa_cli_token['WebServerHostname']) - payload = f"{mwaa_cli_command} {dag_name} --conf '{serialized_args}'" - print(f"[ CLI PAYLOAD ]: {payload}") - headers = { - 'Authorization': 'Bearer ' + mwaa_cli_token['CliToken'], - 'Content-Type': 'text/plain' - } - conn.request("POST", "/aws_mwaa/cli", payload, headers) - res = conn.getresponse() - data = res.read() - dict_str = data.decode("UTF-8") - mydata = ast.literal_eval(dict_str) - print(f"[ DATA ]: {mydata}") - print(f"[ STDOUT ]: {base64.b64decode(mydata['stdout'])}") - return { - 'statusCode': 200, - 'body': json.dumps('Hello from Lambda!') - } \ No newline at end of file diff --git a/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/requirements.txt b/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/requirements.txt deleted file mode 100644 index 1db657b..0000000 --- a/terraform/veda-wfs3/functions/s3_event_bridge_to_sfn_execute/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -boto3 \ No newline at end of file diff --git a/terraform/veda-wfs3/github_deploy_user.tf b/terraform/veda-wfs3/github_deploy_user.tf deleted file mode 100644 index ad737c6..0000000 --- a/terraform/veda-wfs3/github_deploy_user.tf +++ /dev/null @@ -1,32 +0,0 @@ -resource "aws_iam_user" "deploy_user" { - name = "veda-wfs3-${var.env}-deploy-user" - path = "/" - tags = var.tags -} - -// NOTE: we need to have extra policies added to our -// deploy user for Github AWS Actions to work -resource "aws_iam_user_policy" "deploy" { - name = "${var.registry_name}_deploy_extended" - user = aws_iam_user.deploy_user.name - policy = data.aws_iam_policy_document.extended_deploy.json -} - -data "aws_iam_policy_document" "extended_deploy" { - statement { - actions = [ - "iam:PassRole", - "ecr:InitiateLayerUpload", - "ecs:RegisterTaskDefinition", - "ecs:DescribeServices", - "ecs:UpdateService", - ] - - resources = [ - module.ecr_registry.registry_arn, - module.ecs_cluster.service_cluster_arn, - module.ecs_cluster.service_arn, - module.ecs_cluster.ecs_execution_role_arn, - ] - } -} \ No newline at end of file diff --git a/terraform/veda-wfs3/rds.tf b/terraform/veda-wfs3/rds.tf deleted file mode 100644 index e75f047..0000000 --- a/terraform/veda-wfs3/rds.tf +++ /dev/null @@ -1,65 +0,0 @@ -resource "aws_db_subnet_group" "db" { - name = "tf-${var.project_name}-${var.env}-subnet-group" - subnet_ids = module.networking.private_subnets_id - tags = { - Name = "tf-${var.project_name}-subnet-group" - } -} - -resource "aws_db_parameter_group" "default" { - name = "tf-${var.project_name}-${var.env}-postgres14-param-group" - family = "postgres14" - - parameter { - name = "work_mem" - # NOTE: I had `work_mem` set to ~100MB and `max_connections` around 75 and TileJSON completely failed - # 16MB - value = var.env == "staging" ? "16384" : "8192" - } - - parameter { - name = "max_connections" - value = "475" - apply_method = "pending-reboot" - } - -# NOTE: here to show what shared_buffers are but doesn't really make sense why it won't provision with these -# parameter { -# name = "shared_buffers" -# value = var.env == "staging" ? "8064856" : "4032428" -# apply_method = "pending-reboot" -# } - - parameter { - name = "seq_page_cost" - value = "1" - } - - parameter { - name = "random_page_cost" - value = "1.2" - } -} - -resource "aws_db_instance" "db" { - db_name = "veda" - identifier = "${var.project_name}-${var.env}" - engine = "postgres" - engine_version = "14.3" - // https://docs.aws.amazon.com/AmazonRDS/latest/APIReference/API_CreateDBInstance.html - allocated_storage = 100 - max_allocated_storage = 500 - storage_type = "gp2" - instance_class = var.env == "staging" ? "db.r5.xlarge" : "db.r5.large" - db_subnet_group_name = aws_db_subnet_group.db.name - vpc_security_group_ids = module.networking.security_groups_ids - skip_final_snapshot = true - apply_immediately = true - backup_retention_period = 7 - username = "postgres" - password = var.db_password - allow_major_version_upgrade = true - parameter_group_name = aws_db_parameter_group.default.name -} - - diff --git a/terraform/veda-wfs3/s3_event_bridge_lambda.tf b/terraform/veda-wfs3/s3_event_bridge_lambda.tf deleted file mode 100644 index 77ae894..0000000 --- a/terraform/veda-wfs3/s3_event_bridge_lambda.tf +++ /dev/null @@ -1,167 +0,0 @@ -##################################################### -# Execution Role -##################################################### -resource "aws_iam_role" "lambda_exec_role" { - provider = aws.west2 - name = "lambda-exec-role-s3-event-bridge-${var.project_name}-${var.env}" - tags = var.tags - - assume_role_policy = <