Skip to content

Fix S3 env variable handling #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Leksidechoice wants to merge 1 commit into
base: v1
Choose a base branch
from
Open

Fix S3 env variable handling #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 40 additions & 16 deletions app/services/s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,28 +20,52 @@ class S3Service:

def __init__(self):
"""Initialize S3 service."""
# Get AWS credentials from environment variables
self.aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID")
self.aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY")
self.bucket_name = os.environ.get("AWS_BUCKET_NAME")
self.region = os.environ.get("AWS_REGION")
# Get AWS/S3 credentials from environment variables
# Prefer the standard AWS_* variables but fall back to S3_* for
# compatibility with the provided `.env.example` and docker compose.
self.aws_access_key_id = (
os.environ.get("AWS_ACCESS_KEY_ID")
or os.environ.get("S3_ACCESS_KEY")
)
self.aws_secret_access_key = (
os.environ.get("AWS_SECRET_ACCESS_KEY")
or os.environ.get("S3_SECRET_KEY")
)
self.bucket_name = (
os.environ.get("AWS_BUCKET_NAME")
or os.environ.get("S3_BUCKET_NAME")
)
self.region = os.environ.get("AWS_REGION") or os.environ.get("S3_REGION")

# Thread pool for handling blocking S3 operations
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=5)

# Print environment variables for debugging (without sensitive info)
logger.debug(f"AWS_ACCESS_KEY_ID: {'*' * 8 if self.aws_access_key_id else 'Not Set'}")
logger.debug(f"AWS_SECRET_ACCESS_KEY: {'*' * 8 if self.aws_secret_access_key else 'Not Set'}")
logger.debug(f"AWS_BUCKET_NAME: {self.bucket_name}")
logger.debug(f"AWS_REGION: {self.region}")

# Check if credentials are provided
using_dummy_credentials = (self.aws_access_key_id == "dummy_access_key_id" or
self.aws_secret_access_key == "dummy_secret_access_key" or
self.bucket_name == "dummy-bucket-name")
logger.debug(
f"AWS/S3 ACCESS_KEY_ID: {'*' * 8 if self.aws_access_key_id else 'Not Set'}"
)
logger.debug(
f"AWS/S3 SECRET_ACCESS_KEY: {'*' * 8 if self.aws_secret_access_key else 'Not Set'}"
)
logger.debug(f"S3 BUCKET_NAME: {self.bucket_name}")
logger.debug(f"S3 REGION: {self.region}")

if using_dummy_credentials:
logger.warning("Using dummy AWS credentials. S3 uploads will return mock URLs instead.")
# Check if credentials are provided or if dummy values are used
missing_creds = (
not self.aws_access_key_id
or not self.aws_secret_access_key
or not self.bucket_name
)
using_dummy_credentials = (
self.aws_access_key_id == "dummy_access_key_id"
or self.aws_secret_access_key == "dummy_secret_access_key"
or self.bucket_name == "dummy-bucket-name"
)

if missing_creds or using_dummy_credentials:
logger.warning(
"AWS/S3 credentials are missing or dummy. S3 uploads will return mock URLs instead."
)
self.s3_client = None
return

Expand Down
24 changes: 12 additions & 12 deletions app/services/video/concatenate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -358,10 +358,10 @@ def check_s3_configuration() -> Dict[str, Any]:

# Get environment variables (masked for security)
env_vars = {
"AWS_REGION": os.environ.get("AWS_REGION", "Not set"),
"AWS_BUCKET_NAME": os.environ.get("AWS_BUCKET_NAME", "Not set"),
"AWS_ACCESS_KEY_ID": "***" if os.environ.get("AWS_ACCESS_KEY_ID") else "Not set",
"AWS_SECRET_ACCESS_KEY": "***" if os.environ.get("AWS_SECRET_ACCESS_KEY") else "Not set"
"AWS_REGION": os.environ.get("AWS_REGION") or os.environ.get("S3_REGION", "Not set"),
"AWS_BUCKET_NAME": os.environ.get("AWS_BUCKET_NAME") or os.environ.get("S3_BUCKET_NAME", "Not set"),
"AWS_ACCESS_KEY_ID": "***" if (os.environ.get("AWS_ACCESS_KEY_ID") or os.environ.get("S3_ACCESS_KEY")) else "Not set",
"AWS_SECRET_ACCESS_KEY": "***" if (os.environ.get("AWS_SECRET_ACCESS_KEY") or os.environ.get("S3_SECRET_KEY")) else "Not set"
}

# Details based on initialization status
Expand All @@ -371,19 +371,19 @@ def check_s3_configuration() -> Dict[str, Any]:
details = "S3 storage is not initialized. Check AWS credentials and bucket configuration."

# Add more specific diagnostics
if not env_vars["AWS_REGION"]:
details += " AWS_REGION is not set."
if not env_vars["AWS_REGION"] or env_vars["AWS_REGION"] == "Not set":
details += " AWS_REGION/S3_REGION is not set."

if not env_vars["AWS_BUCKET_NAME"]:
details += " No bucket name is set (check AWS_BUCKET_NAME)."
if not env_vars["AWS_BUCKET_NAME"] or env_vars["AWS_BUCKET_NAME"] == "Not set":
details += " No bucket name is set (check S3_BUCKET_NAME or AWS_BUCKET_NAME)."
else:
details += f" Using AWS_BUCKET_NAME='{bucket_name}'."

if not os.environ.get("AWS_ACCESS_KEY_ID"):
details += " AWS_ACCESS_KEY_ID is not set."
if not (os.environ.get("AWS_ACCESS_KEY_ID") or os.environ.get("S3_ACCESS_KEY")):
details += " AWS_ACCESS_KEY_ID/S3_ACCESS_KEY is not set."

if not os.environ.get("AWS_SECRET_ACCESS_KEY"):
details += " AWS_SECRET_ACCESS_KEY is not set."
if not (os.environ.get("AWS_SECRET_ACCESS_KEY") or os.environ.get("S3_SECRET_KEY")):
details += " AWS_SECRET_ACCESS_KEY/S3_SECRET_KEY is not set."

return {
"initialized": is_initialized,
Expand Down