Skip to content

Commit 87caee3

Browse files
Villagers654Villagers654
committed
Merge branch 'modpack-v2' into launcher
# Conflicts: # .github/scripts/cloudflare-r2-preview.mjs # .github/workflows/ci-cd.yml # .github/workflows/pr-preview.yml # mock-db/scripts/generate-from-mongo.mjs
2 parents a3c1d22 + 6c9a29e commit 87caee3

9 files changed

Lines changed: 297 additions & 163 deletions

File tree

‎.github/scripts/cloudflare-r2-preview.mjs‎

Lines changed: 63 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import { createHash } from "node:crypto";
44

55
const API_BASE = "https://api.cloudflare.com/client/v4";
66
const R2_BUCKET_ITEM_WRITE = "Workers R2 Storage Bucket Item Write";
7+
const TOKEN_OWNER_ACCOUNT = "account";
8+
const TOKEN_OWNER_USER = "user";
79

810
const mode = process.argv[2];
911

@@ -46,6 +48,30 @@ function endpointFor(accountId, jurisdiction) {
4648
return `https://${accountId}.r2.cloudflarestorage.com`;
4749
}
4850

51+
function tokenOwner() {
52+
const owner = optional("CLOUDFLARE_API_TOKEN_OWNER", TOKEN_OWNER_ACCOUNT);
53+
if (![TOKEN_OWNER_ACCOUNT, TOKEN_OWNER_USER].includes(owner)) {
54+
throw new Error(
55+
`Unsupported CLOUDFLARE_API_TOKEN_OWNER '${owner}'. Use '${TOKEN_OWNER_ACCOUNT}' or '${TOKEN_OWNER_USER}'.`,
56+
);
57+
}
58+
return owner;
59+
}
60+
61+
function tokenCollectionPath(accountId) {
62+
return tokenOwner() === TOKEN_OWNER_USER
63+
? "/user/tokens"
64+
: `/accounts/${accountId}/tokens`;
65+
}
66+
67+
function tokenItemPath(accountId, tokenId) {
68+
return `${tokenCollectionPath(accountId)}/${encodeURIComponent(tokenId)}`;
69+
}
70+
71+
function tokenPermissionGroupsPath(accountId) {
72+
return `${tokenCollectionPath(accountId)}/permission_groups`;
73+
}
74+
4975
function bucketResource(accountId, jurisdiction, bucketName) {
5076
return `com.cloudflare.edge.r2.bucket.${accountId}_${jurisdiction}_${bucketName}`;
5177
}
@@ -68,6 +94,25 @@ function appendGitHubEnv(values) {
6894
});
6995
}
7096

97+
async function writeShellEnvFile(values) {
98+
const envFile = optional("R2_RUNTIME_ENV_FILE");
99+
if (!envFile) {
100+
return;
101+
}
102+
103+
const { appendFileSync } = await import("node:fs");
104+
const lines = Object.entries(values).map(([key, value]) => {
105+
const escaped = String(value).replace(/'/g, "'\\''");
106+
return `${key}='${escaped}'`;
107+
});
108+
appendFileSync(envFile, `${lines.join("\n")}\n`, "utf8");
109+
}
110+
111+
async function exportRuntimeValues(values) {
112+
await appendGitHubEnv(values);
113+
await writeShellEnvFile(values);
114+
}
115+
71116
async function cloudflare(token, path, init = {}) {
72117
const response = await fetch(`${API_BASE}${path}`, {
73118
...init,
@@ -150,21 +195,24 @@ async function ensureBucket(accountId, token, bucketName, jurisdiction) {
150195
console.log(`Created R2 bucket '${bucketName}'.`);
151196
}
152197

153-
async function permissionGroupId(token, permissionName) {
154-
const payload = await cloudflare(token, "/user/tokens/permission_groups");
198+
async function permissionGroupId(accountId, token, permissionName) {
199+
const payload = await cloudflare(
200+
token,
201+
`${tokenPermissionGroupsPath(accountId)}?name=${encodeURIComponent(permissionName)}`,
202+
);
155203
const match = payload.result?.find((group) => group.name === permissionName);
156204
if (!match?.id) {
157205
throw new Error(`Cloudflare permission group '${permissionName}' was not found.`);
158206
}
159207
return match.id;
160208
}
161209

162-
async function revokeToken(token, tokenId, reason) {
210+
async function revokeToken(accountId, token, tokenId, reason) {
163211
if (!tokenId) {
164212
return;
165213
}
166214
try {
167-
await cloudflare(token, `/user/tokens/${encodeURIComponent(tokenId)}`, {
215+
await cloudflare(token, tokenItemPath(accountId, tokenId), {
168216
method: "DELETE",
169217
});
170218
console.log(`Revoked Cloudflare token '${tokenId}'${reason ? ` (${reason})` : ""}.`);
@@ -178,7 +226,7 @@ async function revokeToken(token, tokenId, reason) {
178226
}
179227

180228
async function createRuntimeToken(accountId, token, bucketName, jurisdiction) {
181-
const groupId = await permissionGroupId(token, R2_BUCKET_ITEM_WRITE);
229+
const groupId = await permissionGroupId(accountId, token, R2_BUCKET_ITEM_WRITE);
182230
const tokenName = optional("R2_TOKEN_NAME", `modtale preview ${bucketName}`);
183231
const expiresOn = optional("R2_TOKEN_EXPIRES_ON");
184232

@@ -199,7 +247,7 @@ async function createRuntimeToken(accountId, token, bucketName, jurisdiction) {
199247
body.expires_on = expiresOn;
200248
}
201249

202-
const payload = await cloudflare(token, "/user/tokens", {
250+
const payload = await cloudflare(token, tokenCollectionPath(accountId), {
203251
method: "POST",
204252
body: JSON.stringify(body),
205253
});
@@ -235,6 +283,7 @@ async function provision() {
235283

236284
await ensureBucket(accountId, bucketToken, bucketName, jurisdiction);
237285
await revokeToken(
286+
accountId,
238287
tokenProvisioner,
239288
optional("EXISTING_R2_RUNTIME_TOKEN_ID"),
240289
"replacing runtime token",
@@ -247,7 +296,7 @@ async function provision() {
247296
jurisdiction,
248297
);
249298

250-
await appendGitHubEnv({
299+
await exportRuntimeValues({
251300
R2_RUNTIME_ACCESS_KEY: credentials.accessKey,
252301
R2_RUNTIME_SECRET_KEY: credentials.secretKey,
253302
R2_RUNTIME_TOKEN_ID: credentials.tokenId,
@@ -267,7 +316,7 @@ async function ensureOnly() {
267316
const jurisdiction = normalizeJurisdiction(optional("CLOUDFLARE_R2_JURISDICTION"));
268317

269318
await ensureBucket(accountId, bucketToken, bucketName, jurisdiction);
270-
await appendGitHubEnv({
319+
await exportRuntimeValues({
271320
R2_RUNTIME_ENDPOINT: endpointFor(accountId, jurisdiction),
272321
});
273322
}
@@ -301,7 +350,12 @@ async function cleanup() {
301350
}
302351

303352
if (tokenProvisioner) {
304-
await revokeToken(tokenProvisioner, optional("R2_RUNTIME_TOKEN_ID"), "preview cleanup");
353+
await revokeToken(
354+
accountId,
355+
tokenProvisioner,
356+
optional("R2_RUNTIME_TOKEN_ID"),
357+
"preview cleanup",
358+
);
305359
}
306360
}
307361

‎.github/workflows/ci-cd.yml‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ jobs:
5353
5454
if [ "$BRANCH_NAME" = "main" ]; then
5555
echo "ENV_TYPE=prod" >> $GITHUB_ENV
56+
echo "BUILD_SERVICE_ACCOUNT=${{ vars.GCP_BUILD_SERVICE_ACCOUNT }}" >> $GITHUB_ENV
57+
echo "RUNTIME_SERVICE_ACCOUNT=${{ vars.GCP_PROD_RUNTIME_SERVICE_ACCOUNT }}" >> $GITHUB_ENV
5658
echo "BACKEND_SERVICE=modtale-backend" >> $GITHUB_ENV
5759
echo "FRONTEND_SERVICE=modtale-frontend" >> $GITHUB_ENV
5860
echo "DB_NAME=modtale" >> $GITHUB_ENV
@@ -76,6 +78,8 @@ jobs:
7678
7779
elif [ "$BRANCH_NAME" = "develop" ]; then
7880
echo "ENV_TYPE=dev" >> $GITHUB_ENV
81+
echo "BUILD_SERVICE_ACCOUNT=${{ vars.GCP_BUILD_SERVICE_ACCOUNT }}" >> $GITHUB_ENV
82+
echo "RUNTIME_SERVICE_ACCOUNT=${{ vars.GCP_DEV_RUNTIME_SERVICE_ACCOUNT }}" >> $GITHUB_ENV
7983
echo "BACKEND_SERVICE=modtale-backend-dev" >> $GITHUB_ENV
8084
echo "FRONTEND_SERVICE=modtale-frontend-dev" >> $GITHUB_ENV
8185
echo "DB_NAME=modtale" >> $GITHUB_ENV
@@ -103,6 +107,7 @@ jobs:
103107
SLUG="${GITHUB_SHA:0:7}"
104108
fi
105109
echo "ENV_TYPE=preview" >> $GITHUB_ENV
110+
echo "BUILD_SERVICE_ACCOUNT=${{ vars.GCP_BUILD_SERVICE_ACCOUNT }}" >> $GITHUB_ENV
106111
echo "BACKEND_SERVICE=modtale-backend-$SLUG" >> $GITHUB_ENV
107112
echo "FRONTEND_SERVICE=modtale-frontend-$SLUG" >> $GITHUB_ENV
108113
echo "DB_NAME=modtale-$SLUG" >> $GITHUB_ENV
@@ -180,7 +185,13 @@ jobs:
180185
echo "::add-mask::$existing_token_id"
181186
export EXISTING_R2_RUNTIME_TOKEN_ID="$existing_token_id"
182187
188+
runtime_env_file="$(mktemp)"
189+
trap 'rm -f "$runtime_env_file"' EXIT
190+
export R2_RUNTIME_ENV_FILE="$runtime_env_file"
183191
node .github/scripts/cloudflare-r2-preview.mjs provision
192+
set -a
193+
. "$runtime_env_file"
194+
set +a
184195
185196
upsert_secret() {
186197
local name="$1"
@@ -239,13 +250,35 @@ jobs:
239250
if: steps.filter.outputs.backend == 'true' || env.BACKEND_EXISTS == 'false'
240251
run: |
241252
cd backend
253+
254+
: "${BUILD_SERVICE_ACCOUNT:?Set GitHub environment variable GCP_BUILD_SERVICE_ACCOUNT.}"
255+
: "${RUNTIME_SERVICE_ACCOUNT:?Set the GitHub environment runtime service account variable for this environment.}"
256+
257+
WARDEN_URL_VALUE=""
258+
if [ "${{ env.ENV_TYPE }}" = "prod" ] || [ "${{ env.ENV_TYPE }}" = "dev" ]; then
259+
WARDEN_URL_VALUE="$(
260+
gcloud secrets versions access latest \
261+
--project "$PROJECT_ID" \
262+
--secret WARDEN_URL
263+
)"
264+
265+
if [ -z "$WARDEN_URL_VALUE" ]; then
266+
echo "::error::Secret WARDEN_URL is empty or unavailable."
267+
exit 1
268+
fi
269+
270+
echo "::add-mask::$WARDEN_URL_VALUE"
271+
fi
272+
242273
gcloud builds submit \
243274
--config=cloudbuild.yml \
275+
--service-account="projects/$PROJECT_ID/serviceAccounts/$BUILD_SERVICE_ACCOUNT" \
244276
--substitutions=_TAG=${{ env.TAG }} .
245277
246278
ARGS=(
247279
"--image" "gcr.io/$PROJECT_ID/modtale-backend:${{ env.TAG }}"
248280
"--region" "$REGION"
281+
"--service-account" "$RUNTIME_SERVICE_ACCOUNT"
249282
"--allow-unauthenticated"
250283
"--use-http2"
251284
"--memory" "1Gi"
@@ -268,7 +301,7 @@ jobs:
268301
ARGS+=("--update-secrets" "R2_ENDPOINT=${{ env.R2_ENDPOINT_SECRET_NAME }}:latest")
269302
270303
if [ "${{ env.ENV_TYPE }}" = "prod" ]; then
271-
ARGS+=("--update-secrets" "WARDEN_URL=WARDEN_URL:latest")
304+
ARGS+=("--update-env-vars" "WARDEN_URL=$WARDEN_URL_VALUE")
272305
ARGS+=("--update-secrets" "SMTP_HOST=SMTP_HOST:latest")
273306
ARGS+=("--update-secrets" "SMTP_USERNAME=SMTP_USERNAME:latest")
274307
ARGS+=("--update-secrets" "SMTP_PASSWORD=SMTP_PASSWORD:latest")
@@ -292,7 +325,7 @@ jobs:
292325
ARGS+=("--update-secrets" "ADMIN_DISCORD_WEBHOOK_URL=ADMIN_DISCORD_WEBHOOK_URL:latest")
293326
ARGS+=("--update-secrets" "STATUS_DISCORD_WEBHOOK_URL=STATUS_DISCORD_WEBHOOK_URL:latest")
294327
elif [ "${{ env.ENV_TYPE }}" = "dev" ]; then
295-
ARGS+=("--update-secrets" "WARDEN_URL=WARDEN_URL:latest")
328+
ARGS+=("--update-env-vars" "WARDEN_URL=$WARDEN_URL_VALUE")
296329
ARGS+=("--update-secrets" "SMTP_HOST=SMTP_HOST:latest")
297330
ARGS+=("--update-secrets" "SMTP_USERNAME=SMTP_USERNAME:latest")
298331
ARGS+=("--update-secrets" "SMTP_PASSWORD=SMTP_PASSWORD:latest")
@@ -312,7 +345,6 @@ jobs:
312345
ARGS+=("--update-secrets" "WIKI_API_KEY=WIKI_API_KEY:latest")
313346
else
314347
: "${RUNTIME_SERVICE_ACCOUNT:?Set environment variable GCP_BRANCH_PREVIEW_RUNTIME_SERVICE_ACCOUNT in the branch-preview GitHub environment.}"
315-
ARGS+=("--service-account" "$RUNTIME_SERVICE_ACCOUNT")
316348
ARGS+=("--update-secrets" "APP_SEEDING_SOURCE_R2_ACCESS_KEY=${{ env.SEEDING_SOURCE_R2_ACCESS_KEY_SECRET_NAME }}:latest")
317349
ARGS+=("--update-secrets" "APP_SEEDING_SOURCE_R2_SECRET_KEY=${{ env.SEEDING_SOURCE_R2_SECRET_KEY_SECRET_NAME }}:latest")
318350
ARGS+=("--update-secrets" "APP_SEEDING_SOURCE_R2_ENDPOINT=${{ env.SEEDING_SOURCE_R2_ENDPOINT_SECRET_NAME }}:latest")
@@ -343,13 +375,19 @@ jobs:
343375
if: steps.filter.outputs.frontend == 'true' || env.FRONTEND_EXISTS == 'false'
344376
run: |
345377
cd frontend
378+
379+
: "${BUILD_SERVICE_ACCOUNT:?Set GitHub environment variable GCP_BUILD_SERVICE_ACCOUNT.}"
380+
: "${RUNTIME_SERVICE_ACCOUNT:?Set the GitHub environment runtime service account variable for this environment.}"
381+
346382
gcloud builds submit \
347383
--config=cloudbuild.yml \
384+
--service-account="projects/$PROJECT_ID/serviceAccounts/$BUILD_SERVICE_ACCOUNT" \
348385
--substitutions=_PUBLIC_API_URL=$API_URL,_TAG=$TAG .
349386
350387
gcloud run deploy $FRONTEND_SERVICE \
351388
--image gcr.io/$PROJECT_ID/modtale-frontend:$TAG \
352389
--region $REGION \
390+
--service-account "$RUNTIME_SERVICE_ACCOUNT" \
353391
--allow-unauthenticated \
354392
--set-env-vars "PUBLIC_API_URL=$API_URL" \
355393
--format="value(status.url)" > frontend_url.txt

0 commit comments

Comments
 (0)