diff --git a/apps/faf-icebreaker/templates/config.yaml b/apps/faf-icebreaker/templates/config.yaml index 3f8c43ff..5890ab6e 100644 --- a/apps/faf-icebreaker/templates/config.yaml +++ b/apps/faf-icebreaker/templates/config.yaml @@ -14,10 +14,11 @@ data: RABBITMQ_HOST: "rabbitmq" RABBITMQ_USER: "faf-icebreaker" RABBITMQ_PORT: "5672" - CLOUDFLARE_ENABLED: "false" + CLOUDFLARE_ENABLED: "true" XIRSYS_ENABLED: "true" XIRSYS_TURN_ENABLED: "true" GEOIPUPDATE_EDITION_IDS: "GeoLite2-City" LOKI_BASE_URL: "http://monitoring-loki-gateway.faf-ops.svc" - FORCE_RELAY: "false" - QUARKUS_LOG_CATEGORY__COM_FAFOREVER__LEVEL: "DEBUG" \ No newline at end of file + FORCE_RELAY: "true" + REAL_IP_HEADER: "Cf-Connecting-Ip" + QUARKUS_LOG_CATEGORY__COM_FAFOREVER__LEVEL: "TRACE" \ No newline at end of file diff --git a/apps/faf-icebreaker/templates/deployment.yaml b/apps/faf-icebreaker/templates/deployment.yaml index f38ed8ef..6c0e0c1d 100644 --- a/apps/faf-icebreaker/templates/deployment.yaml +++ b/apps/faf-icebreaker/templates/deployment.yaml @@ -32,7 +32,7 @@ spec: - name: geolite-db mountPath: /usr/share/GeoIP containers: - - image: faforever/faf-icebreaker:1.1.9 + - image: faforever/faf-icebreaker:1.2.0-RC4 imagePullPolicy: Always name: faf-icebreaker envFrom: diff --git a/apps/faf-icebreaker/templates/local-secret.yaml b/apps/faf-icebreaker/templates/local-secret.yaml index 31d072e5..481deae6 100644 --- a/apps/faf-icebreaker/templates/local-secret.yaml +++ b/apps/faf-icebreaker/templates/local-secret.yaml @@ -13,6 +13,7 @@ stringData: RABBITMQ_PASSWORD: "banana" XIRSYS_IDENT: "banana" XIRSYS_SECRET: "banana" + HETZNER_API_KEY: "banana" JWT_PRIVATE_KEY_PATH: |- -----BEGIN PRIVATE KEY----- MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDXsCsl9W0vnW2k diff --git a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt index 4e3859cb..4d4747c1 100755 --- a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt +++ b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt @@ -1,7 +1,12 @@ -import org.apache.commons.compress.archivers.zip.Zip64Mode +@file:Suppress("PackageDirectoryMismatch") + +package com.faforever.coopdeployer + +import com.faforever.FafDatabase +import com.faforever.GitRepo +import com.faforever.Log import org.apache.commons.compress.archivers.zip.ZipArchiveEntry import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream -import org.eclipse.jgit.api.Git import org.slf4j.LoggerFactory import java.io.IOException import java.net.URI @@ -15,12 +20,7 @@ import java.nio.file.StandardCopyOption import java.nio.file.attribute.FileTime import java.nio.file.attribute.PosixFilePermission import java.security.MessageDigest -import java.sql.Connection -import java.sql.DriverManager import java.time.Duration -import java.util.zip.CRC32 -import java.util.zip.ZipEntry -import java.util.zip.ZipOutputStream import kotlin.io.path.inputStream private val log = LoggerFactory.getLogger("CoopDeployer") @@ -34,33 +34,6 @@ fun Path.setPerm664() { Files.setPosixFilePermissions(this, perms) } -data class FeatureModGitRepo( - val workDir: Path, - val repoUrl: String, - val gitRef: String, -) { - fun checkout(): Path { - if (Files.exists(workDir.resolve(".git"))) { - log.info("Repo exists — fetching and checking out $gitRef...") - Git.open(workDir.toFile()).use { git -> - git.fetch().call() - git.checkout().setName(gitRef).call() - } - } else { - log.info("Cloning repository $repoUrl") - Git.cloneRepository() - .setURI(repoUrl) - .setDirectory(workDir.toFile()) - .call() - log.info("Checking out $gitRef") - Git.open(workDir.toFile()).use { git -> - git.checkout().setName(gitRef).call() - } - } - - return workDir - } -} data class GithubReleaseAssetDownloader( val repoOwner: String = "FAForever", @@ -160,25 +133,14 @@ data class GithubReleaseAssetDownloader( } -data class FafDatabase( - val host: String, - val database: String, - val username: String, - val password: String, +data class CoopDatabase( val dryRun: Boolean -) : AutoCloseable { +) : FafDatabase() { /** * Definition of an existing file in the database */ data class PatchFile(val mod: String, val fileId: Int, val name: String, val md5: String, val version: Int) - private val connection: Connection = - DriverManager.getConnection( - "jdbc:mariadb://$host/$database?useSSL=false&serverTimezone=UTC", - username, - password - ) - fun getCurrentPatchFile(mod: String, fileId: Int): PatchFile? { val sql = """ SELECT uf.fileId, uf.name, uf.md5, t.v @@ -191,7 +153,7 @@ data class FafDatabase( WHERE uf.fileId = ? """.trimIndent() - connection.prepareStatement(sql).use { stmt -> + prepareStatement(sql).use { stmt -> stmt.setInt(1, fileId) val rs = stmt.executeQuery() while (rs.next()) { @@ -213,12 +175,12 @@ data class FafDatabase( } val del = "DELETE FROM updates_${mod}_files WHERE fileId=? AND version=?" val ins = "INSERT INTO updates_${mod}_files (fileId, version, name, md5, obselete) VALUES (?, ?, ?, ?, 0)" - connection.prepareStatement(del).use { + prepareStatement(del).use { it.setInt(1, fileId) it.setInt(2, version) it.executeUpdate() } - connection.prepareStatement(ins).use { + prepareStatement(ins).use { it.setInt(1, fileId) it.setInt(2, version) it.setString(3, name) @@ -226,10 +188,6 @@ data class FafDatabase( it.executeUpdate() } } - - override fun close() { - connection.close() - } } private const val MINIMUM_ZIP_DATE = 315532800000L // 1980-01-01 @@ -238,7 +196,7 @@ private val MINIMUM_ZIP_FILE_TIME = FileTime.fromMillis(MINIMUM_ZIP_DATE) class Patcher( val patchVersion: Int, val targetDir: Path, - val db: FafDatabase, + val db: CoopDatabase, val dryRun: Boolean, ) { /** @@ -409,22 +367,18 @@ class Patcher( } fun main() { + Log.init() + val PATCH_VERSION = System.getenv("PATCH_VERSION") ?: error("PATCH_VERSION required") val REPO_URL = System.getenv("GIT_REPO_URL") ?: "https://github.com/FAForever/fa-coop.git" val GIT_REF = System.getenv("GIT_REF") ?: "v$PATCH_VERSION" - val WORKDIR = System.getenv("GIT_WORKDIR") ?: "/tmp/fa-coop-kt" + val WORKDIR = System.getenv("GIT_WORKDIR") ?: "/tmp/fa-coop" val DRYRUN = (System.getenv("DRY_RUN") ?: "false").lowercase() in listOf("1", "true", "yes") - - val DB_HOST = System.getenv("DATABASE_HOST") ?: "localhost" - val DB_NAME = System.getenv("DATABASE_NAME") ?: "faf" - val DB_USER = System.getenv("DATABASE_USERNAME") ?: "root" - val DB_PASS = System.getenv("DATABASE_PASSWORD") ?: "banana" - val TARGET_DIR = Paths.get("./legacy-featured-mod-files") log.info("=== Kotlin Coop Deployer v{} ===", PATCH_VERSION) - val repo = FeatureModGitRepo( + val repo = GitRepo( workDir = Paths.get(WORKDIR), repoUrl = REPO_URL, gitRef = GIT_REF @@ -476,13 +430,7 @@ fun main() { Patcher.PatchFile(25, "FAF_Coop_Operation_Tight_Spot_VO.v%d.nx2", null), ) - FafDatabase( - host = DB_HOST, - database = DB_NAME, - username = DB_USER, - password = DB_PASS, - dryRun = DRYRUN - ).use { db -> + CoopDatabase(dryRun = DRYRUN).use { db -> val patcher = Patcher( patchVersion = PATCH_VERSION.toInt(), targetDir = TARGET_DIR, diff --git a/apps/faf-legacy-deployment/scripts/CoopMapDeployer.kt b/apps/faf-legacy-deployment/scripts/CoopMapDeployer.kt new file mode 100644 index 00000000..238fd48f --- /dev/null +++ b/apps/faf-legacy-deployment/scripts/CoopMapDeployer.kt @@ -0,0 +1,238 @@ +@file:Suppress("PackageDirectoryMismatch") + +package com.faforever.coopmapdeployer + +import com.faforever.FafDatabase +import com.faforever.GitRepo +import com.faforever.Log +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream +import org.slf4j.LoggerFactory +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.attribute.FileTime +import java.security.MessageDigest +import kotlin.io.path.copyTo +import kotlin.io.path.createDirectories +import kotlin.io.path.exists +import kotlin.io.path.isDirectory +import kotlin.io.path.isRegularFile +import kotlin.io.path.readBytes +import kotlin.io.path.readText +import kotlin.io.path.walk + +private val log = LoggerFactory.getLogger("coop-maps-updater") + +private const val FIXED_TIMESTAMP = 1078100502L // 2004-03-01T00:21:42Z +private val FIXED_FILE_TIME = FileTime.fromMillis(FIXED_TIMESTAMP) + + +data class CoopMap( + val folderName: String, + val mapId: Int, + val mapType: Int +) { + fun zipName(version: Int) = + "${folderName.lowercase()}.v${version.toString().padStart(4, '0')}.zip" + + fun folderName(version: Int) = + "${folderName.lowercase()}.v${version.toString().padStart(4, '0')}" +} + +private val coopMaps = listOf( + CoopMap("X1CA_Coop_001", 1, 0), + CoopMap("X1CA_Coop_002", 3, 0), + CoopMap("X1CA_Coop_003", 4, 0), + CoopMap("X1CA_Coop_004", 5, 0), + CoopMap("X1CA_Coop_005", 6, 0), + CoopMap("X1CA_Coop_006", 7, 0), + + CoopMap("SCCA_Coop_A01", 8, 1), + CoopMap("SCCA_Coop_A02", 9, 1), + CoopMap("SCCA_Coop_A03", 10, 1), + CoopMap("SCCA_Coop_A04", 11, 1), + CoopMap("SCCA_Coop_A05", 12, 1), + CoopMap("SCCA_Coop_A06", 13, 1), + + CoopMap("SCCA_Coop_R01", 20, 2), + CoopMap("SCCA_Coop_R02", 21, 2), + CoopMap("SCCA_Coop_R03", 22, 2), + CoopMap("SCCA_Coop_R04", 23, 2), + CoopMap("SCCA_Coop_R05", 24, 2), + CoopMap("SCCA_Coop_R06", 25, 2), + + CoopMap("SCCA_Coop_E01", 14, 3), + CoopMap("SCCA_Coop_E02", 15, 3), + CoopMap("SCCA_Coop_E03", 16, 3), + CoopMap("SCCA_Coop_E04", 17, 3), + CoopMap("SCCA_Coop_E05", 18, 3), + CoopMap("SCCA_Coop_E06", 19, 3), + + CoopMap("FAF_Coop_Prothyon_16", 26, 4), + CoopMap("FAF_Coop_Fort_Clarke_Assault", 27, 4), + CoopMap("FAF_Coop_Theta_Civilian_Rescue", 28, 4), + CoopMap("FAF_Coop_Novax_Station_Assault", 31, 4), + CoopMap("FAF_Coop_Operation_Tha_Atha_Aez", 32, 4), + CoopMap("FAF_Coop_Havens_Invasion", 33, 4), + CoopMap("FAF_Coop_Operation_Rescue", 35, 4), + CoopMap("FAF_Coop_Operation_Uhthe_Thuum_QAI", 36, 4), + CoopMap("FAF_Coop_Operation_Yath_Aez", 37, 4), + CoopMap("FAF_Coop_Operation_Ioz_Shavoh_Kael", 38, 4), + CoopMap("FAF_Coop_Operation_Trident", 39, 4), + CoopMap("FAF_Coop_Operation_Blockade", 40, 4), + CoopMap("FAF_Coop_Operation_Golden_Crystals", 41, 4), + CoopMap("FAF_Coop_Operation_Holy_Raid", 42, 4), + CoopMap("FAF_Coop_Operation_Tight_Spot", 45, 4), + CoopMap("FAF_Coop_Operation_Overlord_Surth_Velsok", 47, 4), + CoopMap("FAF_Coop_Operation_Rebels_Rest", 48, 4), + CoopMap("FAF_Coop_Operation_Red_Revenge", 49, 4), +) + +data class CoopMapDatabase( + val dryRun: Boolean +) : FafDatabase() { + fun getLatestVersion(map: CoopMap): Int { + createStatement().use { st -> + st.executeQuery("SELECT version FROM coop_map WHERE id=${map.mapId}") + .use { rs -> + if (!rs.next()) error("Map ${map.mapId} not found") + return rs.getInt(1) + } + } + } + + fun update(map: CoopMap, version: Int) { + val sql = """ + UPDATE coop_map + SET version=$version, + filename='maps/${map.zipName(version)}' + WHERE id=${map.mapId} + """.trimIndent() + + createStatement().use { it.executeUpdate(sql) } + } +} + +private fun processCoopMap( + db: CoopMapDatabase, + map: CoopMap, + simulate: Boolean, + gitDir: String, + mapsDir: String +) { + log.info("Processing $map") + + val tmp = Files.createTempDirectory("coop-map") + try { + Files.walk(Path.of(gitDir, map.folderName)).forEach { + val target = tmp.resolve(Path.of(gitDir, map.folderName).relativize(it)) + if (it.isDirectory()) target.createDirectories() + else it.copyTo(target) + } + + val files = tmp.walk().filter { it.isRegularFile() }.toList() + val currentVersion = db.getLatestVersion(map) + + val currentZip = Path.of(mapsDir, map.zipName(currentVersion)) + val tmpZip = tmp.resolve(map.zipName(currentVersion)) + + createZip(map, currentVersion, files, tmp, tmpZip) + + val changed = currentVersion == 0 || + !currentZip.exists() || + md5(currentZip) != md5(tmpZip) + + if (!changed) { + log.info("$map unchanged") + return + } + + val newVersion = currentVersion + 1 + log.info("$map updated → v$newVersion") + + if (!simulate) { + val finalZip = Path.of(mapsDir, map.zipName(newVersion)) + createZip(map, newVersion, files, tmp, finalZip) + db.update(map, newVersion) + } + } finally { + tmp.toFile().deleteRecursively() + } +} + +private fun createZip( + map: CoopMap, + version: Int, + files: List, + base: Path, + out: Path +) { + ZipArchiveOutputStream(out.toFile()).use { zip -> + zip.setMethod(ZipArchiveEntry.DEFLATED) + + files.forEach { file -> + val rel = base.relativize(file) + val entryPath = "/${map.folderName(version)}/$rel" + + val bytes = file.readText() + .replace( + "/maps/${map.folderName}/", + "/maps/${map.folderName(version)}/" + ).toByteArray() + + val entry = ZipArchiveEntry(entryPath).apply { + // Ensure deterministic times + setTime(FIXED_FILE_TIME) + setCreationTime(FIXED_FILE_TIME) + setLastModifiedTime(FIXED_FILE_TIME) + setLastAccessTime(FIXED_FILE_TIME) + + size = bytes.size.toLong() + } + + zip.putArchiveEntry(entry) + zip.write(bytes) + zip.closeArchiveEntry() + } + + zip.finish() + } +} + +private fun md5(path: Path): String { + val md = MessageDigest.getInstance("MD5") + md.update(path.readBytes()) + return md.digest().joinToString("") { "%02x".format(it) } +} + +fun main(args: Array) { + Log.init() + + val MAP_DIR = System.getenv("MAP_DIR") ?: "/opt/faf/data/faf-coop-maps" + val PATCH_VERSION = System.getenv("PATCH_VERSION") ?: error("PATCH_VERSION required") + val REPO_URL = System.getenv("GIT_REPO_URL") ?: "https://github.com/FAForever/faf-coop-maps" + val GIT_REF = System.getenv("GIT_REF") ?: "v$PATCH_VERSION" + val WORKDIR = System.getenv("GIT_WORKDIR") ?: "/tmp/faf-coop-maps" + val DRYRUN = (System.getenv("DRY_RUN") ?: "false").lowercase() in listOf("1", "true", "yes") + + log.info("=== Kotlin Coop Map Deployer v{} ===", PATCH_VERSION) + + Files.createDirectories(Paths.get(MAP_DIR)) + + GitRepo( + workDir = Paths.get(WORKDIR), + repoUrl = REPO_URL, + gitRef = GIT_REF, + ).checkout() + + CoopMapDatabase(dryRun = DRYRUN).use { db -> + coopMaps.forEach { + try { + processCoopMap(db, it, DRYRUN, WORKDIR, MAP_DIR) + } catch (e: Exception) { + log.warn("Failed processing $it", e) + } + } + } +} diff --git a/apps/faf-legacy-deployment/scripts/Utils.kt b/apps/faf-legacy-deployment/scripts/Utils.kt new file mode 100644 index 00000000..8b0b9b5a --- /dev/null +++ b/apps/faf-legacy-deployment/scripts/Utils.kt @@ -0,0 +1,74 @@ +package com.faforever + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.Logger +import org.eclipse.jgit.api.Git +import org.slf4j.LoggerFactory +import java.nio.file.Files +import java.nio.file.Path +import java.sql.Connection +import java.sql.DriverManager +import java.sql.PreparedStatement +import java.sql.Statement + +object Log { + fun init() { + val level = System.getenv("LOG_LEVEL") ?: "INFO" + val root = LoggerFactory + .getLogger(Logger.ROOT_LOGGER_NAME) as Logger + root.level = Level.toLevel(level, Level.INFO) + } +} + +data class GitRepo( + val workDir: Path, + val repoUrl: String, + val gitRef: String, +) { + private val log = LoggerFactory.getLogger(GitRepo::class.simpleName) + + fun checkout(): Path { + if (Files.exists(workDir.resolve(".git"))) { + log.info("Repo exists — fetching and checking out $gitRef...") + Git.open(workDir.toFile()).use { git -> + git.fetch().call() + git.checkout().setName(gitRef).call() + } + } else { + log.info("Cloning repository $repoUrl") + Git.cloneRepository() + .setURI(repoUrl) + .setDirectory(workDir.toFile()) + .call() + log.info("Checking out $gitRef") + Git.open(workDir.toFile()).use { git -> + git.checkout().setName(gitRef).call() + } + } + + return workDir + } +} + +abstract class FafDatabase : AutoCloseable { + private val host = System.getenv("DATABASE_HOST") ?: "localhost" + private val database = System.getenv("DATABASE_NAME") ?: "faf" + private val username = System.getenv("DATABASE_USERNAME") ?: "root" + private val password = System.getenv("DATABASE_PASSWORD") ?: "banana" + + private val connection: Connection = + DriverManager.getConnection( + "jdbc:mariadb://$host/$database?useSSL=false&serverTimezone=UTC", + username, + password + ) + + fun createStatement(): Statement = connection.createStatement() + + fun prepareStatement(sql: String): PreparedStatement = connection.prepareStatement(sql) + + override fun close() { + connection.close() + } +} + diff --git a/apps/faf-legacy-deployment/scripts/build.gradle.kts b/apps/faf-legacy-deployment/scripts/build.gradle.kts index 10f0c44b..de560733 100644 --- a/apps/faf-legacy-deployment/scripts/build.gradle.kts +++ b/apps/faf-legacy-deployment/scripts/build.gradle.kts @@ -12,11 +12,7 @@ dependencies { implementation("org.eclipse.jgit:org.eclipse.jgit:7.5.0.202512021534-r") implementation("org.apache.commons:commons-compress:1.28.0") implementation("org.slf4j:slf4j-api:2.0.13") - runtimeOnly("ch.qos.logback:logback-classic:1.5.23") -} - -application { - mainClass.set("CoopDeployerKt") // filename + Kt + implementation("ch.qos.logback:logback-classic:1.5.23") } // Use the root level for files @@ -24,4 +20,20 @@ sourceSets { main { kotlin.srcDirs(".") } +} + +tasks.register("deployCoop") { + group = "application" + description = "Deploy coop" + + classpath = sourceSets.main.get().runtimeClasspath + mainClass.set("com.faforever.coopdeployer.CoopDeployerKt") +} + +tasks.register("deployCoopMaps") { + group = "application" + description = "Deploy coop maps" + + classpath = sourceSets.main.get().runtimeClasspath + mainClass.set("com.faforever.coopmapdeployer.CoopMapDeployerKt") } \ No newline at end of file diff --git a/apps/faf-legacy-deployment/scripts/deploy-coop-maps.py b/apps/faf-legacy-deployment/scripts/deploy-coop-maps.py deleted file mode 100644 index b28badef..00000000 --- a/apps/faf-legacy-deployment/scripts/deploy-coop-maps.py +++ /dev/null @@ -1,381 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*- - -""" -clone: https://github.com/FAForever/faf-coop-maps - -FAF coop maps updater - -All default settings are setup for FAF production! -Override the directory settings for local testing. -To get more help run - $ pipenv run patch-coop-maps -h - -Default usage: - $ pipenv run patch-coop-maps -s -""" -import argparse -import hashlib -import logging -import os -import shutil -import subprocess -import sys -import zipfile -from tempfile import TemporaryDirectory -from typing import NamedTuple, List - -import mysql.connector - -logger: logging.Logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - -fixed_file_timestamp = 1078100502 # 2004-03-01T00:21:42Z - - -db_config = { - "host": os.getenv("DATABASE_HOST", "localhost"), - "user": os.getenv("DATABASE_USERNAME", "root"), - "password": os.getenv("DATABASE_PASSWORD", "banana"), - "database": os.getenv("DATABASE_NAME", "faf_lobby"), -} - - -def get_db_connection(): - """Create and return a MySQL connection.""" - try: - conn = mysql.connector.connect(**db_config) - if conn.is_connected(): - logger.debug(f"Connected to MySQL at {db_config['host']}") - return conn - except Error as e: - logger.error(f"MySQL connection failed: {e}") - sys.exit(1) - - -def run_sql(conn, sql: str) -> str: - """ - Run an SQL query directly on the MySQL database instead of via Docker. - Returns output in a string format similar to the old implementation. - """ - logger.debug(f"Executing SQL query:\n{sql}") - try: - with conn.cursor() as cursor: - cursor.execute(sql) - - # If it's a SELECT query, fetch and format results - if sql.strip().lower().startswith("select"): - rows = cursor.fetchall() - column_names = [desc[0] for desc in cursor.description] - # Simulate the Docker mysql CLI tabular text output - lines = ["\t".join(column_names)] - for row in rows: - lines.append("\t".join(str(x) for x in row)) - result = "\n".join(lines) - else: - conn.commit() - result = "Query OK" - - logger.debug(f"SQL result:\n{result}") - return result - - except Error as e: - logger.error(f"SQL execution failed: {e}") - sys.exit(1) - - -class CoopMap(NamedTuple): - folder_name: str - map_id: int - map_type: int - - def build_zip_filename(self, version: int) -> str: - return f"{self.folder_name.lower()}.v{version:04d}.zip" - - def build_folder_name(self, version: int) -> str: - return f"{self.folder_name.lower()}.v{version:04d}" - - -# Coop maps are in db table `coop_map` -coop_maps: List[CoopMap] = [ - # Forged Alliance missions - CoopMap("X1CA_Coop_001", 1, 0), - CoopMap("X1CA_Coop_002", 3, 0), - CoopMap("X1CA_Coop_003", 4, 0), - CoopMap("X1CA_Coop_004", 5, 0), - CoopMap("X1CA_Coop_005", 6, 0), - CoopMap("X1CA_Coop_006", 7, 0), - - # Vanilla Aeon missions - CoopMap("SCCA_Coop_A01", 8, 1), - CoopMap("SCCA_Coop_A02", 9, 1), - CoopMap("SCCA_Coop_A03", 10, 1), - CoopMap("SCCA_Coop_A04", 11, 1), - CoopMap("SCCA_Coop_A05", 12, 1), - CoopMap("SCCA_Coop_A06", 13, 1), - - # Vanilla Cybran missions - CoopMap("SCCA_Coop_R01", 20, 2), - CoopMap("SCCA_Coop_R02", 21, 2), - CoopMap("SCCA_Coop_R03", 22, 2), - CoopMap("SCCA_Coop_R04", 23, 2), - CoopMap("SCCA_Coop_R05", 24, 2), - CoopMap("SCCA_Coop_R06", 25, 2), - - # Vanilla UEF missions - CoopMap("SCCA_Coop_E01", 14, 3), - CoopMap("SCCA_Coop_E02", 15, 3), - CoopMap("SCCA_Coop_E03", 16, 3), - CoopMap("SCCA_Coop_E04", 17, 3), - CoopMap("SCCA_Coop_E05", 18, 3), - CoopMap("SCCA_Coop_E06", 19, 3), - - # Custom missions - CoopMap("FAF_Coop_Prothyon_16", 26, 4), - CoopMap("FAF_Coop_Fort_Clarke_Assault", 27, 4), - CoopMap("FAF_Coop_Theta_Civilian_Rescue", 28, 4), - CoopMap("FAF_Coop_Novax_Station_Assault", 31, 4), - CoopMap("FAF_Coop_Operation_Tha_Atha_Aez", 32, 4), - CoopMap("FAF_Coop_Havens_Invasion", 33, 4), - CoopMap("FAF_Coop_Operation_Rescue", 35, 4), - CoopMap("FAF_Coop_Operation_Uhthe_Thuum_QAI", 36, 4), - CoopMap("FAF_Coop_Operation_Yath_Aez", 37, 4), - CoopMap("FAF_Coop_Operation_Ioz_Shavoh_Kael", 38, 4), - CoopMap("FAF_Coop_Operation_Trident", 39, 4), - CoopMap("FAF_Coop_Operation_Blockade", 40, 4), - CoopMap("FAF_Coop_Operation_Golden_Crystals", 41, 4), - CoopMap("FAF_Coop_Operation_Holy_Raid", 42, 4), - CoopMap("FAF_Coop_Operation_Tight_Spot", 45, 4), - CoopMap("FAF_Coop_Operation_Overlord_Surth_Velsok", 47, 4), - CoopMap("FAF_Coop_Operation_Rebel's_Rest", 48, 4), - CoopMap("FAF_Coop_Operation_Red_Revenge", 49, 4), -] - -def fix_file_timestamps(files: List[str]) -> None: - for file in files: - logger.debug(f"Fixing timestamp in {file}") - os.utime(file, (fixed_file_timestamp, fixed_file_timestamp)) - - -def fix_folder_paths(folder_name: str, files: List[str], new_version: int) -> None: - old_maps_lua_path = f"/maps/{folder_name}/" - new_maps_lua_path = f"/maps/{folder_name.lower()}.v{new_version:04d}/" - - for file in files: - logger.debug(f"Fixing lua folder path in {file}: '{old_maps_lua_path}' -> '{new_maps_lua_path}'") - - with open(file, "rb") as file_handler: - data = file_handler.read() - data = data.replace(old_maps_lua_path.encode(), new_maps_lua_path.encode()) - - with open(file, "wb") as file_handler: - file_handler.seek(0) - file_handler.write(data) - - -def get_latest_map_version(coop_map: CoopMap) -> int: - logger.debug(f"Fetching latest map version for coop map {coop_map}") - - query = f""" - SELECT version FROM coop_map WHERE id = {coop_map.map_id}; - """ - result = run_sql(query).split("\n") - assert len(result) == 3, f"Mysql returned wrong result! Either map id {coop_map.map_id} is not in table coop_map" \ - f" or the where clause is wrong. Result: " + "\n".join(result) - return int(result[1]) - - -def new_file_is_different(old_file_name: str, new_file_name: str) -> bool: - old_file_md5 = calc_md5(old_file_name) - new_file_md5 = calc_md5(new_file_name) - - logger.debug(f"MD5 hash of {old_file_name} is: {old_file_md5}") - logger.debug(f"MD5 hash of {new_file_name} is: {new_file_md5}") - - return old_file_md5 != new_file_md5 - - -def update_database(coop_map: CoopMap, new_version: int) -> None: - logger.debug(f"Updating coop map {coop_map} in database to version {new_version}") - - query = f""" - UPDATE coop_map - SET version = {new_version}, filename = "maps/{coop_map.build_zip_filename(new_version)}" - WHERE id = {coop_map.map_id} - """ - run_sql(query) - - -def copytree(src, dst, symlinks=False, ignore=None): - """ - Reason for that method is because shutil.copytree will raise exception on existing - temporary directory - """ - - for item in os.listdir(src): - s = os.path.join(src, item) - d = os.path.join(dst, item) - if os.path.isdir(s): - shutil.copytree(s, d, symlinks, ignore) - else: - shutil.copy2(s, d) - - -def create_zip_package(coop_map: CoopMap, version: int, files: List[str], tmp_folder_path: str, zip_file_path: str): - fix_folder_paths(coop_map.folder_name, files, version) - fix_file_timestamps(files) - with zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_BZIP2) as zip_file: - for path in files: - zip_file.write(path, arcname=f"/{coop_map.build_folder_name(version)}/{os.path.relpath(path, tmp_folder_path)}") - - -def process_coop_map(coop_map: CoopMap, simulate: bool, git_directory:str, coop_maps_path: str): - logger.info(f"Processing: {coop_map}") - - temp_dir = TemporaryDirectory() - copytree(os.path.join(git_directory, coop_map.folder_name), temp_dir.name) - processing_files = [] - for root, dirs, files in os.walk(temp_dir.name): - for f in files: - processing_files.append(os.path.relpath(os.path.join(root, f), temp_dir.name)) - - logger.debug(f"Files to process in {coop_map}: {processing_files}") - current_version = get_latest_map_version(coop_map) - current_file_path = os.path.join(coop_maps_path, coop_map.build_zip_filename(current_version)) - zip_file_path = os.path.join(temp_dir.name, coop_map.build_zip_filename(current_version)) - create_zip_package(coop_map, current_version, processing_files, temp_dir.name, zip_file_path) - if current_version == 0 or new_file_is_different(current_file_path, zip_file_path): - new_version = current_version + 1 - - if current_version == 0: - logger.info(f"{coop_map} first upload. New version: {new_version}") - else: - logger.info(f"{coop_map} has changed. New version: {new_version}") - - if not simulate: - temp_dir.cleanup() - temp_dir = TemporaryDirectory() - copytree(os.path.join(git_directory, coop_map.folder_name), temp_dir.name) - - zip_file_path = os.path.join(coop_maps_path, coop_map.build_zip_filename(new_version)) - create_zip_package(coop_map, new_version, processing_files, temp_dir.name, zip_file_path) - - update_database(coop_map, new_version) - else: - logger.info(f"Updating database skipped due to simulation") - else: - logger.info(f"{coop_map} remains unchanged") - temp_dir.cleanup() - - -def calc_md5(filename: str) -> str: - """ - Calculate the MD5 hash of a file - """ - hash_md5 = hashlib.md5() - with open(filename, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hash_md5.update(chunk) - return hash_md5.hexdigest() - - -def run_checked_shell(cmd: List[str]) -> subprocess.CompletedProcess: - """ - Runs a command as a shell process and checks for success - Output is captured in the result object - :param cmd: command to run - :return: CompletedProcess of the execution - """ - logger.debug("Run shell command: {cmd}".format(cmd=cmd)) - return subprocess.run(cmd, check=True, stdout=subprocess.PIPE) - - -def run_sql(sql: str, container: str = "faf-db", database: str = "faf_lobby") -> str: - - """ - Run a sql-query against the faf-db in the docker container - :param database: name of the database where to run the query - :param container: name of the docker container where to run the query - :param sql: the sql-query to run - :return: the query output as string - """ - try: - sql_text_result = run_checked_shell( - ["docker", "exec", "-u", "root", container, "mysql", database, "-e", sql] - ).stdout.decode() # type: str - logger.debug(f"SQL output >>> \n{sql_text_result}<<<") - return sql_text_result - except subprocess.CalledProcessError as e: - logger.error(f"""Executing sql query failed: {sql}\n\t\tError message: {str(e)}""") - exit(1) - - -def git_checkout(path: str, tag: str) -> None: - """ - Checkout a git tag of the git repository. This requires the repo to be checked out in the path folder! - - :param path: the path of the git repository to checkout - :param tag: version of the git tag (full name) - :return: nothing - """ - cwd = os.getcwd() - os.chdir(path) - logger.debug(f"Git checkout from path {path}") - - try: - run_checked_shell(["git", "fetch"]) - run_checked_shell(["git", "checkout", tag]) - except subprocess.CalledProcessError as e: - logger.error(f"git checkout failed - please check the error message: {e.stderr}") - exit(1) - finally: - os.chdir(cwd) - - -def create_zip(content: List[str], relative_to: str, output_file: str) -> None: - logger.debug(f"Zipping files to file `{output_file}`: {content}") - - with zipfile.ZipFile(output_file, 'w', zipfile.ZIP_DEFLATED) as zip_file: - for path in content: - if os.path.isdir(path): - cwd = os.getcwd() - os.chdir(path) - - for root, dirs, files in os.walk(path): - for next_file in files: - file_path = os.path.join(root, next_file) - zip_file.write(file_path, os.path.relpath(file_path, relative_to)) - - os.chdir(cwd) - else: - zip_file.write(path, os.path.relpath(path, relative_to)) - - -if __name__ == "__main__": - # Setting up logger - stream_handler = logging.StreamHandler(sys.stdout) - stream_handler.setFormatter(logging.Formatter('%(levelname)-5s - %(message)s')) - logger.addHandler(stream_handler) - - # Setting up CLI arguments - parser = argparse.ArgumentParser(description=__doc__) - - parser.add_argument("version", help="the git tag name of the version") - parser.add_argument("-s", "--simulate", dest="simulate", action="store_true", default=False, - help="only runs a simulation without updating the database") - parser.add_argument("--git-directory", dest="git_directory", action="store", - default="/opt/featured-mods/faf-coop-maps", - help="base directory of the faf-coop-maps repository") - parser.add_argument("--maps-directory", dest="coop_maps_path", action="store", - default="/opt/faf/data/maps", - help="directory of the coop map files (content server)") - - args = parser.parse_args() - - git_checkout(args.git_directory, args.version) - - for coop_map in coop_maps: - try: - process_coop_map(coop_map, args.simulate, args.git_directory, args.coop_maps_path) - except Exception as error: - logger.warning(f"Unable to parse {coop_map}", exc_info=True) diff --git a/apps/faf-legacy-deployment/templates/config.yaml b/apps/faf-legacy-deployment/templates/config.yaml new file mode 100644 index 00000000..4c282829 --- /dev/null +++ b/apps/faf-legacy-deployment/templates/config.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: faf-legacy-deployment + labels: + app: faf-legacy-deployment +data: + DATABASE_HOST: "mariadb" + DATABASE_NAME: "faf_lobby" diff --git a/apps/faf-legacy-deployment/templates/deploy-coop-maps.yaml b/apps/faf-legacy-deployment/templates/deploy-coop-maps.yaml index f1629a38..ce877034 100644 --- a/apps/faf-legacy-deployment/templates/deploy-coop-maps.yaml +++ b/apps/faf-legacy-deployment/templates/deploy-coop-maps.yaml @@ -1,18 +1,3 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: faf-deploy-coop-maps - labels: - app: faf-deploy-coop-maps -data: - PATCH_VERSION: "65" - DATABASE_HOST: "mariadb" - DATABASE_NAME: "faf_lobby" - "deploy-coop-maps.py": |- -{{ tpl ( .Files.Get "scripts/deploy-coop-maps.py" ) . | indent 4 }} - ---- - kind: CronJob apiVersion: batch/v1 metadata: @@ -35,22 +20,34 @@ spec: template: spec: containers: - - image: python:3.13 + - image: gradle:9.2-jdk21 imagePullPolicy: Always - name: faf-coop-deployment + name: faf-deploy-coop + workingDir: /workspace + env: + - name: PATCH_VERSION + value: "9.0.2" envFrom: - configMapRef: - name: faf-deploy-coop-maps + name: faf-legacy-deployment - secretRef: name: faf-legacy-deployment - command: [ "sh" ] - args: [ "-c", "pip install mysql-connector-python && python3 /tmp/deploy-coop-maps.py" ] + command: + - "sh" + - "-c" + - "cp /scripts/* /workspace && gradle deployCoopMaps" + # We need to mount single files via subpath because Gradle breaks otherwise (symbolic link to read-only directory) volumeMounts: - - mountPath: /tmp/deploy-coop-maps.py - name: faf-deploy-coop-maps - subPath: "deploy-coop-maps.py" + - mountPath: /scripts + name: faf-deploy-scripts + - mountPath: /workspace/legacy-featured-mod-files + name: faf-featured-mods restartPolicy: Never volumes: - - name: faf-deploy-coop-maps + - name: faf-deploy-scripts configMap: - name: "faf-deploy-coop-maps" + name: "faf-deploy-scripts" + - name: faf-featured-mods + hostPath: + path: /opt/faf/data/legacy-featured-mod-files + type: Directory \ No newline at end of file diff --git a/apps/faf-legacy-deployment/templates/deploy-coop.yaml b/apps/faf-legacy-deployment/templates/deploy-coop.yaml index c4c7a668..ac01eb55 100644 --- a/apps/faf-legacy-deployment/templates/deploy-coop.yaml +++ b/apps/faf-legacy-deployment/templates/deploy-coop.yaml @@ -1,16 +1,3 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: faf-deploy-coop - labels: - app: faf-deploy-coop -data: - PATCH_VERSION: "66" - DATABASE_HOST: "mariadb" - DATABASE_NAME: "faf_lobby" - ---- - kind: CronJob apiVersion: batch/v1 metadata: @@ -37,20 +24,22 @@ spec: imagePullPolicy: Always name: faf-deploy-coop workingDir: /workspace + env: + - name: PATCH_VERSION + value: "66" envFrom: - configMapRef: - name: faf-deploy-coop + name: faf-legacy-deployment - secretRef: name: faf-legacy-deployment - command: [ "gradle", "run" ] + command: + - "sh" + - "-c" + - "cp /scripts/* /workspace && gradle deployCoop" # We need to mount single files via subpath because Gradle breaks otherwise (symbolic link to read-only directory) volumeMounts: - - mountPath: /workspace/build.gradle.kts - name: faf-deploy-scripts - subPath: "build.gradle.kts" - - mountPath: /workspace/CoopDeployer.kt + - mountPath: /scripts name: faf-deploy-scripts - subPath: "CoopDeployer.kt" - mountPath: /workspace/legacy-featured-mod-files name: faf-featured-mods restartPolicy: Never diff --git a/apps/faf-lobby-server/config/config.yaml b/apps/faf-lobby-server/config/config.yaml index 5a5679e6..77514dc2 100644 --- a/apps/faf-lobby-server/config/config.yaml +++ b/apps/faf-lobby-server/config/config.yaml @@ -1,5 +1,5 @@ CONFIGURATION_REFRESH_TIME: 60 -LOG_LEVEL: "INFO" +LOG_LEVEL: "DEBUG" LISTEN: - ADDRESS: @@ -40,6 +40,9 @@ NEWBIE_TIME_BONUS: .25 MAXIMUM_NEWBIE_TIME_BONUS: 3.0 MINORITY_BONUS: 1 +LADDER_TOP_PLAYER_SEARCH_EXPANSION_MAX: 1.0 +LADDER_TOP_PLAYER_SEARCH_EXPANSION_STEP: 1.0 + QUEUE_POP_TIME_MAX: 90 # LADDER_VIOLATIONS_ENABLED: false \ No newline at end of file diff --git a/apps/faf-replay-server/config/config.yaml b/apps/faf-replay-server/config/config.yaml index c46329c7..8c15c806 100644 --- a/apps/faf-replay-server/config/config.yaml +++ b/apps/faf-replay-server/config/config.yaml @@ -17,7 +17,7 @@ storage: replay: forced_timeout_s: 18000 time_with_zero_writers_to_end_replay_s: 30 - delay_s: 300 + delay_s: 30 update_interval_s: 1 merge_quorum_size: 2 - stream_comparison_distance_b: 4096 \ No newline at end of file + stream_comparison_distance_b: 4096 diff --git a/cluster/storage/values.yaml b/cluster/storage/values.yaml index 374e8dd4..5abc8e17 100644 --- a/cluster/storage/values.yaml +++ b/cluster/storage/values.yaml @@ -24,6 +24,12 @@ managedStorages: size: 500Gi pvc: namespace: faf-apps + - pv: + name: faf-replays-old + folderName: replays-old + size: 500Gi + pvc: + namespace: faf-apps - pv: name: faf-maps folderName: maps diff --git a/cluster/telepresence/Chart.yaml b/cluster/telepresence/Chart.yaml new file mode 100644 index 00000000..34f52f1e --- /dev/null +++ b/cluster/telepresence/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: telepresence +version: 1.0.0 +dependencies: + - name: telepresence-oss + version: 2.25.1 + repository: oci://ghcr.io/telepresenceio \ No newline at end of file diff --git a/cluster/telepresence/values.yaml b/cluster/telepresence/values.yaml new file mode 100644 index 00000000..72fa2a8d --- /dev/null +++ b/cluster/telepresence/values.yaml @@ -0,0 +1,3 @@ +namespace: telepresence +telepresence: + releaseName: traffic-manager \ No newline at end of file diff --git a/ops/infisical/Chart.yaml b/ops/infisical/Chart.yaml index 67698d17..2d550c29 100644 --- a/ops/infisical/Chart.yaml +++ b/ops/infisical/Chart.yaml @@ -3,5 +3,5 @@ name: infisical version: 1.0.0 dependencies: - name: secrets-operator - version: 0.10.13 + version: 0.10.19 repository: https://dl.cloudsmith.io/public/infisical/helm-charts/helm/charts/ diff --git a/ops/monitoring/Chart.yaml b/ops/monitoring/Chart.yaml index 541e70ff..b48a5be9 100644 --- a/ops/monitoring/Chart.yaml +++ b/ops/monitoring/Chart.yaml @@ -5,9 +5,12 @@ dependencies: - name: victoria-metrics-k8s-stack version: 0.60.1 repository: https://victoriametrics.github.io/helm-charts/ -- name: promtail - version: 6.17.0 - repository: https://grafana.github.io/helm-charts - name: loki version: 6.40.0 repository: https://grafana.github.io/helm-charts +- name: k8s-monitoring + version: 3.5.1 + repository: https://grafana.github.io/helm-charts +- name: k8s-monitoring + version: 3.5.1 + repository: https://grafana.github.io/helm-charts diff --git a/ops/monitoring/values-prod.yaml b/ops/monitoring/values-prod.yaml new file mode 100644 index 00000000..063295f5 --- /dev/null +++ b/ops/monitoring/values-prod.yaml @@ -0,0 +1,3 @@ +k8s-monitoring: + cluster: + name: faforever-com \ No newline at end of file diff --git a/ops/monitoring/values-test.yaml b/ops/monitoring/values-test.yaml new file mode 100644 index 00000000..4bce1a76 --- /dev/null +++ b/ops/monitoring/values-test.yaml @@ -0,0 +1,3 @@ +k8s-monitoring: + cluster: + name: faforever-xyz \ No newline at end of file diff --git a/ops/monitoring/values.yaml b/ops/monitoring/values.yaml index 05ce39f8..1113c317 100644 --- a/ops/monitoring/values.yaml +++ b/ops/monitoring/values.yaml @@ -94,8 +94,6 @@ victoria-metrics-k8s-stack: loki: deploymentMode: SingleBinary - singleBinary: - replicas: 1 # Disable Simple Scalable read: @@ -131,9 +129,6 @@ loki: limits_config: retention_period: 168h - limits_config: - retention_period: 168h - # We know it's working test: enabled: false @@ -145,7 +140,30 @@ loki: chunksCache: enabled: false -promtail: - config: - clients: - - url: http://monitoring-loki-gateway/loki/api/v1/push + # Disable anti-affinity + singleBinary: + replicas: 1 + affinity: null + + gateway: + affinity: null + + +k8s-monitoring: + # Where Alloy pushes data to + # We can add our own Loki, Thanos... + destinations: + - name: local-loki + type: loki + url: http://monitoring-loki-gateway/loki/api/v1/push + + # We are using 1% of this chart, just logs for now, as it implements lots of things that vm-stack provides + # We could switch to this + vmsingle + + # Collectors + alloy-logs: + enabled: true + + # Features + podLogs: + enabled: true \ No newline at end of file