Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ server/olake-server
/tmp/
/dist/
olake-data/
amoro-logs/
fusion-logs/
/server/temporal-worker/

# ----------------------------
Expand Down
21 changes: 10 additions & 11 deletions docker-compose-v1.yml
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
x-app-defaults:
host_persistence_path: &hostPersistencePath ${PWD}/olake-data
amoro_log_persistence_path: &amoroLogPersistencePath ${PWD}/amoro-logs
fusion_log_persistence_path: &fusionLogPersistencePath ${PWD}/fusion-logs
worker_config_volume_details: &workerConfigVolumeDetails
type: bind
source: *hostPersistencePath
target: /tmp/olake-config
amoro_log_volume_details: &amoroLogVolumeDetails
fusion_log_volume_details: &fusionLogVolumeDetails
type: bind
source: *amoroLogPersistencePath
target: /mnt/amoro-logs
source: *fusionLogPersistencePath
target: /mnt/fusion-logs

x-signup-defaults:
username: &defaultUsername "admin"
Expand Down Expand Up @@ -236,21 +236,21 @@ services:
- "1261:1261" # Thrift optimizing service
environment:
<<: *sharedEnvs
AMORO_LOG_DIR: /mnt/amoro-logs
JVM_XMS: "1024"
JVM_XMX: "2048"
KUBECONFIG: /root/.kube/config
LOG_DIR: /mnt/amoro-logs/compaction
FUSION_CONF_DIR: /usr/local/fusion/conf
LOG_DIR: /mnt/fusion-logs
volumes:
- fusion-config:/fusion-config:ro
- kind-kubeconfig:/root/.kube:ro
- spark-home:/opt/spark:ro
- <<: *amoroLogVolumeDetails
- <<: *fusionLogVolumeDetails
entrypoint:
- /bin/sh
- -c
- |
cp /fusion-config/config.yaml /usr/local/amoro/conf/config.yaml
cp /fusion-config/config.yaml /usr/local/fusion/conf/config.yaml
exec /entrypoint.sh ams
tty: true
stdin_open: true
Expand Down Expand Up @@ -340,7 +340,7 @@ services:
networks:
- olake-network
environment:
AMORO_LOG_DIR: *amoroLogPersistencePath
LOG_DIR: *fusionLogPersistencePath
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- kind-kubeconfig:/output
Expand All @@ -351,11 +351,10 @@ services:
- |
set -e

LOG_DIR="$${AMORO_LOG_DIR}"
echo "==> Log directory: $$LOG_DIR"

echo "==> Generating kind cluster config..."
sed "s|AMORO_LOG_PLACEHOLDER|$$LOG_DIR|g" /fusion-config/kind-config.yaml > /tmp/kind-config-gen.yaml
sed "s|FUSION_LOG_PLACEHOLDER|$${LOG_DIR}|g" /fusion-config/kind-config.yaml > /tmp/kind-config-gen.yaml
echo "==> Installing kind..."
apk add --no-cache curl >/dev/null 2>&1
curl -sLo /usr/local/bin/kind https://kind.sigs.k8s.io/dl/v0.23.0/kind-linux-amd64
Expand Down
14 changes: 8 additions & 6 deletions optimization/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -142,12 +142,14 @@ containers:
spark-conf.spark.task.maxFailures: "4"
# Mount shared log volume so driver/executor pods write logs to the same path
# that LogController reads from (via the Kind node hostPath extraMount).
spark-conf.spark.kubernetes.driver.volumes.hostPath.amoro-logs.mount.path: "/mnt/amoro-logs"
spark-conf.spark.kubernetes.driver.volumes.hostPath.amoro-logs.mount.readOnly: "false"
spark-conf.spark.kubernetes.driver.volumes.hostPath.amoro-logs.options.path: "/mnt/amoro-logs"
spark-conf.spark.kubernetes.executor.volumes.hostPath.amoro-logs.mount.path: "/mnt/amoro-logs"
spark-conf.spark.kubernetes.executor.volumes.hostPath.amoro-logs.mount.readOnly: "false"
spark-conf.spark.kubernetes.executor.volumes.hostPath.amoro-logs.options.path: "/mnt/amoro-logs"
spark-conf.spark.kubernetes.driver.volumes.hostPath.fusion-logs.mount.path: "/mnt/fusion-logs"
spark-conf.spark.kubernetes.driver.volumes.hostPath.fusion-logs.mount.readOnly: "false"
spark-conf.spark.kubernetes.driver.volumes.hostPath.fusion-logs.options.path: "/mnt/fusion-logs"
spark-conf.spark.kubernetes.executor.volumes.hostPath.fusion-logs.mount.path: "/mnt/fusion-logs"
spark-conf.spark.kubernetes.executor.volumes.hostPath.fusion-logs.mount.readOnly: "false"
spark-conf.spark.kubernetes.executor.volumes.hostPath.fusion-logs.options.path: "/mnt/fusion-logs"
spark-conf.spark.kubernetes.driverEnv.LOG_DIR: "/mnt/fusion-logs"
spark-conf.spark.executorEnv.LOG_DIR: "/mnt/fusion-logs"
# Force Log4j2 to use the routing config (belt-and-suspenders with the Dockerfile COPY)
spark-conf.spark.driver.extraJavaOptions: "-Dlog4j2.configurationFile=file:///opt/spark/conf/log4j2.xml"
spark-conf.spark.executor.extraJavaOptions: "-Dlog4j2.configurationFile=file:///opt/spark/conf/log4j2.xml"
12 changes: 6 additions & 6 deletions optimization/kind-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ nodes:
- role: control-plane
image: kindest/node:v1.28.0
extraMounts:
- hostPath: AMORO_LOG_PLACEHOLDER
containerPath: /mnt/amoro-logs
- hostPath: FUSION_LOG_PLACEHOLDER
containerPath: /mnt/fusion-logs
kubeadmConfigPatches:
- |
kind: InitConfiguration
Expand All @@ -28,8 +28,8 @@ nodes:
- role: worker
image: kindest/node:v1.28.0
extraMounts:
- hostPath: AMORO_LOG_PLACEHOLDER
containerPath: /mnt/amoro-logs
- hostPath: FUSION_LOG_PLACEHOLDER
containerPath: /mnt/fusion-logs
kubeadmConfigPatches:
- |
kind: JoinConfiguration
Expand All @@ -39,8 +39,8 @@ nodes:
- role: worker
image: kindest/node:v1.28.0
extraMounts:
- hostPath: AMORO_LOG_PLACEHOLDER
containerPath: /mnt/amoro-logs
- hostPath: FUSION_LOG_PLACEHOLDER
containerPath: /mnt/fusion-logs
kubeadmConfigPatches:
- |
kind: JoinConfiguration
Expand Down
Loading