From 1c1b16e7bd4ad78727101931a0db0178a7e00947 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Mon, 2 Mar 2026 19:59:06 -0500 Subject: [PATCH] Fixed bug in experiment scripts, updated Utilities/rsync scripts to rsync files in repo root --- Utilities/cloudlab_setup/single_node/rsync.sh | 31 ++++++++++++++----- .../rsync_and_selective_install_internal.sh | 31 ++++++++++++++----- .../experiment_utils/services/query_engine.py | 2 +- Utilities/root_files.conf | 6 ++++ Utilities/shared_utils.sh | 28 +++++++++++++++++ 5 files changed, 83 insertions(+), 15 deletions(-) create mode 100644 Utilities/root_files.conf diff --git a/Utilities/cloudlab_setup/single_node/rsync.sh b/Utilities/cloudlab_setup/single_node/rsync.sh index 9a24557..db7afa2 100755 --- a/Utilities/cloudlab_setup/single_node/rsync.sh +++ b/Utilities/cloudlab_setup/single_node/rsync.sh @@ -17,22 +17,39 @@ COMPONENTS_CONF_FILE="$THIS_DIR/../../components.conf" readarray -t COMPONENTS < <(load_components_config "$COMPONENTS_CONF_FILE") readarray -t DIRS_TO_RSYNC < <(build_rsync_paths "$THIS_DIR" "${COMPONENTS[@]}") +ROOT_FILES_CONF_FILE="$THIS_DIR/../../root_files.conf" +readarray -t ROOT_FILES < <(load_components_config "$ROOT_FILES_CONF_FILE") +readarray -t FILES_TO_RSYNC < <(build_rsync_file_paths "$THIS_DIR" "${ROOT_FILES[@]}") + echo "The following directories will be rsynced to $HOSTNAME:$DESTINATION_DIR:" for DIR in "${DIRS_TO_RSYNC[@]}"; do echo " $DIR" done +echo "The following root files will be rsynced to $HOSTNAME:$DESTINATION_DIR:" +for FILE in "${FILES_TO_RSYNC[@]}"; do + echo " $FILE" +done -RSYNC_OUTPUT=$(perform_rsync "$USERNAME" "$HOSTNAME" "$DESTINATION_DIR" "$OPTIONS" "${DIRS_TO_RSYNC[@]}") +RSYNC_OUTPUT=$(perform_rsync "$USERNAME" "$HOSTNAME" "$DESTINATION_DIR" "$OPTIONS" "${DIRS_TO_RSYNC[@]}" "${FILES_TO_RSYNC[@]}") echo "" echo $RSYNC_OUTPUT SYNCED_COMPONENTS=($(parse_rsync_output "$RSYNC_OUTPUT" "${DIRS_TO_RSYNC[@]}" "${COMPONENTS[@]}")) +SYNCED_FILES=($(parse_rsync_output_files "$RSYNC_OUTPUT" "${FILES_TO_RSYNC[@]}")) -if [ ${#SYNCED_COMPONENTS[@]} -eq 0 ]; then - echo "No components had changes to sync." +if [ ${#SYNCED_COMPONENTS[@]} -eq 0 ] && [ ${#SYNCED_FILES[@]} -eq 0 ]; then + echo "No components or files had changes to sync." else - echo "Components that were synced due to changes:" - for COMPONENT in "${SYNCED_COMPONENTS[@]}"; do - echo " $COMPONENT" - done + if [ ${#SYNCED_COMPONENTS[@]} -gt 0 ]; then + echo "Components that were synced due to changes:" + for COMPONENT in "${SYNCED_COMPONENTS[@]}"; do + echo " $COMPONENT" + done + fi + if [ ${#SYNCED_FILES[@]} -gt 0 ]; then + echo "Root files that were synced due to changes:" + for FILE in "${SYNCED_FILES[@]}"; do + echo " $FILE" + done + fi fi diff --git a/Utilities/cloudlab_setup/single_node/rsync_and_selective_install_internal.sh b/Utilities/cloudlab_setup/single_node/rsync_and_selective_install_internal.sh index 0e67cd1..76c0dc2 100755 --- a/Utilities/cloudlab_setup/single_node/rsync_and_selective_install_internal.sh +++ b/Utilities/cloudlab_setup/single_node/rsync_and_selective_install_internal.sh @@ -17,23 +17,40 @@ COMPONENTS_CONF_FILE="$THIS_DIR/../../components.conf" readarray -t COMPONENTS < <(load_components_config "$COMPONENTS_CONF_FILE") readarray -t DIRS_TO_RSYNC < <(build_rsync_paths "$THIS_DIR" "${COMPONENTS[@]}") +ROOT_FILES_CONF_FILE="$THIS_DIR/../../root_files.conf" +readarray -t ROOT_FILES < <(load_components_config "$ROOT_FILES_CONF_FILE") +readarray -t FILES_TO_RSYNC < <(build_rsync_file_paths "$THIS_DIR" "${ROOT_FILES[@]}") + echo "The following directories will be rsynced to $HOSTNAME:$DESTINATION_DIR:" for DIR in "${DIRS_TO_RSYNC[@]}"; do echo " $DIR" done +echo "The following root files will be rsynced to $HOSTNAME:$DESTINATION_DIR:" +for FILE in "${FILES_TO_RSYNC[@]}"; do + echo " $FILE" +done -RSYNC_OUTPUT=$(perform_rsync "$USERNAME" "$HOSTNAME" "$DESTINATION_DIR" "$OPTIONS" "${DIRS_TO_RSYNC[@]}") +RSYNC_OUTPUT=$(perform_rsync "$USERNAME" "$HOSTNAME" "$DESTINATION_DIR" "$OPTIONS" "${DIRS_TO_RSYNC[@]}" "${FILES_TO_RSYNC[@]}") echo "" SYNCED_COMPONENTS=($(parse_rsync_output "$RSYNC_OUTPUT" "${DIRS_TO_RSYNC[@]}" "${COMPONENTS[@]}")) +SYNCED_FILES=($(parse_rsync_output_files "$RSYNC_OUTPUT" "${FILES_TO_RSYNC[@]}")) -if [ ${#SYNCED_COMPONENTS[@]} -eq 0 ]; then - echo "No components had changes to sync." +if [ ${#SYNCED_COMPONENTS[@]} -eq 0 ] && [ ${#SYNCED_FILES[@]} -eq 0 ]; then + echo "No components or files had changes to sync." else - echo "Components that were synced due to changes:" - for COMPONENT in "${SYNCED_COMPONENTS[@]}"; do - echo " $COMPONENT" - done + if [ ${#SYNCED_COMPONENTS[@]} -gt 0 ]; then + echo "Components that were synced due to changes:" + for COMPONENT in "${SYNCED_COMPONENTS[@]}"; do + echo " $COMPONENT" + done + fi + if [ ${#SYNCED_FILES[@]} -gt 0 ]; then + echo "Root files that were synced due to changes:" + for FILE in "${SYNCED_FILES[@]}"; do + echo " $FILE" + done + fi echo "" echo "Installing synced internal components on $HOSTNAME..." diff --git a/Utilities/experiments/experiment_utils/services/query_engine.py b/Utilities/experiments/experiment_utils/services/query_engine.py index 63c99f9..65959ef 100644 --- a/Utilities/experiments/experiment_utils/services/query_engine.py +++ b/Utilities/experiments/experiment_utils/services/query_engine.py @@ -511,7 +511,7 @@ def _start_bare_metal( prometheus_server = f"http://{prometheus_host}:{prometheus_port}" cmd = ( - "mkdir -p {}; ./target/release/query_engine_rust " + "mkdir -p {}; ../target/release/query_engine_rust " "--kafka-topic {} " "--input-format {} " "--config {}/inference_config.yaml " diff --git a/Utilities/root_files.conf b/Utilities/root_files.conf new file mode 100644 index 0000000..060b167 --- /dev/null +++ b/Utilities/root_files.conf @@ -0,0 +1,6 @@ +# Root-level files to rsync alongside component directories. +# Paths are relative to the repository root. + +Cargo.toml +Cargo.lock +pyproject.toml diff --git a/Utilities/shared_utils.sh b/Utilities/shared_utils.sh index 9a8e539..889a237 100644 --- a/Utilities/shared_utils.sh +++ b/Utilities/shared_utils.sh @@ -56,6 +56,34 @@ parse_rsync_output() { printf '%s\n' "${synced_components[@]}" } +build_rsync_file_paths() { + local this_dir="$1" + local files=("${@:2}") + local files_to_rsync=() + + for file in "${files[@]}"; do + files_to_rsync+=("$this_dir/../../../$file") + done + + printf '%s\n' "${files_to_rsync[@]}" +} + +parse_rsync_output_files() { + local rsync_output="$1" + shift + local file_paths=("$@") + local synced_files=() + + for file_path in "${file_paths[@]}"; do + local filename=$(basename "$file_path") + if echo "$rsync_output" | grep -v ".git/" | grep -qE " ${filename}$"; then + synced_files+=("$filename") + fi + done + + printf '%s\n' "${synced_files[@]}" +} + perform_rsync() { local username="$1" local hostname="$2"