Fixed bug in experiment scripts, updated Utilities/rsync scripts to rsync files in repo root

Utilities/cloudlab_setup/single_node/rsync.sh

@@ -17,22 +17,39 @@ COMPONENTS_CONF_FILE="$THIS_DIR/../../components.conf"
 readarray -t COMPONENTS < <(load_components_config "$COMPONENTS_CONF_FILE")
 readarray -t DIRS_TO_RSYNC < <(build_rsync_paths "$THIS_DIR" "${COMPONENTS[@]}")
 
+ROOT_FILES_CONF_FILE="$THIS_DIR/../../root_files.conf"
+readarray -t ROOT_FILES < <(load_components_config "$ROOT_FILES_CONF_FILE")
+readarray -t FILES_TO_RSYNC < <(build_rsync_file_paths "$THIS_DIR" "${ROOT_FILES[@]}")
+
 echo "The following directories will be rsynced to $HOSTNAME:$DESTINATION_DIR:"
 for DIR in "${DIRS_TO_RSYNC[@]}"; do
     echo "  $DIR"
 done
+echo "The following root files will be rsynced to $HOSTNAME:$DESTINATION_DIR:"
+for FILE in "${FILES_TO_RSYNC[@]}"; do
+    echo "  $FILE"
+done
 
-RSYNC_OUTPUT=$(perform_rsync "$USERNAME" "$HOSTNAME" "$DESTINATION_DIR" "$OPTIONS" "${DIRS_TO_RSYNC[@]}")
+RSYNC_OUTPUT=$(perform_rsync "$USERNAME" "$HOSTNAME" "$DESTINATION_DIR" "$OPTIONS" "${DIRS_TO_RSYNC[@]}" "${FILES_TO_RSYNC[@]}")
 
 echo ""
 echo $RSYNC_OUTPUT
 SYNCED_COMPONENTS=($(parse_rsync_output "$RSYNC_OUTPUT" "${DIRS_TO_RSYNC[@]}" "${COMPONENTS[@]}"))
+SYNCED_FILES=($(parse_rsync_output_files "$RSYNC_OUTPUT" "${FILES_TO_RSYNC[@]}"))
 
-if [ ${#SYNCED_COMPONENTS[@]} -eq 0 ]; then
-    echo "No components had changes to sync."
+if [ ${#SYNCED_COMPONENTS[@]} -eq 0 ] && [ ${#SYNCED_FILES[@]} -eq 0 ]; then
+    echo "No components or files had changes to sync."
 else
-    echo "Components that were synced due to changes:"
-    for COMPONENT in "${SYNCED_COMPONENTS[@]}"; do
-        echo "  $COMPONENT"
-    done
+    if [ ${#SYNCED_COMPONENTS[@]} -gt 0 ]; then
+        echo "Components that were synced due to changes:"
+        for COMPONENT in "${SYNCED_COMPONENTS[@]}"; do
+            echo "  $COMPONENT"
+        done
+    fi
+    if [ ${#SYNCED_FILES[@]} -gt 0 ]; then
+        echo "Root files that were synced due to changes:"
+        for FILE in "${SYNCED_FILES[@]}"; do
+            echo "  $FILE"
+        done
+    fi
 fi

Utilities/cloudlab_setup/single_node/rsync_and_selective_install_internal.sh

@@ -17,23 +17,40 @@ COMPONENTS_CONF_FILE="$THIS_DIR/../../components.conf"
 readarray -t COMPONENTS < <(load_components_config "$COMPONENTS_CONF_FILE")
 readarray -t DIRS_TO_RSYNC < <(build_rsync_paths "$THIS_DIR" "${COMPONENTS[@]}")
 
+ROOT_FILES_CONF_FILE="$THIS_DIR/../../root_files.conf"
+readarray -t ROOT_FILES < <(load_components_config "$ROOT_FILES_CONF_FILE")
+readarray -t FILES_TO_RSYNC < <(build_rsync_file_paths "$THIS_DIR" "${ROOT_FILES[@]}")
+
 echo "The following directories will be rsynced to $HOSTNAME:$DESTINATION_DIR:"
 for DIR in "${DIRS_TO_RSYNC[@]}"; do
     echo "  $DIR"
 done
+echo "The following root files will be rsynced to $HOSTNAME:$DESTINATION_DIR:"
+for FILE in "${FILES_TO_RSYNC[@]}"; do
+    echo "  $FILE"
+done
 
-RSYNC_OUTPUT=$(perform_rsync "$USERNAME" "$HOSTNAME" "$DESTINATION_DIR" "$OPTIONS" "${DIRS_TO_RSYNC[@]}")
+RSYNC_OUTPUT=$(perform_rsync "$USERNAME" "$HOSTNAME" "$DESTINATION_DIR" "$OPTIONS" "${DIRS_TO_RSYNC[@]}" "${FILES_TO_RSYNC[@]}")
 
 echo ""
 SYNCED_COMPONENTS=($(parse_rsync_output "$RSYNC_OUTPUT" "${DIRS_TO_RSYNC[@]}" "${COMPONENTS[@]}"))
+SYNCED_FILES=($(parse_rsync_output_files "$RSYNC_OUTPUT" "${FILES_TO_RSYNC[@]}"))
 
-if [ ${#SYNCED_COMPONENTS[@]} -eq 0 ]; then
-    echo "No components had changes to sync."
+if [ ${#SYNCED_COMPONENTS[@]} -eq 0 ] && [ ${#SYNCED_FILES[@]} -eq 0 ]; then
+    echo "No components or files had changes to sync."
 else
-    echo "Components that were synced due to changes:"
-    for COMPONENT in "${SYNCED_COMPONENTS[@]}"; do
-        echo "  $COMPONENT"
-    done
+    if [ ${#SYNCED_COMPONENTS[@]} -gt 0 ]; then
+        echo "Components that were synced due to changes:"
+        for COMPONENT in "${SYNCED_COMPONENTS[@]}"; do
+            echo "  $COMPONENT"
+        done
+    fi
+    if [ ${#SYNCED_FILES[@]} -gt 0 ]; then
+        echo "Root files that were synced due to changes:"
+        for FILE in "${SYNCED_FILES[@]}"; do
+            echo "  $FILE"
+        done
+    fi
 
     echo ""
     echo "Installing synced internal components on $HOSTNAME..."

Utilities/experiments/experiment_utils/services/query_engine.py

@@ -511,7 +511,7 @@ def _start_bare_metal(
         prometheus_server = f"http://{prometheus_host}:{prometheus_port}"
 
         cmd = (
-            "mkdir -p {}; ./target/release/query_engine_rust "
+            "mkdir -p {}; ../target/release/query_engine_rust "
             "--kafka-topic {} "
             "--input-format {} "
             "--config {}/inference_config.yaml "

Utilities/root_files.conf

@@ -0,0 +1,6 @@
+# Root-level files to rsync alongside component directories.
+# Paths are relative to the repository root.
+
+Cargo.toml
+Cargo.lock
+pyproject.toml

Utilities/shared_utils.sh

@@ -56,6 +56,34 @@ parse_rsync_output() {
     printf '%s\n' "${synced_components[@]}"
 }
 
+build_rsync_file_paths() {
+    local this_dir="$1"
+    local files=("${@:2}")
+    local files_to_rsync=()
+
+    for file in "${files[@]}"; do
+        files_to_rsync+=("$this_dir/../../../$file")
+    done
+
+    printf '%s\n' "${files_to_rsync[@]}"
+}
+
+parse_rsync_output_files() {
+    local rsync_output="$1"
+    shift
+    local file_paths=("$@")
+    local synced_files=()
+
+    for file_path in "${file_paths[@]}"; do
+        local filename=$(basename "$file_path")
+        if echo "$rsync_output" | grep -v ".git/" | grep -qE " ${filename}$"; then
+            synced_files+=("$filename")
+        fi
+    done
+
+    printf '%s\n' "${synced_files[@]}"
+}
+
 perform_rsync() {
     local username="$1"
     local hostname="$2"