Skip to content

Fix Feedsim DLRM benchmark missing files for Ubuntu #822

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LDong-Arm wants to merge 1 commit into
base: v2-beta
Choose a base branch
from
Open

Fix Feedsim DLRM benchmark missing files for Ubuntu #822

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions packages/feedsim/install_feedsim_aarch64_ubuntu.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@ cp "${BENCHPRESS_ROOT}/packages/feedsim/run-feedsim-multi.sh" "${FEEDSIM_ROOT_SR
chmod u+x "${FEEDSIM_ROOT_SRC}/run.sh"
chmod u+x "${FEEDSIM_ROOT_SRC}/run-feedsim-multi.sh"

# Copy production size distribution JSONs (consumed by run.sh / DriverNodeRank).
cp "${BENCHPRESS_ROOT}/packages/feedsim/feed_aggregator_req_sizes.json" "${FEEDSIM_ROOT_SRC}/feed_aggregator_req_sizes.json"
cp "${BENCHPRESS_ROOT}/packages/feedsim/feed_aggregator_resp_sizes.json" "${FEEDSIM_ROOT_SRC}/feed_aggregator_resp_sizes.json"
# Phase 6 session-mode driver loads rpc_dist.json from the FEEDSIM_ROOT runtime dir.
cp "${BENCHPRESS_ROOT}/packages/feedsim/rpc_dist.json" "${FEEDSIM_ROOT_SRC}/rpc_dist.json"

msg "Installing third-party dependencies..."
# Sync feedsim source with --delete so `./benchpress install -f` actually picks up
# source changes. The previous "skip if dir exists" guard meant -f never refreshed
Expand Down Expand Up @@ -259,6 +265,46 @@ else
msg "[SKIPPED] DLRM model already installed"
fi

# Download Silesia compression corpus for story-based requests (Phase 3)
SILESIA_DIR="${FEEDSIM_ROOT_SRC}/silesia"
SILESIA_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-silesia/silesia.tar.gz"
if ! [ -d "$SILESIA_DIR" ] || [ -z "$(ls -A "$SILESIA_DIR" 2>/dev/null)" ]; then
msg "Downloading Silesia corpus..."
mkdir -p "$SILESIA_DIR"
cd "$SILESIA_DIR" || { msg "[ERROR] cannot cd to $SILESIA_DIR"; exit 1; }
if wget -q "$SILESIA_URL" -O silesia.tar.gz 2>/dev/null; then
tar -xzf silesia.tar.gz && rm -f silesia.tar.gz
msg "Silesia corpus downloaded: $(ls | wc -l) files, $(du -sh . | cut -f1)"
else
msg "[INFO] GitHub dataset not available, trying original Silesia host..."
SILESIA_FALLBACK_URL="https://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip"
if wget -q "$SILESIA_FALLBACK_URL" -O silesia.zip 2>/dev/null; then
unzip -q silesia.zip && rm -f silesia.zip
msg "Silesia corpus downloaded: $(ls | wc -l) files, $(du -sh . | cut -f1)"
else
msg "[WARNING] Silesia download failed — story-based requests will be unavailable"
fi
fi
cd "${FEEDSIM_THIRD_PARTY_SRC}"
else
msg "[SKIPPED] Silesia corpus already present at $SILESIA_DIR"
fi

# Extract example TLS certs for mock_services (used when --tls_cert/--tls_key
# are passed; see run-feedsim-multi.sh). The tarball ships example.crt and
# example.key suitable for benchmark use only (no peer verification).
CERTS_DIR="${FEEDSIM_ROOT_SRC}/certs"
CERTS_TARBALL="${BENCHPRESS_ROOT}/packages/common/certs.tar.gz"
if [ -f "$CERTS_TARBALL" ]; then
mkdir -p "$CERTS_DIR"
# --strip-components=1 drops the top-level `certs/` directory inside the
# tarball so the files land directly at $CERTS_DIR/example.{crt,key}.
tar -xzf "$CERTS_TARBALL" -C "$CERTS_DIR" --strip-components=1
msg "Extracted TLS certs to $CERTS_DIR"
else
msg "[WARNING] $CERTS_TARBALL not found; TLS for mock_services will be unavailable"
fi


# Installing FeedSim
cd "${FEEDSIM_ROOT_SRC}"
Expand Down
47 changes: 47 additions & 0 deletions packages/feedsim/install_feedsim_ubuntu.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,12 @@ cp "${BENCHPRESS_ROOT}/packages/feedsim/run-feedsim-multi.sh" "${FEEDSIM_ROOT_SR
chmod u+x "${FEEDSIM_ROOT_SRC}/run.sh"
chmod u+x "${FEEDSIM_ROOT_SRC}/run-feedsim-multi.sh"

# Copy production size distribution JSONs (consumed by run.sh / DriverNodeRank).
cp "${BENCHPRESS_ROOT}/packages/feedsim/feed_aggregator_req_sizes.json" "${FEEDSIM_ROOT_SRC}/feed_aggregator_req_sizes.json"
cp "${BENCHPRESS_ROOT}/packages/feedsim/feed_aggregator_resp_sizes.json" "${FEEDSIM_ROOT_SRC}/feed_aggregator_resp_sizes.json"
# Phase 6 session-mode driver loads rpc_dist.json from the FEEDSIM_ROOT runtime dir.
cp "${BENCHPRESS_ROOT}/packages/feedsim/rpc_dist.json" "${FEEDSIM_ROOT_SRC}/rpc_dist.json"

msg "Installing third-party dependencies..."
# Sync feedsim source with --delete so `./benchpress install -f` actually picks up
# source changes. The previous "skip if dir exists" guard meant -f never refreshed
Expand Down Expand Up @@ -265,6 +271,47 @@ else
msg "[SKIPPED] DLRM model already installed"
fi

# Download Silesia compression corpus for story-based requests (Phase 3)
SILESIA_DIR="${FEEDSIM_ROOT_SRC}/silesia"
SILESIA_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-silesia/silesia.tar.gz"
if ! [ -d "$SILESIA_DIR" ] || [ -z "$(ls -A "$SILESIA_DIR" 2>/dev/null)" ]; then
msg "Downloading Silesia corpus..."
mkdir -p "$SILESIA_DIR"
cd "$SILESIA_DIR" || { msg "[ERROR] cannot cd to $SILESIA_DIR"; exit 1; }
if wget -q "$SILESIA_URL" -O silesia.tar.gz 2>/dev/null; then
tar -xzf silesia.tar.gz && rm -f silesia.tar.gz
msg "Silesia corpus downloaded: $(ls | wc -l) files, $(du -sh . | cut -f1)"
else
msg "[INFO] GitHub dataset not available, trying original Silesia host..."
SILESIA_FALLBACK_URL="https://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip"
if wget -q "$SILESIA_FALLBACK_URL" -O silesia.zip 2>/dev/null; then
unzip -q silesia.zip && rm -f silesia.zip
msg "Silesia corpus downloaded: $(ls | wc -l) files, $(du -sh . | cut -f1)"
else
msg "[WARNING] Silesia download failed — story-based requests will be unavailable"
fi
fi
cd "${FEEDSIM_THIRD_PARTY_SRC}"
else
msg "[SKIPPED] Silesia corpus already present at $SILESIA_DIR"
fi

# Extract example TLS certs for mock_services (used when --tls_cert/--tls_key
# are passed; see run-feedsim-multi.sh). The tarball ships example.crt and
# example.key suitable for benchmark use only (no peer verification).
CERTS_DIR="${FEEDSIM_ROOT_SRC}/certs"
CERTS_TARBALL="${BENCHPRESS_ROOT}/packages/common/certs.tar.gz"
if [ -f "$CERTS_TARBALL" ]; then
mkdir -p "$CERTS_DIR"
# --strip-components=1 drops the top-level `certs/` directory inside the
# tarball so the files land directly at $CERTS_DIR/example.{crt,key}.
tar -xzf "$CERTS_TARBALL" -C "$CERTS_DIR" --strip-components=1
msg "Extracted TLS certs to $CERTS_DIR"
else
msg "[WARNING] $CERTS_TARBALL not found; TLS for mock_services will be unavailable"
fi



# Installing FeedSim
cd "${FEEDSIM_ROOT_SRC}"
Expand Down
Loading