diff --git a/benchpress/config/jobs.yml b/benchpress/config/jobs.yml index c11c66333..c97ffeb07 100644 --- a/benchpress/config/jobs.yml +++ b/benchpress/config/jobs.yml @@ -311,14 +311,20 @@ - '--use-async {use_async}' - '--base-port {base_port}' - '-T {stats_port}' + - '-w {server_workers}' + - '--thrift-server-workers {thrift_server_workers}' + - '--cassandra-heap {cassandra_heap}' vars: - - 'duration=1M' + - 'duration=3S' - 'iterations=1' - - 'reps=1000' + - 'reps=0' - 'interpreter=cpython' - 'use_async=1' - 'base_port=16668' - 'stats_port=8001' + - 'server_workers=0' + - 'thrift_server_workers=0' + - 'cassandra_heap=1536M' hooks: - hook: copymove options: diff --git a/packages/django_workload/README.md b/packages/django_workload/README.md index c1d573ea3..760944904 100644 --- a/packages/django_workload/README.md +++ b/packages/django_workload/README.md @@ -205,6 +205,33 @@ Please create a backup of this folder to avoid data loss. These mini jobs will reuse the dataset generated in the previous step. + +#### How to tune the memory usage + +`server_workers` is the knob that dominates peak memory usage. Each uWSGI worker +loads its own Python interpreter, Django and models, at roughly **0.57GB per +worker**. It defaults to `0`, meaning one worker per usable core, so the +footprint scales with the machine (or with the cpuset the job runs in). +`thrift_server_workers` also defaults to `0` (per-core) but costs only +~0.04GB each, so it is not the one to tune first. + +To adjust the memory usage, please see following commands: + +``` +./benchpress_cli.py run django_workload_mini -r standalone \ + -i '{"server_workers": "2"}' +``` + +| `server_workers` | Peak memory usage (GB) | +|------------------|-------------------------------| +| 2 | ~3.8 | +| 8 | 6.86 | +| 16 | 12.08 | + +`server_workers=2` is the recommended value to stay under a 4GB budget. + +The other large contributor is Cassandra's heap which is controled by `cassandra_heap` with default 1536M for the mini jobs right now. + ### Parameters We provide the following parameters you can customize for DjangoBench workload: @@ -261,6 +288,8 @@ Example: ./run.sh -r standalone --interpreter cinder --use-jit 1 --skip-datagen 0 ``` + + ## Reporting Once the benchmark finishes on the django benchmarking machine, benchpress will diff --git a/packages/django_workload/srcs/bin/run.sh b/packages/django_workload/srcs/bin/run.sh index 8c1d57da6..a29e5da32 100755 --- a/packages/django_workload/srcs/bin/run.sh +++ b/packages/django_workload/srcs/bin/run.sh @@ -268,7 +268,7 @@ Proxy shell script to executes django-workload benchmark -r role (clientserver, client, server or db, default is clientserver) -h display this help and exit For role "server", "clientserver": - -w number of server workers (default NPROC) + -w number of server workers, 0 = use the default (default NPROC) -c ip address of the cassandra server (required) --interpreter python interpreter to use (cpython or cinder, default is cpython) @@ -300,6 +300,10 @@ For role "client": -z ip address of the django server (required when role is 'client', default is ::1) For role "db": -y number of cassandra concurrent writes (default 128) + --cassandra-heap + cassandra JVM heap size, e.g. 2G or 512M. Sets -Xms/-Xmx to + this value and -Xmn to a quarter of it, overriding conf/jvm.options + (default: unset, i.e. use conf/jvm.options unchanged) -b ip address that cassandra will bind to (default to the first IP from "hostname -i": $(hostname -i)) --thrift-server-workers number of thrift server workers (default: nproc; pass 0 for auto) @@ -592,6 +596,14 @@ ${python_libs}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} # we need to restart Cassandra after loaing an snapshot echo "Cassandra is loaded using the snapshot" fi + + # Re-generate the database if not found one + if [ "$skip_data_setup" = true ] && [ -z "$(find \ + "${CASSANDRA_DATA_PATH}/${KEY_SPACE_NAME}" -name '*-Data.db' -print -quit 2>/dev/null)" ]; then + echo "skip-data-setup requested but keyspace ${KEY_SPACE_NAME} is missing; generating the database this run" + skip_data_setup=false + fi + if [ "$skip_data_setup" = false ] && ! [ "${SKIP_DATAGEN}" = 1 ]; then echo "Generating database " DJANGO_SETTINGS_MODULE=cluster_settings ./"${venv_dir}"/bin/django-admin flush @@ -868,6 +880,9 @@ main() { # in start_thrift_servers(). Override with --thrift-server-workers N. thrift_server_workers=0 + local cassandra_heap + cassandra_heap= + local use_jit use_jit=0 @@ -905,6 +920,14 @@ main() { base_port="${1#*=}" shift ;; + --cassandra-heap) + cassandra_heap="$2" + shift 2 + ;; + --cassandra-heap=*) + cassandra_heap="${1#*=}" + shift + ;; --thrift-server-workers) thrift_server_workers="$2" shift 2 @@ -955,8 +978,11 @@ main() { while getopts 'w:x:y:i:p:d:l:s:r:c:z:b:L:t:ST:h' OPTION "${@}"; do case "$OPTION" in w) - # Use readlink to get absolute path if relative is given - num_server_workers="${OPTARG}" + # 0 keeps the nproc default, so a job can request the core-scaled + # value and still expose -w for overriding. + if [ "${OPTARG}" != "0" ]; then + num_server_workers="${OPTARG}" + fi ;; x) num_client_workers="${OPTARG}" @@ -1075,6 +1101,21 @@ main() { export SKIP_DATAGEN="${skip_datagen}" + # overwrite the cassandra heap size + if [ -n "$cassandra_heap" ]; then + if ! [[ "$cassandra_heap" =~ ^[0-9]+[GgMm]?$ ]]; then + echo "Invalid cassandra_heap: '$cassandra_heap' (expected e.g. 1536M or 2G)" >&2 + exit 1 + fi + _heap_n="${cassandra_heap%[GgMm]}" + case "${cassandra_heap#"$_heap_n"}" in + G|g) _heap_mb=$(( _heap_n * 1024 )) ;; + *) _heap_mb="$_heap_n" ;; + esac + export JVM_EXTRA_OPTS="-Xms$(( _heap_mb / 2 ))M -Xmx${_heap_mb}M -Xmn$(( _heap_mb / 4 ))M" + echo "Cassandra heap override: ${JVM_EXTRA_OPTS}" + fi + if [ "$role" = "db" ]; then start_thrift_servers "$thrift_server_workers" start_cassandra "$num_cassandra_writes" "$cassandra_bind_addr"; @@ -1094,7 +1135,6 @@ main() { start_clientserver "$cassandra_addr" "$num_server_workers" "$num_client_workers" \ "$duration" "$siege_logs_path" "$urls_path" "$iterations" "$reps" "$interpreter" \ "${use_async}" "${use_jit}" "${worker_transport}"; - pgrep -f cassandra | xargs kill else echo "Role $role is invalid, it can only be 'db' or 'clientserver' or 'standalone'";