Reduce the memory usage of django_workload_mini

benchpress/config/jobs.yml

@@ -311,14 +311,20 @@
         - '--use-async {use_async}'
         - '--base-port {base_port}'
         - '-T {stats_port}'
+        - '-w {server_workers}'
+        - '--thrift-server-workers {thrift_server_workers}'
+        - '--cassandra-heap {cassandra_heap}'
       vars:
-        - 'duration=1M'
+        - 'duration=3S'
         - 'iterations=1'
-        - 'reps=1000'
+        - 'reps=0'
         - 'interpreter=cpython'
         - 'use_async=1'
         - 'base_port=16668'
         - 'stats_port=8001'
+        - 'server_workers=0'
+        - 'thrift_server_workers=0'
+        - 'cassandra_heap=1536M'
   hooks:
     - hook: copymove
       options:

packages/django_workload/README.md

@@ -205,6 +205,33 @@ Please create a backup of this folder to avoid data loss.
 
 These mini jobs will reuse the dataset generated in the previous step.
 
+
+#### How to tune the memory usage
+
+`server_workers` is the knob that dominates peak memory usage. Each uWSGI worker
+loads its own Python interpreter, Django and models, at roughly **0.57GB per
+worker**. It defaults to `0`, meaning one worker per usable core, so the
+footprint scales with the machine (or with the cpuset the job runs in).
+`thrift_server_workers` also defaults to `0` (per-core) but costs only
+~0.04GB each, so it is not the one to tune first.
+
+To adjust the memory usage, please see following commands:
+
+```
+./benchpress_cli.py run django_workload_mini -r standalone \
+  -i '{"server_workers": "2"}'
+```
+
+| `server_workers` | Peak memory usage (GB) |
+|------------------|-------------------------------|
+| 2                | ~3.8                        |
+| 8                | 6.86                        |
+| 16               | 12.08                       |
+
+`server_workers=2` is the recommended value to stay under a 4GB budget.
+
+The other large contributor is Cassandra's heap which is controled by `cassandra_heap` with default 1536M for the mini jobs right now.
+
 ### Parameters
 
 We provide the following parameters you can customize for DjangoBench workload:
@@ -261,6 +288,8 @@ Example:
 ./run.sh -r standalone --interpreter cinder --use-jit 1 --skip-datagen 0
 ```
 
+
+
 ## Reporting
 
 Once the benchmark finishes on the django benchmarking machine, benchpress will

packages/django_workload/srcs/bin/run.sh

@@ -268,7 +268,7 @@ Proxy shell script to executes django-workload benchmark
     -r          role (clientserver, client, server or db, default is clientserver)
     -h          display this help and exit
 For role "server", "clientserver":
-    -w          number of server workers (default NPROC)
+    -w          number of server workers, 0 = use the default (default NPROC)
     -c          ip address of the cassandra server (required)
     --interpreter
                 python interpreter to use (cpython or cinder, default is cpython)
@@ -300,6 +300,10 @@ For role "client":
     -z          ip address of the django server (required when role is 'client', default is ::1)
 For role "db":
     -y          number of cassandra concurrent writes (default 128)
+    --cassandra-heap
+                cassandra JVM heap size, e.g. 2G or 512M. Sets -Xms/-Xmx to
+                this value and -Xmn to a quarter of it, overriding conf/jvm.options
+                (default: unset, i.e. use conf/jvm.options unchanged)
     -b          ip address that cassandra will bind to (default to the first IP from "hostname -i": $(hostname -i))
     --thrift-server-workers
                 number of thrift server workers (default: nproc; pass 0 for auto)
@@ -592,6 +596,14 @@ ${python_libs}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
       # we need to restart Cassandra after loaing an snapshot
       echo "Cassandra is loaded using the snapshot"
     fi
+
+    # Re-generate the database if not found one
+    if [ "$skip_data_setup" = true ] && [ -z "$(find \
+         "${CASSANDRA_DATA_PATH}/${KEY_SPACE_NAME}" -name '*-Data.db' -print -quit 2>/dev/null)" ]; then
+      echo "skip-data-setup requested but keyspace ${KEY_SPACE_NAME} is missing; generating the database this run"
+      skip_data_setup=false
+    fi
+
     if [ "$skip_data_setup" = false ] && ! [ "${SKIP_DATAGEN}" = 1 ]; then
       echo "Generating database "
       DJANGO_SETTINGS_MODULE=cluster_settings ./"${venv_dir}"/bin/django-admin flush
@@ -868,6 +880,9 @@ main() {
   # in start_thrift_servers(). Override with --thrift-server-workers N.
   thrift_server_workers=0
 
+  local cassandra_heap
+  cassandra_heap=
+
   local use_jit
   use_jit=0
 
@@ -905,6 +920,14 @@ main() {
         base_port="${1#*=}"
         shift
         ;;
+      --cassandra-heap)
+        cassandra_heap="$2"
+        shift 2
+        ;;
+      --cassandra-heap=*)
+        cassandra_heap="${1#*=}"
+        shift
+        ;;
       --thrift-server-workers)
         thrift_server_workers="$2"
         shift 2
@@ -955,8 +978,11 @@ main() {
   while getopts 'w:x:y:i:p:d:l:s:r:c:z:b:L:t:ST:h' OPTION "${@}"; do
     case "$OPTION" in
       w)
-        # Use readlink to get absolute path if relative is given
-        num_server_workers="${OPTARG}"
+        # 0 keeps the nproc default, so a job can request the core-scaled
+        # value and still expose -w for overriding.
+        if [ "${OPTARG}" != "0" ]; then
+          num_server_workers="${OPTARG}"
+        fi
         ;;
       x)
         num_client_workers="${OPTARG}"
@@ -1075,6 +1101,21 @@ main() {
   export SKIP_DATAGEN="${skip_datagen}"
 
 
+  # overwrite the cassandra heap size
+  if [ -n "$cassandra_heap" ]; then
+    if ! [[ "$cassandra_heap" =~ ^[0-9]+[GgMm]?$ ]]; then
+      echo "Invalid cassandra_heap: '$cassandra_heap' (expected e.g. 1536M or 2G)" >&2
+      exit 1
+    fi
+    _heap_n="${cassandra_heap%[GgMm]}"
+    case "${cassandra_heap#"$_heap_n"}" in
+      G|g) _heap_mb=$(( _heap_n * 1024 )) ;;
+      *)   _heap_mb="$_heap_n" ;;
+    esac
+    export JVM_EXTRA_OPTS="-Xms$(( _heap_mb / 2 ))M -Xmx${_heap_mb}M -Xmn$(( _heap_mb / 4 ))M"
+    echo "Cassandra heap override: ${JVM_EXTRA_OPTS}"
+  fi
+
   if [ "$role" = "db" ]; then
     start_thrift_servers "$thrift_server_workers"
     start_cassandra "$num_cassandra_writes" "$cassandra_bind_addr";
@@ -1094,7 +1135,6 @@ main() {
     start_clientserver "$cassandra_addr" "$num_server_workers" "$num_client_workers" \
       "$duration" "$siege_logs_path" "$urls_path" "$iterations" "$reps" "$interpreter" \
       "${use_async}" "${use_jit}" "${worker_transport}";
-    pgrep -f cassandra | xargs kill
 
   else
     echo "Role $role is invalid, it can only be 'db' or 'clientserver' or 'standalone'";