From 7b68700e83a7c008ac438f67a4e6ac0580b35600 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Sat, 24 Nov 2018 00:46:11 -0500 Subject: [PATCH 01/44] Fix style errors Change-Id: Id6fd7197d0ed2b66038a89628f1cb4e64d826962 Signed-off-by: Brian J. Murrell --- src/cnss/cnss.c | 4 +++- src/ioc/ioc.h | 17 +++++++++-------- src/ioc/ioc_main.c | 41 ++++++++++++++++++++++++++--------------- src/ioc/ops/create.c | 6 ++++-- src/ioc/ops/forget.c | 3 ++- src/ioc/ops/fsync.c | 3 ++- src/ioc/ops/lookup.c | 3 ++- src/ioc/ops/open.c | 6 ++++-- src/ioc/ops/write.c | 3 ++- src/ionss/config.c | 18 +++++++++++------- src/ionss/ionss.c | 3 ++- 11 files changed, 67 insertions(+), 40 deletions(-) diff --git a/src/cnss/cnss.c b/src/cnss/cnss.c index 34b21a17..769d679e 100644 --- a/src/cnss/cnss.c +++ b/src/cnss/cnss.c @@ -980,7 +980,9 @@ int main(int argc, char **argv) "Plugin %s(%p) calling destroy_plugin_data at %p", entry->fns->name, entry->fns->handle, - FN_TO_PVOID(entry->fns->destroy_plugin_data)); + FN_TO_PVOID( + entry->fns->destroy_plugin_data + )); entry->fns->destroy_plugin_data(entry->fns->handle); } diff --git a/src/ioc/ioc.h b/src/ioc/ioc.h index 2cab4489..4f8575e2 100644 --- a/src/ioc/ioc.h +++ b/src/ioc/ioc.h @@ -453,15 +453,16 @@ struct fuse_lowlevel_ops *iof_get_fuse_ops(uint64_t); IOF_TRACE_DOWN(ioc_req); \ } while (0) -#define IOC_REPLY_IOCTL(handle, req, gah_info) \ - do { \ - int __rc; \ - IOF_TRACE_DEBUG(handle, "Returning ioctl"); \ - __rc = fuse_reply_ioctl(req, 0, &(gah_info), sizeof(gah_info)); \ - if (__rc != 0) \ - IOF_TRACE_ERROR(handle, \ +#define IOC_REPLY_IOCTL(handle, req, gah_info) \ + do { \ + int __rc; \ + IOF_TRACE_DEBUG(handle, "Returning ioctl"); \ + __rc = fuse_reply_ioctl(req, 0, &(gah_info), \ + sizeof(gah_info)); \ + if (__rc != 0) \ + IOF_TRACE_ERROR(handle, \ "fuse_reply_ioctl returned %d:%s", \ - __rc, strerror(-__rc)); \ + __rc, strerror(-__rc)); \ } while (0) struct ioc_request; diff --git a/src/ioc/ioc_main.c b/src/ioc/ioc_main.c index 2668b4b4..8021a133 100644 --- a/src/ioc/ioc_main.c +++ b/src/ioc/ioc_main.c @@ -318,10 +318,12 @@ static void gah_decref(struct iof_projection_info *fs_handle) if (ref > 1) { int rc; - rc = fuse_lowlevel_notify_inval_entry(fs_handle->session, - ie->parent, - ie->name, - strlen(ie->name)); + rc = fuse_lowlevel_notify_inval_entry( + fs_handle->session, + ie->parent, + ie->name, + strlen(ie->name) + ); IOF_TRACE_INFO(ie, "inval returned %d", rc); if (rc == -ENOENT) { @@ -1328,14 +1330,16 @@ fh_reset(void *arg) } rc = crt_req_create(fh->open_req.fsh->proj.crt_ctx, NULL, - FS_TO_OP(fh->open_req.fsh, open), &fh->open_req.rpc); + FS_TO_OP(fh->open_req.fsh, open), + &fh->open_req.rpc); if (rc || !fh->open_req.rpc) { D_FREE(fh->ie); return false; } rc = crt_req_create(fh->open_req.fsh->proj.crt_ctx, NULL, - FS_TO_OP(fh->open_req.fsh, create), &fh->creat_req.rpc); + FS_TO_OP(fh->open_req.fsh, create), + &fh->creat_req.rpc); if (rc || !fh->creat_req.rpc) { D_FREE(fh->ie); crt_req_decref(fh->open_req.rpc); @@ -1343,7 +1347,8 @@ fh_reset(void *arg) } rc = crt_req_create(fh->open_req.fsh->proj.crt_ctx, NULL, - FS_TO_OP(fh->open_req.fsh, close), &fh->release_req.rpc); + FS_TO_OP(fh->open_req.fsh, close), + &fh->release_req.rpc); if (rc || !fh->release_req.rpc) { D_FREE(fh->ie); crt_req_decref(fh->open_req.rpc); @@ -2173,16 +2178,20 @@ initialize_projection(struct iof_state *iof_state, IOF_TRACE_DEBUG(fs_handle, "Setting timeout to %d", fs_info->timeout); - ret = crt_context_set_timeout(fs_handle->proj.crt_ctx, fs_info->timeout); + ret = crt_context_set_timeout(fs_handle->proj.crt_ctx, + fs_info->timeout); if (ret != -DER_SUCCESS) { IOF_TRACE_ERROR(iof_state, "Context timeout not set"); D_GOTO(err, 0); } for (i = 0; i < fs_handle->ctx_num; i++) { - fs_handle->ctx_array[i].crt_ctx = fs_handle->proj.crt_ctx; - fs_handle->ctx_array[i].poll_interval = iof_state->iof_ctx.poll_interval; - fs_handle->ctx_array[i].callback_fn = iof_state->iof_ctx.callback_fn; + fs_handle->ctx_array[i].crt_ctx = + fs_handle->proj.crt_ctx; + fs_handle->ctx_array[i].poll_interval = + iof_state->iof_ctx.poll_interval; + fs_handle->ctx_array[i].callback_fn = + iof_state->iof_ctx.callback_fn; /* TODO: Much better error checking is required here, not least * terminating the thread if there are any failures in the rest @@ -2713,9 +2722,10 @@ static void iof_finish(void *arg) /*send a detach RPC to IONSS*/ rc = crt_req_create(iof_state->iof_ctx.crt_ctx, &group->grp.psr_ep, - CRT_PROTO_OPC(iof_state->handshake_proto->cpf_base, - iof_state->handshake_proto->cpf_ver, - 1), + CRT_PROTO_OPC( + iof_state->handshake_proto->cpf_base, + iof_state->handshake_proto->cpf_ver, + 1), &rpc); if (rc != -DER_SUCCESS || !rpc) { IOF_TRACE_ERROR(iof_state, @@ -2767,7 +2777,8 @@ static void iof_finish(void *arg) rc = crt_context_destroy(iof_state->iof_ctx.crt_ctx, false); if (rc != -DER_SUCCESS) - IOF_TRACE_ERROR(iof_state, "Could not destroy context %d", + IOF_TRACE_ERROR(iof_state, + "Could not destroy context %d", rc); IOF_TRACE_DOWN(&iof_state->iof_ctx); } diff --git a/src/ioc/ops/create.c b/src/ioc/ops/create.c index bd4c942c..1555c13d 100644 --- a/src/ioc/ops/create.c +++ b/src/ioc/ops/create.c @@ -44,7 +44,8 @@ static bool ioc_create_ll_cb(struct ioc_request *request) { - struct iof_file_handle *handle = container_of(request, struct iof_file_handle, creat_req); + struct iof_file_handle *handle = container_of( + request, struct iof_file_handle, creat_req); struct iof_projection_info *fs_handle = request->fsh; struct iof_create_out *out = crt_reply_get(request->rpc); struct fuse_file_info fi = {0}; @@ -183,7 +184,8 @@ void ioc_ll_create(fuse_req_t req, fuse_ino_t parent, const char *name, IOF_TRACE_UP(handle, fs_handle, fs_handle->fh_pool->reg.name); IOF_TRACE_UP(&handle->creat_req, handle, "creat_req"); - IOF_TRACE_LINK(handle->creat_req.rpc, &handle->creat_req, "creat_file_rpc"); + IOF_TRACE_LINK(handle->creat_req.rpc, &handle->creat_req, + "creat_file_rpc"); handle->common.projection = &fs_handle->proj; handle->creat_req.req = req; diff --git a/src/ioc/ops/forget.c b/src/ioc/ops/forget.c index 06048881..efacac8f 100644 --- a/src/ioc/ops/forget.c +++ b/src/ioc/ops/forget.c @@ -68,7 +68,8 @@ ioc_forget_one(struct iof_projection_info *fs_handle, if (rc != -DER_SUCCESS) { IOF_TRACE_ERROR(fs_handle, "Invalid refcount %lu on %p", nlookup, - container_of(rlink, struct ioc_inode_entry, ie_htl)); + container_of(rlink, struct ioc_inode_entry, + ie_htl)); } } diff --git a/src/ioc/ops/fsync.c b/src/ioc/ops/fsync.c index 10aac101..c1c40ebe 100644 --- a/src/ioc/ops/fsync.c +++ b/src/ioc/ops/fsync.c @@ -50,7 +50,8 @@ void ioc_ll_fsync(fuse_req_t req, fuse_ino_t ino, int datasync, struct fuse_file_info *fi) { - struct iof_file_handle *handle = (struct iof_file_handle *)fi->fh; + struct iof_file_handle *handle = + (struct iof_file_handle *)fi->fh; struct iof_projection_info *fs_handle = handle->open_req.fsh; struct ioc_request *request; crt_opcode_t opcode; diff --git a/src/ioc/ops/lookup.c b/src/ioc/ops/lookup.c index f14b4d58..086dbf63 100644 --- a/src/ioc/ops/lookup.c +++ b/src/ioc/ops/lookup.c @@ -89,7 +89,8 @@ iof_entry_cb(struct ioc_request *request) * the parent anyway, so keep that one, but drop one in the call * to ie_close(). */ - IOF_TRACE_INFO(container_of(rlink, struct ioc_inode_entry, ie_htl), + IOF_TRACE_INFO(container_of(rlink, struct ioc_inode_entry, + ie_htl), "Existing file %lu " GAH_PRINT_STR, entry.ino, GAH_PRINT_VAL(out->gah)); atomic_fetch_sub(&desc->ie->ie_ref, 1); diff --git a/src/ioc/ops/open.c b/src/ioc/ops/open.c index 6557baa7..dd3a0a1a 100644 --- a/src/ioc/ops/open.c +++ b/src/ioc/ops/open.c @@ -44,7 +44,8 @@ static bool ioc_open_ll_cb(struct ioc_request *request) { - struct iof_file_handle *handle = container_of(request, struct iof_file_handle, open_req); + struct iof_file_handle *handle = container_of( + request, struct iof_file_handle, open_req); struct iof_open_out *out = crt_reply_get(request->rpc); struct fuse_file_info fi = {0}; @@ -125,7 +126,8 @@ void ioc_ll_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) } IOF_TRACE_UP(handle, fs_handle, fs_handle->fh_pool->reg.name); IOF_TRACE_UP(&handle->open_req, handle, "open_req"); - IOF_TRACE_LINK(handle->open_req.rpc, &handle->open_req, "open_file_rpc"); + IOF_TRACE_LINK(handle->open_req.rpc, &handle->open_req, + "open_file_rpc"); handle->common.projection = &fs_handle->proj; handle->open_req.req = req; diff --git a/src/ioc/ops/write.c b/src/ioc/ops/write.c index 9d5dd7ca..fe8ec117 100644 --- a/src/ioc/ops/write.c +++ b/src/ioc/ops/write.c @@ -44,7 +44,8 @@ static bool write_cb(struct ioc_request *request) { - struct iof_wb *wb = container_of(request, struct iof_wb, wb_req); + struct iof_wb *wb = container_of(request, + struct iof_wb, wb_req); struct iof_writex_out *out = crt_reply_get(request->rpc); struct iof_writex_in *in = crt_req_get(request->rpc); diff --git a/src/ionss/config.c b/src/ionss/config.c index d3d90e17..bcf9f9b9 100644 --- a/src/ionss/config.c +++ b/src/ionss/config.c @@ -110,7 +110,7 @@ const uint32_t default_max_iov_write_size = 64; const uint32_t default_max_read_count = 3; const uint32_t default_max_write_count = 3; const uint32_t default_inode_htable_size = 5; -const uint32_t default_cnss_thread_count = 0; +const uint32_t default_cnss_thread_count; const uint32_t default_cnss_timeout = 60; const bool default_cnss_threads = true; const bool default_fuse_read_buf = true; @@ -305,13 +305,17 @@ static int parse_node(yaml_document_t *document, yaml_node_t *node, ret = sel_option->setter(sel_option, document, val_node); if (ret) { if (val_node->type == YAML_SCALAR_NODE) { - IOF_LOG_WARNING("Unknown configuration value %s %.*s", - sel_option->key, - (int)val_node->data.scalar.length, - (char *)val_node->data.scalar.value); + IOF_LOG_WARNING( + "Unknown configuration value %s %.*s", + sel_option->key, + (int)val_node->data.scalar.length, + (char *)val_node->data.scalar.value + ); } else { - IOF_LOG_WARNING("Unknown configuration value %s", - sel_option->key); + IOF_LOG_WARNING( + "Unknown configuration value %s", + sel_option->key + ); } return ret; } diff --git a/src/ionss/ionss.c b/src/ionss/ionss.c index b929e8aa..2e2eb116 100644 --- a/src/ionss/ionss.c +++ b/src/ionss/ionss.c @@ -2934,7 +2934,8 @@ int main(int argc, char **argv) base.fs_list[i].max_iov_write = projection->max_iov_write_size; base.fs_list[i].htable_size = projection->inode_htable_size; base.fs_list[i].timeout = projection->cnss_timeout; - base.fs_list[i].cnss_thread_count = projection->cnss_thread_count; + base.fs_list[i].cnss_thread_count = + projection->cnss_thread_count; base.fs_list[i].flags = IOF_FS_DEFAULT; if (projection->failover) From c9aa95d546d076ad912751ded2664a3b0050bd3b Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 15 Nov 2018 15:44:21 -0500 Subject: [PATCH 02/44] Initial pipeline support Change-Id: Ia2b2b05176f306382928ae338eebe150e5edb4be Signed-off-by: Brian J. Murrell --- Jenkinsfile | 253 +++++++++++++++++++++++++++ test/Dockerfile.centos:7 | 40 +++++ utils/docker/Dockerfile.centos:7 | 66 +++++++ utils/docker/Dockerfile.ubuntu:18.04 | 73 ++++++++ 4 files changed, 432 insertions(+) create mode 100644 Jenkinsfile create mode 100644 test/Dockerfile.centos:7 create mode 100644 utils/docker/Dockerfile.centos:7 create mode 100644 utils/docker/Dockerfile.ubuntu:18.04 diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..5ce82a8d --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,253 @@ +// To use a test branch (i.e. PR) until it lands to master +// I.e. for testing library changes +@Library(value="pipeline-lib@debug-iof") _ + +pipeline { + agent any + + environment { + GITHUB_USER = credentials('aa4ae90b-b992-4fb6-b33b-236a53a26f77') + BAHTTPS_PROXY = "${env.HTTP_PROXY ? '--build-arg HTTP_PROXY="' + env.HTTP_PROXY + '" --build-arg http_proxy="' + env.HTTP_PROXY + '"' : ''}" + BAHTTP_PROXY = "${env.HTTP_PROXY ? '--build-arg HTTPS_PROXY="' + env.HTTPS_PROXY + '" --build-arg https_proxy="' + env.HTTPS_PROXY + '"' : ''}" + UID=sh(script: "id -u", returnStdout: true) + BUILDARGS = "--build-arg NOBUILD=1 --build-arg UID=$env.UID $env.BAHTTP_PROXY $env.BAHTTPS_PROXY" + } + + options { + // preserve stashes so that jobs can be started at the test stage + preserveStashes(buildCount: 5) + } + + stages { + stage('Pre-build') { + parallel { + stage('checkpatch') { + agent { + dockerfile { + filename 'Dockerfile.centos:7' + dir 'utils/docker' + label 'docker_runner' + additionalBuildArgs '$BUILDARGS' + } + } + steps { + checkPatch user: GITHUB_USER_USR, + password: GITHUB_USER_PSW, + ignored_files: "src/control/vendor/*" + } + post { + /* temporarily moved into stepResult due to JENKINS-39203 + success { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'checkpatch', context: 'pre-build/checkpatch', status: 'SUCCESS' + } + unstable { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'checkpatch', context: 'pre-build/checkpatch', status: 'FAILURE' + } + failure { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'checkpatch', context: 'pre-build/checkpatch', status: 'ERROR' + } + */ + always { + archiveArtifacts artifacts: 'pylint.log', allowEmptyArchive: true + } + } + } + } + } + stage('Build') { + // abort other builds if/when one fails to avoid wasting time + // and resources + failFast true + parallel { + stage('Build on CentOS 7') { + agent { + dockerfile { + filename 'Dockerfile.centos:7' + dir 'utils/docker' + label 'docker_runner' + additionalBuildArgs '$BUILDARGS' + } + } + steps { + sconsBuild clean: "_build.external-Linux" + stash name: 'CentOS-install', includes: 'install/**' + stash name: 'CentOS-build-vars', includes: '.build_vars-Linux.*' + //stash name: 'CentOS-tests', includes: 'build/src/rdb/raft/src/tests_main, build/src/common/tests/btree_direct, build/src/common/tests/btree, src/common/tests/btree.sh, build/src/common/tests/sched, build/src/client/api/tests/eq_tests, src/vos/tests/evt_ctl.sh, build/src/vos/vea/tests/vea_ut, src/rdb/raft_tests/raft_tests.py' + } + post { + always { + recordIssues enabledForFailure: true, + aggregatingResults: true, + id: "analysis-centos7", + tools: [ + [tool: [$class: 'GnuMakeGcc']], + [tool: [$class: 'CppCheck']], + ], + filters: [excludeFile('.*\\/_build\\.external\\/.*'), + excludeFile('_build\\.external\\/.*')] + } + /* temporarily moved into stepResult due to JENKINS-39203 + success { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'CentOS 7 Build', context: 'build/centos7', status: 'SUCCESS' + } + unstable { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'CentOS 7 Build', context: 'build/centos7', status: 'FAILURE' + } + failure { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'CentOS 7 Build', context: 'build/centos7', status: 'ERROR' + } + */ + } + } + stage('Build on Ubuntu 18.04') { + agent { + dockerfile { + filename 'Dockerfile.ubuntu:18.04' + dir 'utils/docker' + label 'docker_runner' + additionalBuildArgs '$BUILDARGS' + } + } + steps { + sh '''echo "Skipping Ubuntu 18 build due to https://jira.hpdd.intel.com/browse/CART-548" + exit 0''' + //sconsBuild clean: "_build.external-Linux" + } + post { + always { + recordIssues enabledForFailure: true, + aggregatingResults: true, + id: "analysis-ubuntu18", + tools: [ + [tool: [$class: 'GnuMakeGcc']], + [tool: [$class: 'CppCheck']], + ], + filters: [excludeFile('.*\\/_build\\.external\\/.*'), + excludeFile('_build\\.external\\/.*')] + } + /* temporarily moved into stepResult due to JENKINS-39203 + success { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'Ubuntu 18 Build', context: 'build/ubuntu18', status: 'SUCCESS' + } + unstable { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'Ubuntu 18 Build', context: 'build/ubuntu18', status: 'FAILURE' + } + failure { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'Ubuntu 18 Build', context: 'build/ubuntu18', status: 'ERROR' + } + */ + } + } + } + } +//NOTESTYET stage('Unit Test') { +//NOTESTYET parallel { +//NOTESTYET stage('run_test.sh') { +//NOTESTYET agent { +//NOTESTYET label 'single' +//NOTESTYET } +//NOTESTYET steps { +//NOTESTYET runTest stashes: [ 'CentOS-tests', 'CentOS-install', 'CentOS-build-vars' ], +//NOTESTYET script: 'LD_LIBRARY_PATH=install/Linux/lib64:install/Linux/lib HOSTPREFIX=wolf-53 bash -x utils/run_test.sh --init && echo "run_test.sh exited successfully with ${PIPESTATUS[0]}" || echo "run_test.sh exited failure with ${PIPESTATUS[0]}"', +//NOTESTYET junit_files: null +//NOTESTYET } +//NOTESTYET post { +//NOTESTYET /* temporarily moved into runTest->stepResult due to JENKINS-39203 +//NOTESTYET success { +//NOTESTYET githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'run_test.sh', context: 'test/run_test.sh', status: 'SUCCESS' +//NOTESTYET } +//NOTESTYET unstable { +//NOTESTYET githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'run_test.sh', context: 'test/run_test.sh', status: 'FAILURE' +//NOTESTYET } +//NOTESTYET failure { +//NOTESTYET githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'run_test.sh', context: 'test/run_test.sh', status: 'ERROR' +//NOTESTYET } +//NOTESTYET */ +//NOTESTYET always { +//NOTESTYET sh '''rm -rf run_test.sh/ +//NOTESTYET mkdir run_test.sh/ +//NOTESTYET [ -f /tmp/daos.log ] && mv /tmp/daos.log run_test.sh/ || true''' +//NOTESTYET archiveArtifacts artifacts: 'run_test.sh/**' +//NOTESTYET } +//NOTESTYET } +//NOTESTYET } +//NOTESTYET } +//NOTESTYET } + stage('Test') { + parallel { + stage('Single node') { + agent { + label 'single' + } + steps { + runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], + script: '''ln -s $(which fusermount) install/Linux/bin/fusermount3 + . ./.build_vars-Linux.sh + DAOS_BASE=${SL_OMPI_PREFIX%/install/*} + if [ "$PWD" != "$DAOS_BASE" ]; then + export LD_LIBRARY_PATH=$PWD/install/lib64:$PWD/install/lib: + fi + if ! OMPI_MCA_rmaps_base_oversubscribe=1 PATH=$PATH:$PWD/install/Linux/bin nosetests-3.4 --exe --with-xunit; then + echo "rc: ${PIPESTATUS[0]}" + fi''', + junit_files: "nosetests.xml" + } + post { + /* temporarily moved into runTest->stepResult due to JENKINS-39203 + success { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: env.STAGE_NAME, context: 'test/functional_quick', status: 'SUCCESS' + } + unstable { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: env.STAGE_NAME, context: 'test/functional_quick', status: 'FAILURE' + } + failure { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: env.STAGE_NAME, context: 'test/functional_quick', status: 'ERROR' + } + */ + always { + junit 'nosetests.xml' + archiveArtifacts artifacts: 'test/output/Testlocal/*/*.log' + } + } + } +//NOTESTYET// we don't need to run this separately when running "all" above +//NOTESTYET// stage('Functional daos_test') { +//NOTESTYET// agent { +//NOTESTYET// label 'cluster_provisioner' +//NOTESTYET// } +//NOTESTYET// steps { +//NOTESTYET// runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], +//NOTESTYET// script: 'bash ftest.sh daos_test; echo "rc: $?"', +//NOTESTYET// // Hrm. I wonder if there is any way to tell Avocado +//NOTESTYET// // to put daos_test's own JUnit files into a job-results +//NOTESTYET// // dir +//NOTESTYET// junit_files: "install/Linux/tmp/*results.xml" +//NOTESTYET// } +//NOTESTYET// post { +//NOTESTYET// /* temporarily moved into runTest->stepResult due to JENKINS-39203 +//NOTESTYET// success { +//NOTESTYET// githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'Functional daos_test', context: 'test/functional_daos_test', status: 'SUCCESS' +//NOTESTYET// } +//NOTESTYET// unstable { +//NOTESTYET// githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'Functional daos_test', context: 'test/functional_daos_test', status: 'FAILURE' +//NOTESTYET// } +//NOTESTYET// failure { +//NOTESTYET// githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'Functional daos_test', context: 'test/functional_daos_test', status: 'ERROR' +//NOTESTYET// } +//NOTESTYET// */ +//NOTESTYET// always { +//NOTESTYET// sh '''rm -rf src/tests/ftest/avocado/job-results/*/html/ "Functional daos_test"/ +//NOTESTYET// mkdir "Functional daos_test"/ +//NOTESTYET// mv src/tests/ftest/avocado/job-results/* "Functional daos_test"/ +//NOTESTYET// pwd +//NOTESTYET// ls *daos.log* && mv -f *daos.log* "Functional daos_test"/ || true +//NOTESTYET// find "Functional daos_test"/ -print || true''' +//NOTESTYET// junit 'install/Linux/tmp/*results.xml, Functional daos_test/*/results.xml' +//NOTESTYET// archiveArtifacts artifacts: 'Functional daos_test/**' +//NOTESTYET// } +//NOTESTYET// } +//NOTESTYET// } + } + } + } +} diff --git a/test/Dockerfile.centos:7 b/test/Dockerfile.centos:7 new file mode 100644 index 00000000..6c8d68e0 --- /dev/null +++ b/test/Dockerfile.centos:7 @@ -0,0 +1,40 @@ +# +# Copyright 2018, Intel Corporation +# +# 'recipe' for Docker to build an image of centOS-based +# environment for testing the IOF project. +# + +# Pull base image +FROM centos:7 +MAINTAINER Brian J. Murrell + +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 + +# Install basic tools +RUN yum install -y epel-release +RUN yum install -y fuse python34-nose python34-pip python34-PyYAML \ + numactl-libs libevent openssh-clients librdmacm \ + infinipath-psm libpsm2 valgrind CUnit rsync + +# Add Jenkins user +ENV USER jenkins +ENV PASSWD jenkins +RUN useradd -u $UID -ms /bin/bash $USER +RUN echo "$USER:$PASSWD" | chpasswd + +# Switch to new user +USER $USER +WORKDIR /home/$USER + +# no python3-tabulate RPM +# TODO: build one +RUN pip3 install --user tabulate + +# Set environment variables +#ENV PATH=/home/daos/daos/install/bin:$PATH +#ENV LD_LIBRARY_PATH=/home/daos/daos/install/lib:/home/daos/daos/install/lib/daos_srv:$LD_LIBRARY_PATH +#ENV CPATH=/home/daos/daos/install/include:$CPATH +#ENV CRT_PHY_ADDR_STR="ofi+sockets" +#ENV OFI_INTERFACE=eth0 diff --git a/utils/docker/Dockerfile.centos:7 b/utils/docker/Dockerfile.centos:7 new file mode 100644 index 00000000..69caa450 --- /dev/null +++ b/utils/docker/Dockerfile.centos:7 @@ -0,0 +1,66 @@ +# +# Copyright 2018, Intel Corporation +# +# 'recipe' for Docker to build an image of centOS-based +# environment for building the DAOS project. +# + +# Pull base image +FROM centos:7 +MAINTAINER Johann Lombardi + +# Build arguments can be set via -build-arg +# set NOBUILD to disable git clone & build +ARG NOBUILD +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 + +# Update distribution +#Nothing to do for CentOS + +# Install basic tools +RUN yum install -y epel-release +RUN yum install -y git gcc gcc-c++ make cmake golang libtool scons boost-devel \ + libuuid-devel openssl-devel libevent-devel libtool-ltdl-devel \ + libcmocka libcmocka-devel readline-devel \ + doxygen pandoc flex patch nasm yasm ninja-build meson \ + CUnit-devel libaio-devel python-pep8 lcov \ + python clang-analyzer sg3_utils libiscsi-devel \ + numactl-devel doxygen graphviz pylint file libyaml-devel \ + ShellCheck python2-pygithub yum-plugin-copr +RUN yum copr -y enable jhli/ipmctl +RUN yum copr -y enable jhli/safeclib +RUN yum install -y libsafec libipmctl-devel + +# Dependencies +# Packages for NVML, PMIx, hwloc and OpenMPI exist in CentOS, but are unfortunately +# outdated. The DAOS build system will rebuild those packages. + +# Add DAOS user +ENV USER daos +ENV PASSWD daos +RUN useradd -u $UID -ms /bin/bash $USER +RUN echo "$USER:$PASSWD" | chpasswd + +# Create directory for DAOS backend storage +RUN mkdir /mnt/daos +RUN chown daos.daos /mnt/daos + +# Switch to new user +USER $USER +WORKDIR /home/$USER + +# Fetch DAOS code +RUN if [ "x$NOBUILD" = "x" ] ; then git clone https://github.com/daos-stack/daos.git; fi +WORKDIR /home/$USER/daos + +# Build DAOS & dependencies +RUN if [ "x$NOBUILD" = "x" ] ; then git submodule init && git submodule update; fi +RUN if [ "x$NOBUILD" = "x" ] ; then scons --build-deps=yes USE_INSTALLED=all install; fi + +# Set environment variables +ENV PATH=/home/daos/daos/install/bin:$PATH +ENV LD_LIBRARY_PATH=/home/daos/daos/install/lib:/home/daos/daos/install/lib/daos_srv:$LD_LIBRARY_PATH +ENV CPATH=/home/daos/daos/install/include:$CPATH +ENV CRT_PHY_ADDR_STR="ofi+sockets" +ENV OFI_INTERFACE=eth0 diff --git a/utils/docker/Dockerfile.ubuntu:18.04 b/utils/docker/Dockerfile.ubuntu:18.04 new file mode 100644 index 00000000..d4396de7 --- /dev/null +++ b/utils/docker/Dockerfile.ubuntu:18.04 @@ -0,0 +1,73 @@ +# +# Copyright 2018, Intel Corporation +# +# 'recipe' for Docker to build an image of Ubuntu-based +# environment for building the DAOS project. +# + +# Pull base image +FROM ubuntu:18.04 +MAINTAINER Johann Lombardi + +# Build arguments can be set via -build-arg +# set NOBUILD to disable git clone & build +ARG NOBUILD +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 + +# Update distribution +# It's better to put the apt-get update in the same "cache layer" as the +# apt-get install command so that the apt database is updated if/when the +# installed packages list below is updated + +# Install basic tools +RUN apt-get update; \ + DEBIAN_FRONTEND=noninteractive \ + apt-get install -y software-properties-common; \ + add-apt-repository ppa:jhli/libsafec; \ + add-apt-repository ppa:jhli/ipmctl; \ + apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y git \ + gcc golang-go make cmake libtool-bin scons autoconf \ + libboost-dev uuid-dev libssl-dev libevent-dev \ + libltdl-dev librdmacm-dev libcmocka0 libcmocka-dev \ + libreadline6-dev curl doxygen pandoc flex patch nasm \ + yasm graphviz doxygen libibverbs-dev librdmacm-dev \ + libcunit1-dev libnuma-dev libaio-dev sg3-utils \ + libiscsi-dev python-dev mscgen ninja-build meson \ + pylint bash locales libyaml-dev libsafec-dev \ + libipmctl-dev +# hack the default shell to bash instead of dash +RUN rm /bin/sh && ln -s bash /bin/sh + +RUN locale-gen en_US.UTF-8 + +# Dependencies + +# Add DAOS user +ENV USER daos +ENV PASSWD daos +RUN useradd -u $UID -ms /bin/bash $USER +RUN echo "$USER:$PASSWD" | chpasswd + +# Create directory for DAOS backend storage +RUN mkdir /mnt/daos +RUN chown daos.daos /mnt/daos + +# Switch to new user +USER $USER +WORKDIR /home/$USER + +# Fetch DAOS code +RUN if [ "x$NOBUILD" = "x" ] ; then git clone https://github.com/daos-stack/daos.git; fi +WORKDIR /home/$USER/daos + +# Build DAOS & dependencies +RUN if [ "x$NOBUILD" = "x" ] ; then git submodule init && git submodule update; fi +RUN if [ "x$NOBUILD" = "x" ] ; then scons --build-deps=yes USE_INSTALLED=all install; fi + +# Set environment variables +ENV PATH=/home/daos/daos/install/bin:$PATH +ENV LD_LIBRARY_PATH=/home/daos/daos/install/lib:/home/daos/daos/install/lib/daos_srv:$LD_LIBRARY_PATH +ENV CPATH=/home/daos/daos/install/include:$CPATH +ENV CRT_PHY_ADDR_STR="ofi+sockets" +ENV OFI_INTERFACE=eth0 From 02608a2f632f1d8f114fb11f38b95428e261c3a1 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Tue, 20 Nov 2018 13:12:32 -0500 Subject: [PATCH 03/44] Some test debug Change-Id: I03f3a21fc660aed70cf67343c199fa357c3287fb Signed-off-by: Brian J. Murrell --- Jenkinsfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5ce82a8d..fa59c734 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -181,7 +181,13 @@ pipeline { } steps { runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], - script: '''ln -s $(which fusermount) install/Linux/bin/fusermount3 + script: '''id || true + pwd || true + ls $HOME || true + pip3 list || true + find /home/jenkins/ -type f | xargs ls -ld || true + ln -s $(which fusermount) install/Linux/bin/fusermount3 + mount . ./.build_vars-Linux.sh DAOS_BASE=${SL_OMPI_PREFIX%/install/*} if [ "$PWD" != "$DAOS_BASE" ]; then From ffce2dc3ca1425eed4c767115a6617cb0d1878a1 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Tue, 27 Nov 2018 21:25:54 -0500 Subject: [PATCH 04/44] Multi-node tests Change-Id: Ifdac5f2c5a10dda06e42071d6d6063800df7ea14 Signed-off-by: Brian J. Murrell --- Jenkinsfile | 65 ++++++++++++-------------- multi-node-test.sh | 111 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 37 deletions(-) create mode 100755 multi-node-test.sh diff --git a/Jenkinsfile b/Jenkinsfile index fa59c734..3ffe5d7d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -216,43 +216,34 @@ pipeline { } } } -//NOTESTYET// we don't need to run this separately when running "all" above -//NOTESTYET// stage('Functional daos_test') { -//NOTESTYET// agent { -//NOTESTYET// label 'cluster_provisioner' -//NOTESTYET// } -//NOTESTYET// steps { -//NOTESTYET// runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], -//NOTESTYET// script: 'bash ftest.sh daos_test; echo "rc: $?"', -//NOTESTYET// // Hrm. I wonder if there is any way to tell Avocado -//NOTESTYET// // to put daos_test's own JUnit files into a job-results -//NOTESTYET// // dir -//NOTESTYET// junit_files: "install/Linux/tmp/*results.xml" -//NOTESTYET// } -//NOTESTYET// post { -//NOTESTYET// /* temporarily moved into runTest->stepResult due to JENKINS-39203 -//NOTESTYET// success { -//NOTESTYET// githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'Functional daos_test', context: 'test/functional_daos_test', status: 'SUCCESS' -//NOTESTYET// } -//NOTESTYET// unstable { -//NOTESTYET// githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'Functional daos_test', context: 'test/functional_daos_test', status: 'FAILURE' -//NOTESTYET// } -//NOTESTYET// failure { -//NOTESTYET// githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'Functional daos_test', context: 'test/functional_daos_test', status: 'ERROR' -//NOTESTYET// } -//NOTESTYET// */ -//NOTESTYET// always { -//NOTESTYET// sh '''rm -rf src/tests/ftest/avocado/job-results/*/html/ "Functional daos_test"/ -//NOTESTYET// mkdir "Functional daos_test"/ -//NOTESTYET// mv src/tests/ftest/avocado/job-results/* "Functional daos_test"/ -//NOTESTYET// pwd -//NOTESTYET// ls *daos.log* && mv -f *daos.log* "Functional daos_test"/ || true -//NOTESTYET// find "Functional daos_test"/ -print || true''' -//NOTESTYET// junit 'install/Linux/tmp/*results.xml, Functional daos_test/*/results.xml' -//NOTESTYET// archiveArtifacts artifacts: 'Functional daos_test/**' -//NOTESTYET// } -//NOTESTYET// } -//NOTESTYET// } + stage('Multi-node') { + agent { + label 'cluster_provisioner' + } + steps { + runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], + script: 'bash -x ./multi-node-test.sh; echo "rc: $?"', + junit_files: null + } + post { + /* temporarily moved into runTest->stepResult due to JENKINS-39203 + success { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'Functional daos_test', context: 'test/functional_daos_test', status: 'SUCCESS' + } + unstable { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'Functional daos_test', context: 'test/functional_daos_test', status: 'FAILURE' + } + failure { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'Functional daos_test', context: 'test/functional_daos_test', status: 'ERROR' + } + */ + always { + sh 'echo "Nothing to do"' + //junit 'install/Linux/tmp/*results.xml, Functional daos_test/*/results.xml' + archiveArtifacts artifacts: 'install/Linux/TESTING/testLogs/**' + } + } + } } } } diff --git a/multi-node-test.sh b/multi-node-test.sh new file mode 100755 index 00000000..99168783 --- /dev/null +++ b/multi-node-test.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +set -ex -o pipefail + +# shellcheck disable=SC1091 +if [ -f .localenv ]; then + # read (i.e. environment, etc.) overrides + . .localenv +fi + +HOSTPREFIX=${HOSTPREFIX-${HOSTNAME%%.*}} +NFS_SERVER=${NFS_SERVER:-$HOSTPREFIX} + +trap 'echo "encountered an unchecked return code, exiting with error"' ERR + +# shellcheck disable=SC1091 +. .build_vars-Linux.sh + +# shellcheck disable=SC2154 +trap 'set +e +i=5 +# due to flakiness on wolf-53, try this several times +while [ $i -gt 0 ]; do + pdsh -R ssh -S -w ${HOSTPREFIX}vm[1-9] "set -x + x=0 + while [ \$x -lt 30 ] && + grep $DAOS_BASE /proc/mounts && + ! sudo umount $DAOS_BASE; do + ps axf + sleep 1 + let x+=1 + done + sudo rmdir $DAOS_BASE || find $DAOS_BASE || true" 2>&1 | dshbak -c + if [ \${PIPESTATUS[0] = 0 ]; then + i=0 + fi + let i-=1 +done' EXIT + +DAOS_BASE=${SL_OMPI_PREFIX%/install/*} +if ! pdsh -R ssh -S -w "${HOSTPREFIX}"vm[1-9] "set -ex +ulimit -c unlimited +sudo mkdir -p $DAOS_BASE +sudo ed <&1 | dshbak -c; then + echo "Cluster setup (i.e. provisioning) failed" + exit 1 +fi + +echo "hit enter to continue" +#read -r +#exit 0 + +cat < install/Linux/TESTING/scripts/iof_fio_main.cfg +{ + "host_list": ["vm2", "vm3"], + "test_mode": "littleChief" +} +EOF +cp install/Linux/TESTING/scripts/iof_{fio,ior}_main.cfg +cp install/Linux/TESTING/scripts/iof_{fio,iozone}_main.cfg +cp install/Linux/TESTING/scripts/iof_{fio,mdtest}_main.cfg + +rm -rf install/Linux/TESTING/testLogs/ +rm -f install/Linux/bin/fusermount3 +ln -s "$(command -v fusermount)" install/Linux/bin/fusermount3 + +# shellcheck disable=SC2029 +if ! ssh "${HOSTPREFIX}"vm1 "set -ex +ulimit -c unlimited +cd $DAOS_BASE + +# now run it! +pushd install/Linux/TESTING +python3 test_runner config=scripts/iof_fio_main.cfg scripts/iof_multi_two_node.yml +mv testLogs/testRun{,-fio} +python3 test_runner config=scripts/iof_ior_main.cfg scripts/iof_multi_two_node.yml +mv testLogs/testRun{,-ior} +python3 test_runner config=scripts/iof_iozone_main.cfg scripts/iof_multi_two_node.yml +mv testLogs/testRun{,-iozone} +python3 test_runner config=scripts/iof_mdtest_main.cfg scripts/iof_m_two_node.yml +mv testLogs/testRun{,-mdtest} + + +ls -l +exit \$rc"; then + rc=${PIPESTATUS[0]} +else + rc=0 +fi + +if false; then +# collect the logs +if ! rpdcp -R ssh -w "${HOSTPREFIX}"vm[1-9] \ + /tmp/Functional_"$TEST_TAG"/\*daos.log "$PWD"/; then + echo "Copying daos.logs from remote nodes failed" + # pass +fi +ls -l +fi +exit "$rc" From bf25039bce74a7e98440cac4c604dabdd4e60b6c Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Wed, 28 Nov 2018 10:20:50 -0500 Subject: [PATCH 05/44] Need the HOSTPREFIX on the test nodes Change-Id: I930fc2fc1b5cccbed07fae14e7a3e50b99b18571 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 99168783..6d4b5691 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -63,7 +63,7 @@ echo "hit enter to continue" cat < install/Linux/TESTING/scripts/iof_fio_main.cfg { - "host_list": ["vm2", "vm3"], + "host_list": ["${HOSTPREFIX}vm2", "${HOSTPREFIX}vm3"], "test_mode": "littleChief" } EOF From 38ed8f02ce60a934df85ea3d34d8e43efc98aac6 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Wed, 28 Nov 2018 10:50:02 -0500 Subject: [PATCH 06/44] Don't let test exit status bail the run Change-Id: Ie65bb01e9e2e5071cfd8255d93658a83440ebf36 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 6d4b5691..26e8af78 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -82,16 +82,27 @@ cd $DAOS_BASE # now run it! pushd install/Linux/TESTING -python3 test_runner config=scripts/iof_fio_main.cfg scripts/iof_multi_two_node.yml +python3 test_runner config=scripts/iof_fio_main.cfg scripts/iof_multi_two_node.yml || { + rc=\${PIPESTATUS[0]} + echo \"Test exited with \$rc\" +} mv testLogs/testRun{,-fio} -python3 test_runner config=scripts/iof_ior_main.cfg scripts/iof_multi_two_node.yml +python3 test_runner config=scripts/iof_ior_main.cfg scripts/iof_multi_two_node.yml || { + rc=\${PIPESTATUS[0]} + echo \"Test exited with \$rc\" +} mv testLogs/testRun{,-ior} -python3 test_runner config=scripts/iof_iozone_main.cfg scripts/iof_multi_two_node.yml +python3 test_runner config=scripts/iof_iozone_main.cfg scripts/iof_multi_two_node.yml || { + rc=\${PIPESTATUS[0]} + echo \"Test exited with \$rc\" +} mv testLogs/testRun{,-iozone} -python3 test_runner config=scripts/iof_mdtest_main.cfg scripts/iof_m_two_node.yml +python3 test_runner config=scripts/iof_mdtest_main.cfg scripts/iof_m_two_node.yml || { + rc=\${PIPESTATUS[0]} + echo \"Test exited with \$rc\" +} mv testLogs/testRun{,-mdtest} - ls -l exit \$rc"; then rc=${PIPESTATUS[0]} From 1558b3085fc5d749e6341dc8eccd1c6d96f79f31 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Wed, 28 Nov 2018 10:54:08 -0500 Subject: [PATCH 07/44] Fix missing } Change-Id: I1409befadf9cabfca79b238c4629813c4be18084 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 26e8af78..c2857fae 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -31,7 +31,7 @@ while [ $i -gt 0 ]; do let x+=1 done sudo rmdir $DAOS_BASE || find $DAOS_BASE || true" 2>&1 | dshbak -c - if [ \${PIPESTATUS[0] = 0 ]; then + if [ \${PIPESTATUS[0]} = 0 ]; then i=0 fi let i-=1 From 1a0eb7401adc4540ef0bb8b68e5748ecb3aa234a Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Wed, 28 Nov 2018 11:19:52 -0500 Subject: [PATCH 08/44] Find failed tests Change-Id: Ie7b9209210974895fe17d440275380e2d20d957e Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/multi-node-test.sh b/multi-node-test.sh index c2857fae..89756bee 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -86,21 +86,25 @@ python3 test_runner config=scripts/iof_fio_main.cfg scripts/iof_multi_two_node.y rc=\${PIPESTATUS[0]} echo \"Test exited with \$rc\" } +find testLogs/testRun -name subtest_results.yml -exec grep -Hi fail {} \\; mv testLogs/testRun{,-fio} python3 test_runner config=scripts/iof_ior_main.cfg scripts/iof_multi_two_node.yml || { rc=\${PIPESTATUS[0]} echo \"Test exited with \$rc\" } +find testLogs/testRun -name subtest_results.yml -exec grep -Hi fail {} \\; mv testLogs/testRun{,-ior} python3 test_runner config=scripts/iof_iozone_main.cfg scripts/iof_multi_two_node.yml || { rc=\${PIPESTATUS[0]} echo \"Test exited with \$rc\" } +find testLogs/testRun -name subtest_results.yml -exec grep -Hi fail {} \\; mv testLogs/testRun{,-iozone} python3 test_runner config=scripts/iof_mdtest_main.cfg scripts/iof_m_two_node.yml || { rc=\${PIPESTATUS[0]} echo \"Test exited with \$rc\" } +find testLogs/testRun -name subtest_results.yml -exec grep -Hi fail {} \\; mv testLogs/testRun{,-mdtest} ls -l From 4071b747ae70e75fc6b73a8fced97b951e80acf5 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Wed, 28 Nov 2018 15:55:20 -0500 Subject: [PATCH 09/44] Update scons_local to get pmix build fix Change-Id: Ibc3aa48a68f009d86cf5182c4bc00041c1e25345 Signed-off-by: Brian J. Murrell --- scons_local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scons_local b/scons_local index 205b87f3..3398a375 160000 --- a/scons_local +++ b/scons_local @@ -1 +1 @@ -Subproject commit 205b87f329cc5e50f539dc04084c41b2b6edefd3 +Subproject commit 3398a37578ac9120727fe310ba4aed916dff4187 From e05dfc6c1c025f2986a153bdc1af5b4617824bd9 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Wed, 28 Nov 2018 16:31:31 -0500 Subject: [PATCH 10/44] Switch scons_local to github copy Change-Id: I7b8a055171cdc30ea2ade7e7b2fcaebc204ea7ba Signed-off-by: Brian J. Murrell --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 58f89d39..87a00e09 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "scons_local"] path = scons_local - url = https://review.hpdd.intel.com/coral/scons_local + url = https://github.com/daos-stack/scons_local.git From df900405ce223907da36fc39d3c7ce2531fae1fb Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 29 Nov 2018 06:22:49 -0500 Subject: [PATCH 11/44] Make finding test output easier Change-Id: If38e0592a08591205bc035d39b82efe8e14cd406 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 89756bee..7f6efa2f 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -86,26 +86,26 @@ python3 test_runner config=scripts/iof_fio_main.cfg scripts/iof_multi_two_node.y rc=\${PIPESTATUS[0]} echo \"Test exited with \$rc\" } -find testLogs/testRun -name subtest_results.yml -exec grep -Hi fail {} \\; mv testLogs/testRun{,-fio} +find testLogs/testRun-fio -name subtest_results.yml -exec grep -Hi fail {} \\; python3 test_runner config=scripts/iof_ior_main.cfg scripts/iof_multi_two_node.yml || { rc=\${PIPESTATUS[0]} echo \"Test exited with \$rc\" } -find testLogs/testRun -name subtest_results.yml -exec grep -Hi fail {} \\; mv testLogs/testRun{,-ior} +find testLogs/testRun-ior -name subtest_results.yml -exec grep -Hi fail {} \\; python3 test_runner config=scripts/iof_iozone_main.cfg scripts/iof_multi_two_node.yml || { rc=\${PIPESTATUS[0]} echo \"Test exited with \$rc\" } -find testLogs/testRun -name subtest_results.yml -exec grep -Hi fail {} \\; mv testLogs/testRun{,-iozone} -python3 test_runner config=scripts/iof_mdtest_main.cfg scripts/iof_m_two_node.yml || { +find testLogs/testRun-iozone -name subtest_results.yml -exec grep -Hi fail {} \\; +python3 test_runner config=scripts/iof_mdtest_main.cfg scripts/iof_multi_two_node.yml || { rc=\${PIPESTATUS[0]} echo \"Test exited with \$rc\" } -find testLogs/testRun -name subtest_results.yml -exec grep -Hi fail {} \\; mv testLogs/testRun{,-mdtest} +find testLogs/testRun-mdtest -name subtest_results.yml -exec grep -Hi fail {} \\; ls -l exit \$rc"; then From 080e64b9d23c45fdd7569bdbea48352e8141fd1d Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 29 Nov 2018 07:01:14 -0500 Subject: [PATCH 12/44] Update to landed scons_local fix for IOF-893 Change-Id: Ibb5564cc841c475361643399bd9119e908fef892 Signed-off-by: Brian J. Murrell --- scons_local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scons_local b/scons_local index 3398a375..3d544ebe 160000 --- a/scons_local +++ b/scons_local @@ -1 +1 @@ -Subproject commit 3398a37578ac9120727fe310ba4aed916dff4187 +Subproject commit 3d544ebe05c90dc648698fd17eb80b61748e4c2c From 3b4fcce9dd23ccfb192473992d3e8c89ed086bd8 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 29 Nov 2018 12:38:37 -0500 Subject: [PATCH 13/44] Add 5 node test Summarize subtest_results.yml files and convert junit. Change-Id: I4b621c75e3e49f64e7a8a22f4d522b5f3dc5cca9 Signed-off-by: Brian J. Murrell --- Jenkinsfile | 64 ++++++++++++++++++++++++---- multi-node-test.sh | 101 ++++++++++++++++++++++++++++++++------------- 2 files changed, 128 insertions(+), 37 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3ffe5d7d..f6449b2d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes -@Library(value="pipeline-lib@debug-iof") _ +@Library(value="pipeline-lib@debug") _ pipeline { agent any @@ -174,11 +174,15 @@ pipeline { //NOTESTYET } //NOTESTYET } stage('Test') { + /* pity we cannot do this. these three tests would fit nicely + * on a single 8 node cluster + * https://issues.jenkins-ci.org/browse/JENKINS-54945 + agent { + label 'cluster_provisioner' + } + */ parallel { stage('Single node') { - agent { - label 'single' - } steps { runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], script: '''id || true @@ -216,14 +220,57 @@ pipeline { } } } - stage('Multi-node') { + stage('Two-node') { + agent { + label 'cluster_provisioner2' + } + steps { + checkoutSCM url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins', + checkoutDir: 'jenkins', + credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' + + checkoutSCM url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit', + checkoutDir: 'scony_python-junit', + credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' + + runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], + script: 'bash -x ./multi-node-test.sh 2; echo "rc: $?"', + junit_files: IOF_5-node_junit.xml + } + post { + /* temporarily moved into runTest->stepResult due to JENKINS-39203 + success { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'Functional daos_test', context: 'test/functional_daos_test', status: 'SUCCESS' + } + unstable { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'Functional daos_test', context: 'test/functional_daos_test', status: 'FAILURE' + } + failure { + githubNotify credentialsId: 'daos-jenkins-commit-status', description: 'Functional daos_test', context: 'test/functional_daos_test', status: 'ERROR' + } + */ + always { + junit 'IOF_5-node_junit.xml' + archiveArtifacts artifacts: 'install/Linux/TESTING/testLogs/**' + } + } + } + stage('Five-node') { agent { label 'cluster_provisioner' } steps { + checkoutSCM url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins', + checkoutDir: 'jenkins', + credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' + + checkoutSCM url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit', + checkoutDir: 'scony_python-junit', + credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' + runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], - script: 'bash -x ./multi-node-test.sh; echo "rc: $?"', - junit_files: null + script: 'bash -x ./multi-node-test.sh 5; echo "rc: $?"', + junit_files: IOF_5-node_junit.xml } post { /* temporarily moved into runTest->stepResult due to JENKINS-39203 @@ -238,8 +285,7 @@ pipeline { } */ always { - sh 'echo "Nothing to do"' - //junit 'install/Linux/tmp/*results.xml, Functional daos_test/*/results.xml' + junit 'IOF_5-node_junit.xml' archiveArtifacts artifacts: 'install/Linux/TESTING/testLogs/**' } } diff --git a/multi-node-test.sh b/multi-node-test.sh index 7f6efa2f..7501cbc4 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -31,7 +31,7 @@ while [ $i -gt 0 ]; do let x+=1 done sudo rmdir $DAOS_BASE || find $DAOS_BASE || true" 2>&1 | dshbak -c - if [ \${PIPESTATUS[0]} = 0 ]; then + if [ ${PIPESTATUS[0]} = 0 ]; then i=0 fi let i-=1 @@ -61,7 +61,8 @@ echo "hit enter to continue" #read -r #exit 0 -cat < install/Linux/TESTING/scripts/iof_fio_main.cfg +if [ "$1" = "2" ]; then + cat < install/Linux/TESTING/scripts/iof_fio_main.cfg { "host_list": ["${HOSTPREFIX}vm2", "${HOSTPREFIX}vm3"], "test_mode": "littleChief" @@ -70,6 +71,20 @@ EOF cp install/Linux/TESTING/scripts/iof_{fio,ior}_main.cfg cp install/Linux/TESTING/scripts/iof_{fio,iozone}_main.cfg cp install/Linux/TESTING/scripts/iof_{fio,mdtest}_main.cfg +elif [ "$1" = "5" ]; then + cat < install/Linux/TESTING/scripts/iof_multi_five_node.cfg +{ + "host_list": [ + "${HOSTPREFIX}vm2", + "${HOSTPREFIX}vm3", + "${HOSTPREFIX}vm4", + "${HOSTPREFIX}vm5", + "${HOSTPREFIX}vm6" + ], + "test_mode": "littleChief" +} +EOF +fi rm -rf install/Linux/TESTING/testLogs/ rm -f install/Linux/bin/fusermount3 @@ -82,38 +97,68 @@ cd $DAOS_BASE # now run it! pushd install/Linux/TESTING -python3 test_runner config=scripts/iof_fio_main.cfg scripts/iof_multi_two_node.yml || { - rc=\${PIPESTATUS[0]} - echo \"Test exited with \$rc\" -} -mv testLogs/testRun{,-fio} -find testLogs/testRun-fio -name subtest_results.yml -exec grep -Hi fail {} \\; -python3 test_runner config=scripts/iof_ior_main.cfg scripts/iof_multi_two_node.yml || { - rc=\${PIPESTATUS[0]} - echo \"Test exited with \$rc\" -} -mv testLogs/testRun{,-ior} -find testLogs/testRun-ior -name subtest_results.yml -exec grep -Hi fail {} \\; -python3 test_runner config=scripts/iof_iozone_main.cfg scripts/iof_multi_two_node.yml || { - rc=\${PIPESTATUS[0]} - echo \"Test exited with \$rc\" -} -mv testLogs/testRun{,-iozone} -find testLogs/testRun-iozone -name subtest_results.yml -exec grep -Hi fail {} \\; -python3 test_runner config=scripts/iof_mdtest_main.cfg scripts/iof_multi_two_node.yml || { - rc=\${PIPESTATUS[0]} - echo \"Test exited with \$rc\" -} -mv testLogs/testRun{,-mdtest} -find testLogs/testRun-mdtest -name subtest_results.yml -exec grep -Hi fail {} \\; - -ls -l +if [ \"$1\" = \"2\" ]; then + python3 test_runner config=scripts/iof_fio_main.cfg \\ + scripts/iof_multi_two_node.yml || { + rc=\${PIPESTATUS[0]} + echo \"Test exited with \$rc\" + } + mv testLogs/testRun{,-fio} + find testLogs/testRun-fio -name subtest_results.yml \\ + -exec grep -Hi fail {} \\; + python3 test_runner config=scripts/iof_ior_main.cfg \\ + scripts/iof_multi_two_node.yml || { + rc=\${PIPESTATUS[0]} + echo \"Test exited with \$rc\" + } + mv testLogs/testRun{,-ior} + find testLogs/testRun-ior -name subtest_results.yml \\ + -exec grep -Hi fail {} \\; + python3 test_runner config=scripts/iof_iozone_main.cfg \\ + scripts/iof_multi_two_node.yml || { + rc=\${PIPESTATUS[0]} + echo \"Test exited with \$rc\" + } + mv testLogs/testRun{,-iozone} + find testLogs/testRun-iozone -name subtest_results.yml \\ + -exec grep -Hi fail {} \\; + python3 test_runner config=scripts/iof_mdtest_main.cfg \\ + scripts/iof_multi_two_node.yml || { + rc=\${PIPESTATUS[0]} + echo \"Test exited with \$rc\" + } + mv testLogs/testRun{,-mdtest} + find testLogs/testRun-mdtest -name subtest_results.yml \\ + -exec grep -Hi fail {} \\; +elif [ \"$1\" = \"5\" ]; then + python3 test_runner config=scripts/iof_multi_five_node.cfg \\ + scripts/iof_multi_five_node.yml || { + rc=\${PIPESTATUS[0]} + echo \"Test exited with \$rc\" + } + mv testLogs/testRun{,-five_node} +fi exit \$rc"; then rc=${PIPESTATUS[0]} else rc=0 fi +{ + cat < results_1.yml + +PYTHONPATH=scony_python-junit/ jenkins/autotest_utils/results_to_junit.py + if false; then # collect the logs if ! rpdcp -R ssh -w "${HOSTPREFIX}"vm[1-9] \ From 93b45862196236ba30d7215af6978a8e999aa4fb Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 29 Nov 2018 14:29:21 -0500 Subject: [PATCH 14/44] checkoutScm -> checkout() until it's fixed Change-Id: I8f25e992600d5efb5c3754d81c96953057c4b6ba Signed-off-by: Brian J. Murrell --- Jenkinsfile | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f6449b2d..f05fb66f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -225,13 +225,41 @@ pipeline { label 'cluster_provisioner2' } steps { - checkoutSCM url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins', + /* not working yet + checkoutScm url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins', checkoutDir: 'jenkins', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' - checkoutSCM url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit', + checkoutScm url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit', checkoutDir: 'scony_python-junit', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' + */ + + checkout([ + $class: 'GitSCM', + branches: [[name: branch]], + extensions: [[$class: 'RelativeTargetDirectory', + relativeTargetDir: 'jenkins']], + submoduleCfg: [], + userRemoteConfigs: [[ + credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a', + refspec: refspec, + url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins' + ]] + ]) + + checkout([ + $class: 'GitSCM', + branches: [[name: branch]], + extensions: [[$class: 'RelativeTargetDirectory', + relativeTargetDir: 'scony_python-junit']], + submoduleCfg: [], + userRemoteConfigs: [[ + credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a', + refspec: refspec, + url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit' + ]] + ]) runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], script: 'bash -x ./multi-node-test.sh 2; echo "rc: $?"', From b7374314ea0f0edd8206e87eddc16b9b74d1ac27 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 29 Nov 2018 15:59:26 -0500 Subject: [PATCH 15/44] Some fixes Change-Id: I17942c3ca0077a9de4dc30b3b0f869c15c08f727 Signed-off-by: Brian J. Murrell --- Jenkinsfile | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f05fb66f..1ff1333a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -183,6 +183,9 @@ pipeline { */ parallel { stage('Single node') { + agent { + label 'single' + } steps { runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], script: '''id || true @@ -222,9 +225,10 @@ pipeline { } stage('Two-node') { agent { - label 'cluster_provisioner2' + label 'cluster_provisioner-2' } steps { + echo "Starting Two-node" /* not working yet checkoutScm url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins', checkoutDir: 'jenkins', @@ -288,6 +292,8 @@ pipeline { label 'cluster_provisioner' } steps { + echo "Starting Five-node" + /* not working yet checkoutSCM url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins', checkoutDir: 'jenkins', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' @@ -295,6 +301,33 @@ pipeline { checkoutSCM url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit', checkoutDir: 'scony_python-junit', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' + */ + + checkout([ + $class: 'GitSCM', + branches: [[name: branch]], + extensions: [[$class: 'RelativeTargetDirectory', + relativeTargetDir: 'jenkins']], + submoduleCfg: [], + userRemoteConfigs: [[ + credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a', + refspec: refspec, + url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins' + ]] + ]) + + checkout([ + $class: 'GitSCM', + branches: [[name: branch]], + extensions: [[$class: 'RelativeTargetDirectory', + relativeTargetDir: 'scony_python-junit']], + submoduleCfg: [], + userRemoteConfigs: [[ + credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a', + refspec: refspec, + url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit' + ]] + ]) runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], script: 'bash -x ./multi-node-test.sh 5; echo "rc: $?"', From 2aa27a4af4039a4a79e6a27b97b0714c17366c94 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 29 Nov 2018 16:39:00 -0500 Subject: [PATCH 16/44] Try PR 8 again Change-Id: Ia78c3f535f812c919375d5b7510fe76d85214c31 Signed-off-by: Brian J. Murrell --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1ff1333a..34da1ee5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -229,7 +229,6 @@ pipeline { } steps { echo "Starting Two-node" - /* not working yet checkoutScm url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins', checkoutDir: 'jenkins', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' @@ -237,8 +236,8 @@ pipeline { checkoutScm url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit', checkoutDir: 'scony_python-junit', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' - */ + /* not working yet checkout([ $class: 'GitSCM', branches: [[name: branch]], @@ -264,6 +263,7 @@ pipeline { url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit' ]] ]) + */ runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], script: 'bash -x ./multi-node-test.sh 2; echo "rc: $?"', @@ -293,7 +293,6 @@ pipeline { } steps { echo "Starting Five-node" - /* not working yet checkoutSCM url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins', checkoutDir: 'jenkins', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' @@ -301,8 +300,8 @@ pipeline { checkoutSCM url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit', checkoutDir: 'scony_python-junit', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' - */ + /* not working yet checkout([ $class: 'GitSCM', branches: [[name: branch]], @@ -328,6 +327,7 @@ pipeline { url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit' ]] ]) + */ runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], script: 'bash -x ./multi-node-test.sh 5; echo "rc: $?"', From b3c3012748d3cc5340125fb41e6bab6cf2fc329e Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 29 Nov 2018 17:20:33 -0500 Subject: [PATCH 17/44] Fixes and debug Change-Id: Ib88e0c6db4887695fdfe4bc03868f05bf59b4108 Signed-off-by: Brian J. Murrell --- Jenkinsfile | 66 +++++------------------------------------------------ 1 file changed, 6 insertions(+), 60 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 34da1ee5..ffc90b5a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -237,34 +237,7 @@ pipeline { checkoutDir: 'scony_python-junit', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' - /* not working yet - checkout([ - $class: 'GitSCM', - branches: [[name: branch]], - extensions: [[$class: 'RelativeTargetDirectory', - relativeTargetDir: 'jenkins']], - submoduleCfg: [], - userRemoteConfigs: [[ - credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a', - refspec: refspec, - url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins' - ]] - ]) - - checkout([ - $class: 'GitSCM', - branches: [[name: branch]], - extensions: [[$class: 'RelativeTargetDirectory', - relativeTargetDir: 'scony_python-junit']], - submoduleCfg: [], - userRemoteConfigs: [[ - credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a', - refspec: refspec, - url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit' - ]] - ]) - */ - + echo "Starting Two-node runTest" runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], script: 'bash -x ./multi-node-test.sh 2; echo "rc: $?"', junit_files: IOF_5-node_junit.xml @@ -282,7 +255,7 @@ pipeline { } */ always { - junit 'IOF_5-node_junit.xml' + //junit 'IOF_5-node_junit.xml' archiveArtifacts artifacts: 'install/Linux/TESTING/testLogs/**' } } @@ -293,42 +266,15 @@ pipeline { } steps { echo "Starting Five-node" - checkoutSCM url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins', + checkoutScm url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins', checkoutDir: 'jenkins', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' - checkoutSCM url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit', + checkoutScm url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit', checkoutDir: 'scony_python-junit', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' - /* not working yet - checkout([ - $class: 'GitSCM', - branches: [[name: branch]], - extensions: [[$class: 'RelativeTargetDirectory', - relativeTargetDir: 'jenkins']], - submoduleCfg: [], - userRemoteConfigs: [[ - credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a', - refspec: refspec, - url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins' - ]] - ]) - - checkout([ - $class: 'GitSCM', - branches: [[name: branch]], - extensions: [[$class: 'RelativeTargetDirectory', - relativeTargetDir: 'scony_python-junit']], - submoduleCfg: [], - userRemoteConfigs: [[ - credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a', - refspec: refspec, - url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit' - ]] - ]) - */ - + echo "Starting Five-node runTest" runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], script: 'bash -x ./multi-node-test.sh 5; echo "rc: $?"', junit_files: IOF_5-node_junit.xml @@ -346,7 +292,7 @@ pipeline { } */ always { - junit 'IOF_5-node_junit.xml' + //junit 'IOF_5-node_junit.xml' archiveArtifacts artifacts: 'install/Linux/TESTING/testLogs/**' } } From 70b988291e98a253898ab9461fff53a3dd0a6268 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Fri, 30 Nov 2018 10:34:10 -0500 Subject: [PATCH 18/44] Rename provisioners; test checkout is killing job Change-Id: I9c5b964f1cae45e83b1893a4117932e6c48aedec Signed-off-by: Brian J. Murrell --- Jenkinsfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ffc90b5a..eadb02a8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -225,10 +225,11 @@ pipeline { } stage('Two-node') { agent { - label 'cluster_provisioner-2' + label 'cluster_provisioner-2_nodes' } steps { echo "Starting Two-node" + /* checkoutScm url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins', checkoutDir: 'jenkins', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' @@ -236,6 +237,7 @@ pipeline { checkoutScm url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit', checkoutDir: 'scony_python-junit', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' + */ echo "Starting Two-node runTest" runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], @@ -262,10 +264,11 @@ pipeline { } stage('Five-node') { agent { - label 'cluster_provisioner' + label 'cluster_provisioner-5_nodes' } steps { echo "Starting Five-node" + /* checkoutScm url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins', checkoutDir: 'jenkins', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' @@ -273,6 +276,7 @@ pipeline { checkoutScm url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit', checkoutDir: 'scony_python-junit', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' + */ echo "Starting Five-node runTest" runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], From 7a99cd9b8f04696417f42b465e8e8e8b36c9b344 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Fri, 30 Nov 2018 10:34:40 -0500 Subject: [PATCH 19/44] Determine 2 node VMs Change-Id: Id915faf4b7c8b1f45aff9710f391b249ce5123c0 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 7501cbc4..e83cb8c6 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -62,9 +62,11 @@ echo "hit enter to continue" #exit 0 if [ "$1" = "2" ]; then + vm1=vm"$(((EXECUTOR_NUMBER+4)*2-1))" + vm2=vm"$(((EXECUTOR_NUMBER+4)*2)) cat < install/Linux/TESTING/scripts/iof_fio_main.cfg { - "host_list": ["${HOSTPREFIX}vm2", "${HOSTPREFIX}vm3"], + "host_list": ["${HOSTPREFIX}${vm1}", "${HOSTPREFIX}${vm2}"], "test_mode": "littleChief" } EOF From b80b45fad5a07e3e3d9bcc683444cc5546dd274d Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Fri, 30 Nov 2018 11:55:56 -0500 Subject: [PATCH 20/44] Fix missing quote Change-Id: I61e93dc583513929d7d846cf36ea69589345cfad Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index e83cb8c6..add2df04 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -63,7 +63,7 @@ echo "hit enter to continue" if [ "$1" = "2" ]; then vm1=vm"$(((EXECUTOR_NUMBER+4)*2-1))" - vm2=vm"$(((EXECUTOR_NUMBER+4)*2)) + vm2=vm"$(((EXECUTOR_NUMBER+4)*2))" cat < install/Linux/TESTING/scripts/iof_fio_main.cfg { "host_list": ["${HOSTPREFIX}${vm1}", "${HOSTPREFIX}${vm2}"], From b4bcdb0319ae558c800684b9573bd8ce407a22a9 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Fri, 30 Nov 2018 11:56:02 -0500 Subject: [PATCH 21/44] Need to quote junit file names Change-Id: Ibbbb92203b1cda60c645c1cc46b85498ef4e4364 Signed-off-by: Brian J. Murrell --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index eadb02a8..881dec8e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -242,7 +242,7 @@ pipeline { echo "Starting Two-node runTest" runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], script: 'bash -x ./multi-node-test.sh 2; echo "rc: $?"', - junit_files: IOF_5-node_junit.xml + junit_files: "IOF_2-node_junit.xml" } post { /* temporarily moved into runTest->stepResult due to JENKINS-39203 @@ -281,7 +281,7 @@ pipeline { echo "Starting Five-node runTest" runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], script: 'bash -x ./multi-node-test.sh 5; echo "rc: $?"', - junit_files: IOF_5-node_junit.xml + junit_files: "IOF_5-node_junit.xml" } post { /* temporarily moved into runTest->stepResult due to JENKINS-39203 From 44770dc8a1c606ae84d2b4e0cc54527dcf8e7bd9 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Fri, 30 Nov 2018 13:39:45 -0500 Subject: [PATCH 22/44] Fix host selection Change-Id: Iedda92e2b1cb54bac5a1b8f272c9afac747c7543 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index add2df04..ffd6c192 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -16,12 +16,21 @@ trap 'echo "encountered an unchecked return code, exiting with error"' ERR # shellcheck disable=SC1091 . .build_vars-Linux.sh +if [ "$1" = "2" ]; then + vm2="$(((${EXECUTOR_NUMBER:-0}+4)*2))" + vm1="$((vm2-1))" + vmrange="$vm1-$vm2" + vm1="vm$vm1" + vm2="vm$vm2" +elif [ "$1" = "5" ]; then + vmrange="2-6" +fi # shellcheck disable=SC2154 trap 'set +e i=5 # due to flakiness on wolf-53, try this several times while [ $i -gt 0 ]; do - pdsh -R ssh -S -w ${HOSTPREFIX}vm[1-9] "set -x + pdsh -R ssh -S -w ${HOSTPREFIX}vm[1,$vmrange] "set -x x=0 while [ \$x -lt 30 ] && grep $DAOS_BASE /proc/mounts && @@ -30,6 +39,7 @@ while [ $i -gt 0 ]; do sleep 1 let x+=1 done + sudo sed -i -e \"/added by multi-node-test-$1.sh/d\" /etc/fstab sudo rmdir $DAOS_BASE || find $DAOS_BASE || true" 2>&1 | dshbak -c if [ ${PIPESTATUS[0]} = 0 ]; then i=0 @@ -38,12 +48,12 @@ while [ $i -gt 0 ]; do done' EXIT DAOS_BASE=${SL_OMPI_PREFIX%/install/*} -if ! pdsh -R ssh -S -w "${HOSTPREFIX}"vm[1-9] "set -ex +if ! pdsh -R ssh -S -w "${HOSTPREFIX}"vm[1,$vmrange] "set -ex ulimit -c unlimited sudo mkdir -p $DAOS_BASE sudo ed < install/Linux/TESTING/scripts/iof_fio_main.cfg { "host_list": ["${HOSTPREFIX}${vm1}", "${HOSTPREFIX}${vm2}"], @@ -163,7 +171,7 @@ PYTHONPATH=scony_python-junit/ jenkins/autotest_utils/results_to_junit.py if false; then # collect the logs -if ! rpdcp -R ssh -w "${HOSTPREFIX}"vm[1-9] \ +if ! rpdcp -R ssh -w "${HOSTPREFIX}"vm[1,$vmrange] \ /tmp/Functional_"$TEST_TAG"/\*daos.log "$PWD"/; then echo "Copying daos.logs from remote nodes failed" # pass From a3224b6028a0975cd1487b08803a4f97e6629ae2 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Fri, 30 Nov 2018 15:03:42 -0500 Subject: [PATCH 23/44] Remove checkout other needed repos disabling Change-Id: Id1c558768782f864aa577068b0688958c7835745 Signed-off-by: Brian J. Murrell --- Jenkinsfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 881dec8e..6acd9107 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -229,7 +229,6 @@ pipeline { } steps { echo "Starting Two-node" - /* checkoutScm url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins', checkoutDir: 'jenkins', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' @@ -237,7 +236,6 @@ pipeline { checkoutScm url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit', checkoutDir: 'scony_python-junit', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' - */ echo "Starting Two-node runTest" runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], @@ -268,7 +266,6 @@ pipeline { } steps { echo "Starting Five-node" - /* checkoutScm url: 'ssh://review.hpdd.intel.com:29418/exascale/jenkins', checkoutDir: 'jenkins', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' @@ -276,7 +273,6 @@ pipeline { checkoutScm url: 'ssh://review.hpdd.intel.com:29418/coral/scony_python-junit', checkoutDir: 'scony_python-junit', credentialsId: 'bf21c68b-9107-4a38-8077-e929e644996a' - */ echo "Starting Five-node runTest" runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], From f941afe2e36cbe787cd33f878151743bad7e63dc Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Fri, 30 Nov 2018 15:08:10 -0500 Subject: [PATCH 24/44] Allow for multiple test runner users Change-Id: Ic1ffa013978d8f13fdd0ebff96d0ca8048404587 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index ffd6c192..6f551a88 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -57,7 +57,15 @@ $NFS_SERVER:$PWD $DAOS_BASE nfs defaults 0 0 # added by multi-node-test-$1.sh . wq EOF -sudo mount $DAOS_BASE +if ! sudo mount $DAOS_BASE; them + if [ \"\${HOSTNAME%%%%.*}\" != \"${HOSTPREFIX}\"vm1 ]; then + # could be already mounted from another test running in parallel + # let's see what that rc is + echo \"mount rc: \${PIPESTATUS[0]}\" + else + exit \${PIPESTATUS[0]}\ + fi +fi # TODO: package this in to an RPM pip3 install --user tabulate From 20044179e057db275736624a4f804d5c72ef442c Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Fri, 30 Nov 2018 15:49:43 -0500 Subject: [PATCH 25/44] Couple of fixes Change-Id: I60d84ddbc86161be41d0e69f35ea2679ae318f80 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 6f551a88..45aa1eca 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -57,13 +57,13 @@ $NFS_SERVER:$PWD $DAOS_BASE nfs defaults 0 0 # added by multi-node-test-$1.sh . wq EOF -if ! sudo mount $DAOS_BASE; them +if ! sudo mount $DAOS_BASE; then if [ \"\${HOSTNAME%%%%.*}\" != \"${HOSTPREFIX}\"vm1 ]; then # could be already mounted from another test running in parallel # let's see what that rc is echo \"mount rc: \${PIPESTATUS[0]}\" else - exit \${PIPESTATUS[0]}\ + exit \${PIPESTATUS[0]} fi fi From 45c9afe4f2ae506096fce35198e80759aa452801 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Fri, 30 Nov 2018 21:59:27 -0500 Subject: [PATCH 26/44] A fix and some debug Change-Id: Ic677433ea4483af1d26e7974b9355557b691e5c8 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 45aa1eca..81ef5cb8 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -58,7 +58,7 @@ $NFS_SERVER:$PWD $DAOS_BASE nfs defaults 0 0 # added by multi-node-test-$1.sh wq EOF if ! sudo mount $DAOS_BASE; then - if [ \"\${HOSTNAME%%%%.*}\" != \"${HOSTPREFIX}\"vm1 ]; then + if [ \"\${HOSTNAME%%%%.*}\" = \"${HOSTPREFIX}\"vm1 ]; then # could be already mounted from another test running in parallel # let's see what that rc is echo \"mount rc: \${PIPESTATUS[0]}\" @@ -102,6 +102,8 @@ elif [ "$1" = "5" ]; then "test_mode": "littleChief" } EOF +pwd +ls -l install/Linux/TESTING/scripts/iof_multi_five_node.cfg fi rm -rf install/Linux/TESTING/testLogs/ @@ -149,6 +151,8 @@ if [ \"$1\" = \"2\" ]; then find testLogs/testRun-mdtest -name subtest_results.yml \\ -exec grep -Hi fail {} \\; elif [ \"$1\" = \"5\" ]; then +pwd +ls -l scripts/iof_multi_five_node.cfg python3 test_runner config=scripts/iof_multi_five_node.cfg \\ scripts/iof_multi_five_node.yml || { rc=\${PIPESTATUS[0]} From 09548ae60d3e69c6e137523f438cf55f2c482080 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Fri, 30 Nov 2018 23:15:18 -0500 Subject: [PATCH 27/44] Move .cfg file creation to test runner Change-Id: Idee60006509b06442804cc89c670d46301777d00 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 51 ++++++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 81ef5cb8..62a8eb6a 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -79,33 +79,6 @@ echo "hit enter to continue" #read -r #exit 0 -if [ "$1" = "2" ]; then - cat < install/Linux/TESTING/scripts/iof_fio_main.cfg -{ - "host_list": ["${HOSTPREFIX}${vm1}", "${HOSTPREFIX}${vm2}"], - "test_mode": "littleChief" -} -EOF -cp install/Linux/TESTING/scripts/iof_{fio,ior}_main.cfg -cp install/Linux/TESTING/scripts/iof_{fio,iozone}_main.cfg -cp install/Linux/TESTING/scripts/iof_{fio,mdtest}_main.cfg -elif [ "$1" = "5" ]; then - cat < install/Linux/TESTING/scripts/iof_multi_five_node.cfg -{ - "host_list": [ - "${HOSTPREFIX}vm2", - "${HOSTPREFIX}vm3", - "${HOSTPREFIX}vm4", - "${HOSTPREFIX}vm5", - "${HOSTPREFIX}vm6" - ], - "test_mode": "littleChief" -} -EOF -pwd -ls -l install/Linux/TESTING/scripts/iof_multi_five_node.cfg -fi - rm -rf install/Linux/TESTING/testLogs/ rm -f install/Linux/bin/fusermount3 ln -s "$(command -v fusermount)" install/Linux/bin/fusermount3 @@ -118,6 +91,16 @@ cd $DAOS_BASE # now run it! pushd install/Linux/TESTING if [ \"$1\" = \"2\" ]; then + cat < scripts/iof_fio_main.cfg +{ + \"host_list\": [\"${HOSTPREFIX}${vm1}\", \"${HOSTPREFIX}${vm2}\"], + \"test_mode\": \"littleChief\" +} +EOF + cp scripts/iof_{fio,ior}_main.cfg + cp scripts/iof_{fio,iozone}_main.cfg + cp scripts/iof_{fio,mdtest}_main.cfg + python3 test_runner config=scripts/iof_fio_main.cfg \\ scripts/iof_multi_two_node.yml || { rc=\${PIPESTATUS[0]} @@ -151,8 +134,18 @@ if [ \"$1\" = \"2\" ]; then find testLogs/testRun-mdtest -name subtest_results.yml \\ -exec grep -Hi fail {} \\; elif [ \"$1\" = \"5\" ]; then -pwd -ls -l scripts/iof_multi_five_node.cfg + cat < scripts/iof_multi_five_node.cfg +{ + \"host_list\": [ + \"${HOSTPREFIX}vm2\", + \"${HOSTPREFIX}vm3\", + \"${HOSTPREFIX}vm4\", + \"${HOSTPREFIX}vm5\", + \"${HOSTPREFIX}vm6\" + ], + \"test_mode\": \"littleChief\" +} +EOF python3 test_runner config=scripts/iof_multi_five_node.cfg \\ scripts/iof_multi_five_node.yml || { rc=\${PIPESTATUS[0]} From 0e4c2257af8f3c5f15d8fc71a6bcdf61b58055c5 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Fri, 30 Nov 2018 23:19:00 -0500 Subject: [PATCH 28/44] Enable junit processing Change-Id: I33300abeb525c6d38403abde319712666e0000e5 Signed-off-by: Brian J. Murrell --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6acd9107..c7f626c7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -255,7 +255,7 @@ pipeline { } */ always { - //junit 'IOF_5-node_junit.xml' + junit 'IOF_5-node_junit.xml' archiveArtifacts artifacts: 'install/Linux/TESTING/testLogs/**' } } @@ -292,7 +292,7 @@ pipeline { } */ always { - //junit 'IOF_5-node_junit.xml' + junit 'IOF_5-node_junit.xml' archiveArtifacts artifacts: 'install/Linux/TESTING/testLogs/**' } } From 6d2174438cd379632a0f1b419d22bec73a9045a3 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Mon, 3 Dec 2018 09:14:54 -0500 Subject: [PATCH 29/44] Each test needs it's own test runner Change-Id: If2d556e123afb339c40bbe2f9d83a21e2c1a7bd4 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 62a8eb6a..4192019c 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -17,12 +17,14 @@ trap 'echo "encountered an unchecked return code, exiting with error"' ERR . .build_vars-Linux.sh if [ "$1" = "2" ]; then - vm2="$(((${EXECUTOR_NUMBER:-0}+4)*2))" - vm1="$((vm2-1))" + test_runner_vm=$((${EXECUTOR_NUMBER:-0}*3+7)) + vm1="$((test_runner_vm+1))" + vm2="$((test_runner_vm+2))" vmrange="$vm1-$vm2" vm1="vm$vm1" vm2="vm$vm2" elif [ "$1" = "5" ]; then + test_runner_vm="vm1" vmrange="2-6" fi # shellcheck disable=SC2154 @@ -58,7 +60,7 @@ $NFS_SERVER:$PWD $DAOS_BASE nfs defaults 0 0 # added by multi-node-test-$1.sh wq EOF if ! sudo mount $DAOS_BASE; then - if [ \"\${HOSTNAME%%%%.*}\" = \"${HOSTPREFIX}\"vm1 ]; then + if [ \"\${HOSTNAME%%%%.*}\" = \"${HOSTPREFIX}\"\"$test_runner_vm\" ]; then # could be already mounted from another test running in parallel # let's see what that rc is echo \"mount rc: \${PIPESTATUS[0]}\" @@ -84,7 +86,7 @@ rm -f install/Linux/bin/fusermount3 ln -s "$(command -v fusermount)" install/Linux/bin/fusermount3 # shellcheck disable=SC2029 -if ! ssh "${HOSTPREFIX}"vm1 "set -ex +if ! ssh "${HOSTPREFIX}""$test_runner_vm" "set -ex ulimit -c unlimited cd $DAOS_BASE From 2d231d2e9e956627c8ea0567e68248ea6ee8287d Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Mon, 3 Dec 2018 10:21:40 -0500 Subject: [PATCH 30/44] Fix up previous Change-Id: I37f4bf945f6e937a06a55db92ebee90f0cb821e9 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 4192019c..1a08b7ff 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -21,6 +21,7 @@ if [ "$1" = "2" ]; then vm1="$((test_runner_vm+1))" vm2="$((test_runner_vm+2))" vmrange="$vm1-$vm2" + test_runner_vm="vm$test_runner_vm" vm1="vm$vm1" vm2="vm$vm2" elif [ "$1" = "5" ]; then @@ -50,7 +51,7 @@ while [ $i -gt 0 ]; do done' EXIT DAOS_BASE=${SL_OMPI_PREFIX%/install/*} -if ! pdsh -R ssh -S -w "${HOSTPREFIX}"vm[1,$vmrange] "set -ex +if ! pdsh -R ssh -S -w "${HOSTPREFIX}$test_runner_vm,${HOSTPREFIX}vm[$vmrange]" "set -ex ulimit -c unlimited sudo mkdir -p $DAOS_BASE sudo ed < Date: Mon, 3 Dec 2018 12:37:06 -0500 Subject: [PATCH 31/44] Use a log_base_path to run tests in parallel Change-Id: I72de3a45443ba580c99ebe3e39b44930ed274826 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 1a08b7ff..93280d34 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -28,6 +28,9 @@ elif [ "$1" = "5" ]; then test_runner_vm="vm1" vmrange="2-6" fi + +log_base_path="testLogs-${1}_node" + # shellcheck disable=SC2154 trap 'set +e i=5 @@ -78,11 +81,10 @@ df -h" 2>&1 | dshbak -c; then exit 1 fi -echo "hit enter to continue" +#echo "hit enter to continue" #read -r #exit 0 -rm -rf install/Linux/TESTING/testLogs/ rm -f install/Linux/bin/fusermount3 ln -s "$(command -v fusermount)" install/Linux/bin/fusermount3 @@ -97,44 +99,46 @@ if [ \"$1\" = \"2\" ]; then cat < scripts/iof_fio_main.cfg { \"host_list\": [\"${HOSTPREFIX}${vm1}\", \"${HOSTPREFIX}${vm2}\"], - \"test_mode\": \"littleChief\" + \"test_mode\": \"littleChief\", + \"log_base_path\": \"$log_base_path\" } EOF cp scripts/iof_{fio,ior}_main.cfg cp scripts/iof_{fio,iozone}_main.cfg cp scripts/iof_{fio,mdtest}_main.cfg + rm -rf install/Linux/TESTING/$log_base_path/ python3 test_runner config=scripts/iof_fio_main.cfg \\ scripts/iof_multi_two_node.yml || { rc=\${PIPESTATUS[0]} echo \"Test exited with \$rc\" } - mv testLogs/testRun{,-fio} - find testLogs/testRun-fio -name subtest_results.yml \\ + mv $log_base_path/testRun{,-fio} + find $log_base_path/testRun-fio -name subtest_results.yml \\ -exec grep -Hi fail {} \\; python3 test_runner config=scripts/iof_ior_main.cfg \\ scripts/iof_multi_two_node.yml || { rc=\${PIPESTATUS[0]} echo \"Test exited with \$rc\" } - mv testLogs/testRun{,-ior} - find testLogs/testRun-ior -name subtest_results.yml \\ + mv $log_base_path/testRun{,-ior} + find $log_base_path/testRun-ior -name subtest_results.yml \\ -exec grep -Hi fail {} \\; python3 test_runner config=scripts/iof_iozone_main.cfg \\ scripts/iof_multi_two_node.yml || { rc=\${PIPESTATUS[0]} echo \"Test exited with \$rc\" } - mv testLogs/testRun{,-iozone} - find testLogs/testRun-iozone -name subtest_results.yml \\ + mv $log_base_path/testRun{,-iozone} + find $log_base_path/testRun-iozone -name subtest_results.yml \\ -exec grep -Hi fail {} \\; python3 test_runner config=scripts/iof_mdtest_main.cfg \\ scripts/iof_multi_two_node.yml || { rc=\${PIPESTATUS[0]} echo \"Test exited with \$rc\" } - mv testLogs/testRun{,-mdtest} - find testLogs/testRun-mdtest -name subtest_results.yml \\ + mv $log_base_path/testRun{,-mdtest} + find $log_base_path/testRun-mdtest -name subtest_results.yml \\ -exec grep -Hi fail {} \\; elif [ \"$1\" = \"5\" ]; then cat < scripts/iof_multi_five_node.cfg @@ -146,15 +150,18 @@ elif [ \"$1\" = \"5\" ]; then \"${HOSTPREFIX}vm5\", \"${HOSTPREFIX}vm6\" ], - \"test_mode\": \"littleChief\" + \"test_mode\": \"littleChief\", + \"log_base_path\": \"$log_base_path\" } EOF + rm -rf install/Linux/TESTING/$log_base_path/ python3 test_runner config=scripts/iof_multi_five_node.cfg \\ scripts/iof_multi_five_node.yml || { rc=\${PIPESTATUS[0]} echo \"Test exited with \$rc\" } - mv testLogs/testRun{,-five_node} + find $log_base_path/testRun -name subtest_results.yml \\ + -exec grep -Hi fail {} \\; fi exit \$rc"; then rc=${PIPESTATUS[0]} @@ -171,8 +178,8 @@ TestGroup: user_name: jenkins Tests: EOF - find install/Linux/TESTING/testLogs -name subtest_results.yml -print0 | \ - xargs -0 cat + find install/Linux/TESTING/$log_base_path -name subtest_results.yml \ + -print0 | xargs -0 cat } > results_1.yml PYTHONPATH=scony_python-junit/ jenkins/autotest_utils/results_to_junit.py From 2c9aaf697b13319b84c6f5e88d7341b82e6645f6 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Mon, 3 Dec 2018 13:22:13 -0500 Subject: [PATCH 32/44] Debug Change-Id: I5d69587b5e2b3c5113bf949dc1c3963be5f26ffc Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 93280d34..0e23292c 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -94,7 +94,7 @@ ulimit -c unlimited cd $DAOS_BASE # now run it! -pushd install/Linux/TESTING +pushd install/Linux/TESTING/ if [ \"$1\" = \"2\" ]; then cat < scripts/iof_fio_main.cfg { @@ -107,7 +107,7 @@ EOF cp scripts/iof_{fio,iozone}_main.cfg cp scripts/iof_{fio,mdtest}_main.cfg - rm -rf install/Linux/TESTING/$log_base_path/ + rm -rf $log_base_path/ python3 test_runner config=scripts/iof_fio_main.cfg \\ scripts/iof_multi_two_node.yml || { rc=\${PIPESTATUS[0]} @@ -154,7 +154,7 @@ elif [ \"$1\" = \"5\" ]; then \"log_base_path\": \"$log_base_path\" } EOF - rm -rf install/Linux/TESTING/$log_base_path/ + rm -rf $log_base_path/ python3 test_runner config=scripts/iof_multi_five_node.cfg \\ scripts/iof_multi_five_node.yml || { rc=\${PIPESTATUS[0]} @@ -178,9 +178,13 @@ TestGroup: user_name: jenkins Tests: EOF + pwd >&2 + ls -ld install install/Linux install/Linux/TESTING >&2 + ls -l install/Linux/TESTING >&2 find install/Linux/TESTING/$log_base_path -name subtest_results.yml \ -print0 | xargs -0 cat } > results_1.yml +cat results_1.yml PYTHONPATH=scony_python-junit/ jenkins/autotest_utils/results_to_junit.py From 2e75c7b8f29f2eb33c9d623eb561fa44f09fed4f Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Mon, 3 Dec 2018 14:40:08 -0500 Subject: [PATCH 33/44] More debug Change-Id: I0ad6d7939f6b6d2619e2df444f9e61c27324bc44 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 0e23292c..4da74a28 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -36,7 +36,7 @@ trap 'set +e i=5 # due to flakiness on wolf-53, try this several times while [ $i -gt 0 ]; do - pdsh -R ssh -S -w ${HOSTPREFIX}vm[1,$vmrange] "set -x + pdsh -R ssh -S -w "${HOSTPREFIX}$test_runner_vm,${HOSTPREFIX}vm[$vmrange]" "set -x x=0 while [ \$x -lt 30 ] && grep $DAOS_BASE /proc/mounts && @@ -179,8 +179,10 @@ TestGroup: Tests: EOF pwd >&2 + hostname >&2 ls -ld install install/Linux install/Linux/TESTING >&2 ls -l install/Linux/TESTING >&2 + df -h install/Linux/TESTING >&2 find install/Linux/TESTING/$log_base_path -name subtest_results.yml \ -print0 | xargs -0 cat } > results_1.yml From f73b0c68959d216e6fe220de47357c2665c9e5dd Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Mon, 3 Dec 2018 15:20:15 -0500 Subject: [PATCH 34/44] More debug Change-Id: I2157071447819ecdd3c6c75beaa368f9188c8ce3 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 4da74a28..ad399e2a 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -163,6 +163,11 @@ EOF find $log_base_path/testRun -name subtest_results.yml \\ -exec grep -Hi fail {} \\; fi +popd +ls -ld install install/Linux install/Linux/TESTING >&2 +ls -l install/Linux/TESTING >&2 +df -h install/Linux/TESTING >&2 +ls -ltar exit \$rc"; then rc=${PIPESTATUS[0]} else @@ -189,14 +194,13 @@ EOF cat results_1.yml PYTHONPATH=scony_python-junit/ jenkins/autotest_utils/results_to_junit.py - -if false; then -# collect the logs -if ! rpdcp -R ssh -w "${HOSTPREFIX}"vm[1,$vmrange] \ - /tmp/Functional_"$TEST_TAG"/\*daos.log "$PWD"/; then - echo "Copying daos.logs from remote nodes failed" - # pass -fi +ls -ltar + +## collect the logs +#if ! rpdcp -R ssh -w "${HOSTPREFIX}"vm[1,$vmrange] \ +# /tmp/Functional_"$TEST_TAG"/\*daos.log "$PWD"/; then +# echo "Copying daos.logs from remote nodes failed" +# # pass +#fi ls -l -fi exit "$rc" From 020ea936150de3e8586936586755008d628122f0 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Mon, 3 Dec 2018 16:06:36 -0500 Subject: [PATCH 35/44] Junit file name fix Change-Id: I2f2d6c726865719535303f66eab52bed4f681777 Signed-off-by: Brian J. Murrell --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c7f626c7..1f04fb3b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -255,7 +255,7 @@ pipeline { } */ always { - junit 'IOF_5-node_junit.xml' + junit 'IOF_2-node_junit.xml' archiveArtifacts artifacts: 'install/Linux/TESTING/testLogs/**' } } From 79150fd87f6e4c65fcd80eee3bf97401e79dceba Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Mon, 3 Dec 2018 16:06:41 -0500 Subject: [PATCH 36/44] More debug Change-Id: I53280ff44f9fc944f0193b9712ef739eae032538 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index ad399e2a..408ec696 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -31,6 +31,8 @@ fi log_base_path="testLogs-${1}_node" +rm -f results_1.yml IOF_[25]-node_junit.xml + # shellcheck disable=SC2154 trap 'set +e i=5 @@ -185,10 +187,21 @@ Tests: EOF pwd >&2 hostname >&2 - ls -ld install install/Linux install/Linux/TESTING >&2 - ls -l install/Linux/TESTING >&2 - df -h install/Linux/TESTING >&2 - find install/Linux/TESTING/$log_base_path -name subtest_results.yml \ + i=10 + while [ "$i" -gt 0 ]; do + ls -ld install install/Linux install/Linux/TESTING >&2 + ls -l install/Linux/TESTING >&2 + df -h install/Linux/TESTING >&2 + ssh "${HOSTPREFIX}$test_runner_vm" "set -x + if [ \"$i\" -lt 3 ]; then + sync; sync + fi + ls -l $PWD/install/Linux/TESTING + $DAOS_BASE/install/Linux/TESTING" >&2 + sleep 5 + let i-=1 + done + find install/Linux/TESTING/"$log_base_path" -name subtest_results.yml \ -print0 | xargs -0 cat } > results_1.yml cat results_1.yml From b4b3f315ed5e18230022a552fc7e349571cb5584 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Mon, 3 Dec 2018 17:12:42 -0500 Subject: [PATCH 37/44] Fix debug error Change-Id: Icf852d710d08e414b90bbffd6fb505d87feb9327 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 408ec696..5ea5023b 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -196,7 +196,7 @@ EOF if [ \"$i\" -lt 3 ]; then sync; sync fi - ls -l $PWD/install/Linux/TESTING + ls -l $PWD/install/Linux/TESTING \ $DAOS_BASE/install/Linux/TESTING" >&2 sleep 5 let i-=1 From f21b6e22113ec53539dc77fdd814842b23f98393 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Mon, 3 Dec 2018 20:39:53 -0500 Subject: [PATCH 38/44] More debug Change-Id: Ia9516df5e55cd89433783de0eb7a552c3138b463 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 5ea5023b..91f3ac94 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -193,11 +193,12 @@ EOF ls -l install/Linux/TESTING >&2 df -h install/Linux/TESTING >&2 ssh "${HOSTPREFIX}$test_runner_vm" "set -x + set +e if [ \"$i\" -lt 3 ]; then sync; sync fi - ls -l $PWD/install/Linux/TESTING \ - $DAOS_BASE/install/Linux/TESTING" >&2 + ls -l $DAOS_BASE/install/Linux/TESTING + exit 0" >&2 sleep 5 let i-=1 done From bee5bce7fdba351f8148612f3da4ff47a02f7be5 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Tue, 4 Dec 2018 08:16:12 -0500 Subject: [PATCH 39/44] Different approach; wean away from NFS Change-Id: Ie4c67659b9bd9894cab4ce9b4462aeff0acd23d6 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 91f3ac94..805f273b 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -176,6 +176,25 @@ else rc=0 fi +hostname +pwd +scp -a "${HOSTPREFIX}$test_runner_vm":$DAOS_BASE/install/Linux/TESTING/"$log_base_path" . +ls -l . "$log_base_path" +# i=10 +# while [ "$i" -gt 0 ]; do +# ls -ld install install/Linux install/Linux/TESTING >&2 +# ls -l install/Linux/TESTING >&2 +# df -h install/Linux/TESTING >&2 +# ssh "${HOSTPREFIX}$test_runner_vm" "set -x +# set +e +# if [ \"$i\" -lt 3 ]; then +# sync; sync +# fi +# ls -l $DAOS_BASE/install/Linux/TESTING +# exit 0" >&2 +# sleep 5 +# let i-=1 +# done { cat <&2 - hostname >&2 - i=10 - while [ "$i" -gt 0 ]; do - ls -ld install install/Linux install/Linux/TESTING >&2 - ls -l install/Linux/TESTING >&2 - df -h install/Linux/TESTING >&2 - ssh "${HOSTPREFIX}$test_runner_vm" "set -x - set +e - if [ \"$i\" -lt 3 ]; then - sync; sync - fi - ls -l $DAOS_BASE/install/Linux/TESTING - exit 0" >&2 - sleep 5 - let i-=1 - done - find install/Linux/TESTING/"$log_base_path" -name subtest_results.yml \ - -print0 | xargs -0 cat + ssh "${HOSTPREFIX}$test_runner_vm" "set -ex + find \"$DAOS_BASE\"/install/Linux/TESTING\"$log_base_path\" \ + -name subtest_results.yml -print0 | xargs -0 cat" } > results_1.yml cat results_1.yml From d37f008f849b1bf37bfc8c146d4d36430703f164 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Tue, 4 Dec 2018 08:50:52 -0500 Subject: [PATCH 40/44] Fixes Change-Id: I0de8d68177636327a06c2ab9a61cef6a069ba269 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 805f273b..5100aa91 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -178,8 +178,8 @@ fi hostname pwd -scp -a "${HOSTPREFIX}$test_runner_vm":$DAOS_BASE/install/Linux/TESTING/"$log_base_path" . -ls -l . "$log_base_path" +#scp -r "${HOSTPREFIX}$test_runner_vm":$DAOS_BASE/install/Linux/TESTING/"$log_base_path" . +#ls -l . "$log_base_path" # i=10 # while [ "$i" -gt 0 ]; do # ls -ld install install/Linux install/Linux/TESTING >&2 From 24c17764302da0827aa2cc71c9e69a2439647a36 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Tue, 4 Dec 2018 10:35:19 -0500 Subject: [PATCH 41/44] Missing path separator Change-Id: I1437b7ee847c2a32d165796dea19e62040abfcca Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 5100aa91..3486429e 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -205,7 +205,7 @@ TestGroup: Tests: EOF ssh "${HOSTPREFIX}$test_runner_vm" "set -ex - find \"$DAOS_BASE\"/install/Linux/TESTING\"$log_base_path\" \ + find \"$DAOS_BASE\"/install/Linux/TESTING/\"$log_base_path\" \ -name subtest_results.yml -print0 | xargs -0 cat" } > results_1.yml cat results_1.yml From b7bb4572a0961124a696c08e396497eb6f5708d2 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Tue, 4 Dec 2018 11:59:26 -0500 Subject: [PATCH 42/44] Fixes for archiving 2 and 5 node Change-Id: I8cefa450a37c39d594b82ea540c282dfe32dcbfa Signed-off-by: Brian J. Murrell --- Jenkinsfile | 4 ++-- multi-node-test.sh | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1f04fb3b..32bb524b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -256,7 +256,7 @@ pipeline { */ always { junit 'IOF_2-node_junit.xml' - archiveArtifacts artifacts: 'install/Linux/TESTING/testLogs/**' + archiveArtifacts artifacts: 'install/Linux/TESTING/testLogs-2_node/**' } } } @@ -293,7 +293,7 @@ pipeline { */ always { junit 'IOF_5-node_junit.xml' - archiveArtifacts artifacts: 'install/Linux/TESTING/testLogs/**' + archiveArtifacts artifacts: 'install/Linux/TESTING/testLogs-5_node/**' } } } diff --git a/multi-node-test.sh b/multi-node-test.sh index 3486429e..4119290a 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -178,7 +178,7 @@ fi hostname pwd -#scp -r "${HOSTPREFIX}$test_runner_vm":$DAOS_BASE/install/Linux/TESTING/"$log_base_path" . +scp -r "${HOSTPREFIX}$test_runner_vm":$DAOS_BASE/install/Linux/TESTING/"$log_base_path" install/Linux/TESTING/ #ls -l . "$log_base_path" # i=10 # while [ "$i" -gt 0 ]; do @@ -204,8 +204,7 @@ TestGroup: user_name: jenkins Tests: EOF - ssh "${HOSTPREFIX}$test_runner_vm" "set -ex - find \"$DAOS_BASE\"/install/Linux/TESTING/\"$log_base_path\" \ + find install/Linux/TESTING/"$log_base_path" \ -name subtest_results.yml -print0 | xargs -0 cat" } > results_1.yml cat results_1.yml From 3512d40b5fb88d0c3b49b5de5f047761d6d3e103 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Tue, 4 Dec 2018 13:51:27 -0500 Subject: [PATCH 43/44] Fix up some quoting Change-Id: Ib5b024412d0f1a68ec09c07f5a562c5079727c88 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 4119290a..96d356f6 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -178,7 +178,7 @@ fi hostname pwd -scp -r "${HOSTPREFIX}$test_runner_vm":$DAOS_BASE/install/Linux/TESTING/"$log_base_path" install/Linux/TESTING/ +scp -r "${HOSTPREFIX}$test_runner_vm:$DAOS_BASE/install/Linux/TESTING/$log_base_path" install/Linux/TESTING/ #ls -l . "$log_base_path" # i=10 # while [ "$i" -gt 0 ]; do @@ -205,7 +205,7 @@ TestGroup: Tests: EOF find install/Linux/TESTING/"$log_base_path" \ - -name subtest_results.yml -print0 | xargs -0 cat" + -name subtest_results.yml -print0 | xargs -0 cat } > results_1.yml cat results_1.yml From 9a687f949deac0eab3b5be4f99c1c923ed939ef9 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Wed, 5 Dec 2018 22:02:57 -0500 Subject: [PATCH 44/44] Install mdtest from jenkins to /testbin Change-Id: Ic1a91a8a164ad7d5499d6c410807f9557f8504e5 Signed-off-by: Brian J. Murrell --- multi-node-test.sh | 20 ++++++++++++++++++-- test/mdtest.yml | 1 + 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/multi-node-test.sh b/multi-node-test.sh index 96d356f6..eb86eee4 100755 --- a/multi-node-test.sh +++ b/multi-node-test.sh @@ -33,6 +33,9 @@ log_base_path="testLogs-${1}_node" rm -f results_1.yml IOF_[25]-node_junit.xml +curl -O 'https://build.hpdd.intel.com/job/mdtest-iof_devel/lastSuccessfulBuild/arch=x86_64,distro=el7/artifact/artifacts/mpich_files.tar.gz' +curl -O 'https://build.hpdd.intel.com/job/mdtest-iof_devel/lastSuccessfulBuild/arch=x86_64,distro=el7/artifact/artifacts/mdtest_files.tar.gz' + # shellcheck disable=SC2154 trap 'set +e i=5 @@ -78,6 +81,12 @@ fi # TODO: package this in to an RPM pip3 install --user tabulate +# install mdtest +sudo rm -rf /testbin/ +sudo mkdir -p /testbin/mdtest +cd /testbin/mdtest +sudo tar xzvf $DAOS_BASE/mpich_files.tar.gz +sudo tar xzvf $DAOS_BASE/mdtest_files.tar.gz df -h" 2>&1 | dshbak -c; then echo "Cluster setup (i.e. provisioning) failed" exit 1 @@ -100,7 +109,8 @@ pushd install/Linux/TESTING/ if [ \"$1\" = \"2\" ]; then cat < scripts/iof_fio_main.cfg { - \"host_list\": [\"${HOSTPREFIX}${vm1}\", \"${HOSTPREFIX}${vm2}\"], + \"should_be_host_list\": [\"${HOSTPREFIX}${vm1}\", \"${HOSTPREFIX}${vm2}\"], + \"host_list\": [\"${HOSTPREFIX}${test_runner_vm}\", \"${HOSTPREFIX}${vm1}\"], \"test_mode\": \"littleChief\", \"log_base_path\": \"$log_base_path\" } @@ -145,13 +155,19 @@ EOF elif [ \"$1\" = \"5\" ]; then cat < scripts/iof_multi_five_node.cfg { - \"host_list\": [ + \"should_be_host_list\": [ \"${HOSTPREFIX}vm2\", \"${HOSTPREFIX}vm3\", \"${HOSTPREFIX}vm4\", \"${HOSTPREFIX}vm5\", \"${HOSTPREFIX}vm6\" ], + \"host_list\": [\"${HOSTPREFIX}${test_runner_vm}\", + \"${HOSTPREFIX}vm2\", + \"${HOSTPREFIX}vm3\", + \"${HOSTPREFIX}vm4\", + \"${HOSTPREFIX}vm5\" + ], \"test_mode\": \"littleChief\", \"log_base_path\": \"$log_base_path\" } diff --git a/test/mdtest.yml b/test/mdtest.yml index b7e7b38c..5c0aeb46 100644 --- a/test/mdtest.yml +++ b/test/mdtest.yml @@ -2,6 +2,7 @@ description: "scripts test module" defaultENV: D_LOG_MASK: "WARN" + LD_LIBRARY_PATH: "/testbin/mdtest/lib" module: name: "mdtest"