diff --git a/compiler/circle-resizer-dredd-recipe-test/CMakeLists.txt b/compiler/circle-resizer-dredd-recipe-test/CMakeLists.txt new file mode 100644 index 00000000000..cd80070cc13 --- /dev/null +++ b/compiler/circle-resizer-dredd-recipe-test/CMakeLists.txt @@ -0,0 +1,117 @@ +if(NOT ENABLE_TEST) + return() +endif(NOT ENABLE_TEST) + +nnas_include(TargetRequire) + +unset(REQUIRED_TARGETS) +list(APPEND REQUIRED_TARGETS circle-inspect) +list(APPEND REQUIRED_TARGETS circle-verify) +list(APPEND REQUIRED_TARGETS circle_resizer) +list(APPEND REQUIRED_TARGETS dredd_rule_lib) +TargetRequire_Return(${REQUIRED_TARGETS}) + +unset(TEST_DEPS) +unset(TEST_NAMES) + +get_target_property(ARTIFACTS_BIN_PATH testDataGenerator BINARY_DIR) + +set(oneValueArgs NEW_INPUTS_SIZES) +set(multiValueArgs "") + +macro(Add RECIPE) + cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + set(NEW_INPUTS_SIZES "") + if(ARG_NEW_INPUTS_SIZES) + set(NEW_INPUTS_SIZES "--input_shapes" "${ARG_NEW_INPUTS_SIZES}") + endif() + + set(CIRCLE_PATH "${ARTIFACTS_BIN_PATH}/${RECIPE}.circle") + set(RESIZED_CIRCLE_PATH "${CMAKE_CURRENT_BINARY_DIR}/${RECIPE}.resized.circle") + + # Generate resized .circle + add_custom_command(OUTPUT ${RESIZED_CIRCLE_PATH} + COMMAND $ --input_path ${CIRCLE_PATH} --output_path ${RESIZED_CIRCLE_PATH} ${NEW_INPUTS_SIZES} + DEPENDS + circle_resizer + ${CIRCLE_PATH} + COMMENT "Generate ${RECIPE}.resized.circle" + ) + + list(APPEND TEST_DEPS ${RESIZED_CIRCLE_PATH}) + list(APPEND TEST_NAMES ${RECIPE}) +endmacro(Add) + + +# Read "test.lst" +include("test.lst") + +## +## Copy testall +## +set(TEST_RUNNER "${CMAKE_CURRENT_BINARY_DIR}/testall.sh") +set(TEST_RUNNER_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/testall.sh") + +add_custom_command( + OUTPUT ${TEST_RUNNER} + COMMAND ${CMAKE_COMMAND} -E copy "${TEST_RUNNER_SOURCE}" "${TEST_RUNNER}" + DEPENDS ${TEST_RUNNER_SOURCE} + COMMENT "Generate test runner" +) + +list(APPEND TEST_DEPS "${TEST_RUNNER}") + +### +### Generate test.config +### +set(TEST_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/test.config") + +add_custom_command( + OUTPUT ${TEST_CONFIG} + COMMAND ${CMAKE_COMMAND} -E remove -f ${TEST_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'CIRCLE_INSPECT_PATH=\"$\"' >> ${TEST_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'CIRCLE_VERIFY_PATH=\"$\"' >> ${TEST_CONFIG} + COMMAND ${CMAKE_COMMAND} -E echo 'CIRCLE_RESIZER_PATH=\"$\"' >> ${TEST_CONFIG} + DEPENDS + circle-inspect + circle-verify + circle_resizer + COMMENT "Generate test configuration" +) + +list(APPEND TEST_DEPS "${TEST_CONFIG}") + +# +# copy rule-lib.sh (a library of shell script functions) +# + +# getting path for rule-lib.sh in dredd-rule-lib +get_target_property(DREDD_RULE_LIB_DIR dredd_rule_lib BINARY_DIR) + +set(RULE_LIB_SOURCE_PATH "${DREDD_RULE_LIB_DIR}/rule-lib.sh") +set(RULE_LIB_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/rule-lib.sh") + +add_custom_command( + OUTPUT ${RULE_LIB_BINARY_PATH} + COMMAND ${CMAKE_COMMAND} -E copy "${RULE_LIB_SOURCE_PATH}" "${RULE_LIB_BINARY_PATH}" + DEPENDS ${RULE_LIB_SOURCE_PATH} + COMMENT "Generate rule lib" +) + +list(APPEND TEST_DEPS "${RULE_LIB_BINARY_PATH}") + +# Generate dependencies +add_custom_target(circle-resizer_dredd_recipe_test ALL DEPENDS ${TEST_DEPS}) +add_dependencies(circle-resizer_dredd_recipe_test common_artifacts_deps) + +get_target_property(ARTIFACTS_BIN_PATH testDataGenerator BINARY_DIR) + +# Run tests +add_test( + NAME resizer_dredd_recipe_test + COMMAND "${TEST_RUNNER}" + "${TEST_CONFIG}" + "${ARTIFACTS_BIN_PATH}" + ${TEST_NAMES} +) diff --git a/compiler/circle-resizer-dredd-recipe-test/README.md b/compiler/circle-resizer-dredd-recipe-test/README.md new file mode 100644 index 00000000000..ea65d7319ee --- /dev/null +++ b/compiler/circle-resizer-dredd-recipe-test/README.md @@ -0,0 +1,31 @@ +# circle-resizer-dredd-recipe-test + +It tests non-functional conditions of a circle model resized by circle-resizer. + +## How to add a test? + +1. Create a directory under `res/TensorFlowLiteRecipes/` or `res/CircleRecipes/`. + +2. Make a recipe (`test.recipe`) for a model under the directory. + +3. Make a rule (`test.rule`) you want to test under the directory. Note, that you can find more information about dredd-test-rules in _dredd-rule-lib_ module. + +4. Add a test to `test.lst` in this module using `Add` macro. + ``` + Add(RECIPE_DIR NEW_INPUTS_SIZES) + ``` + - `NEW_INPUTS_SIZES`: New shapes of Circle model inputs in comma-separated format like `[1,2,3],[4,5]` for a model with 2 inputs. + +## Example + +``` +# TensorFlowLiteRecipes +res/TensorFlowLiteRecipes/PRelu_000 +├── test.recipe # What you want to test +└── test.rule # Non-functional conditions to be satisfied + +# test.lst +... +Add(PRelu_000 NEW_INPUTS_SIZES [1,4,4,5],[1,1,5]) +... +``` diff --git a/compiler/circle-resizer-dredd-recipe-test/requires.cmake b/compiler/circle-resizer-dredd-recipe-test/requires.cmake new file mode 100644 index 00000000000..d5e55f70dc6 --- /dev/null +++ b/compiler/circle-resizer-dredd-recipe-test/requires.cmake @@ -0,0 +1,5 @@ +require("circle-inspect") +require("circle-resizer") +require("circle-verify") +require("common-artifacts") +require("dredd-rule-lib") diff --git a/compiler/circle-resizer-dredd-recipe-test/test.lst b/compiler/circle-resizer-dredd-recipe-test/test.lst new file mode 100644 index 00000000000..3326ce0993c --- /dev/null +++ b/compiler/circle-resizer-dredd-recipe-test/test.lst @@ -0,0 +1,19 @@ +## EXAMPLE +# +# Add(RECIPE_DIR NEW_INPUTS_SIZES [1,2,3]) +# + +## TFLITE RECIPE + +# two inputs, one output +Add(PRelu_002 NEW_INPUTS_SIZES [1,4,4,5],[1,1,5]) +# scalar output +Add(ReduceAny_dynamic_004 NEW_INPUTS_SIZES [4,5,6]) +# change rank +Add(GreaterEqual_001 NEW_INPUTS_SIZES [1,2,3],[1,2,3]) +# one inputs, two outputs +Add(Split_001 NEW_INPUTS_SIZES [8,1,2]) +# bigger graph +Add(Net_FullyConnected_Gelu_001 NEW_INPUTS_SIZES [2,16]) +# from dynamic to static +Add(Inf_StridedSlice_002 NEW_INPUTS_SIZES [1,10,10,5]) diff --git a/compiler/circle-resizer-dredd-recipe-test/testall.sh b/compiler/circle-resizer-dredd-recipe-test/testall.sh new file mode 100755 index 00000000000..18f47137c13 --- /dev/null +++ b/compiler/circle-resizer-dredd-recipe-test/testall.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +# Need at least 2 arguments +if [[ $# -lt 2 ]]; then + echo "USAGE: $0 ..." + echo + echo "ARGUMENTS:" + echo " [test.config path]" + echo " [WORKDIR]" + echo " [Prefix1]" + echo " [Prefix2]" + echo " ..." + exit 255 +fi + +WORKDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +CONFIG_PATH="$1"; shift +RESOURCE_DIR="$1"; shift + +source "${CONFIG_PATH}" + +echo "-- Found circle-resizer: ${CIRCLE_RESIZER_PATH}" +echo "-- Found circle-inspect: ${CIRCLE_INSPECT_PATH}" +echo "-- Found circle-verify: ${CIRCLE_VERIFY_PATH}" +echo "-- Found common-artifacts: ${RESOURCE_DIR}" + +TESTED=() +PASSED=() +FAILED=() + +pushd ${WORKDIR} +while [[ $# -ne 0 ]]; do + PREFIX="$1"; shift + + TESTED+=("${PREFIX}") + + PASSED_TAG="${PREFIX}.passed" + + rm -f "${PASSED_TAG}" + + cat > "${PREFIX}.log" <( + exec 2>&1 + + echo "-- Found circle: ${PREFIX}.resized.circle" + + # Exit immediately if any command fails + set -e + # Show commands + set -x + + # + # Check if rule is satisfied + # + + # Note: turn off 'command printing'. Otherwise printing will be so messy + set +x + + # (COMPILED_FILE, INSPECT_PROG_PATH, VERIFY_PROG_PATH, ERROR_LOG) must be set for rule-lib.sh + COMPILED_FILE="${PREFIX}.resized.circle" + INSPECT_PROG_PATH=${CIRCLE_INSPECT_PATH} + VERIFY_PROG_PATH=${CIRCLE_VERIFY_PATH} + ERROR_LOG="${PREFIX}.error" + + rm -f "${ERROR_LOG}" + + # in case error while running rule-lib.sh, prints error msg + trap 'echo "** ERROR **" ; cat "${ERROR_LOG}"' ERR + + source rule-lib.sh + source "${RESOURCE_DIR}/${PREFIX}.rule" + + # unset + trap - ERR + set -x + + # At this point, the exit code of all commands is 0 + # If not 0, execution of this script ends because of "set -e" + touch "${PASSED_TAG}" + ) + + if [[ -f "${PASSED_TAG}" ]]; then + PASSED+=("$PREFIX") + else + FAILED+=("$PREFIX") + fi +done +popd + +if [[ ${#TESTED[@]} -ne ${#PASSED[@]} ]]; then + echo "FAILED" + for TEST in "${FAILED[@]}" + do + echo "- ${TEST}" + done + exit 255 +fi + +echo "PASSED" +exit 0