diff --git a/compiler/common-artifacts/CMakeLists.txt b/compiler/common-artifacts/CMakeLists.txt index 9795a705094..51166349139 100644 --- a/compiler/common-artifacts/CMakeLists.txt +++ b/compiler/common-artifacts/CMakeLists.txt @@ -1,14 +1,14 @@ #[[ Generate common python virtual enviornment ]] -# NOTE find_package try to use at least python3.8 as follows depending on platform version -# Ubuntu18.04; explictly installed python3.8 (default is python3.6) -# Ubuntu20.04; default python3.8 +# NOTE find_package try to use at least python3.10 as follows depending on platform version +# Ubuntu20.04; explictly installed python3.10 (default is python3.8) # Ubuntu22.04; default python3.10 # Ubuntu24.04; default python3.12 # refer https://github.com/Samsung/ONE/issues/9962 -# find python 3.8 or above -find_package(Python 3.8 EXACT COMPONENTS Interpreter QUIET) +# refer https://github.com/Samsung/ONE/issues/15226 +# find python 3.10 or above +find_package(Python 3.10 EXACT COMPONENTS Interpreter QUIET) if(NOT Python_FOUND) - find_package(Python 3.8 COMPONENTS Interpreter QUIET) + find_package(Python 3.10 COMPONENTS Interpreter QUIET) endif() if(NOT Python_Interpreter_FOUND) @@ -16,10 +16,8 @@ if(NOT Python_Interpreter_FOUND) return() endif() -# NOTE assume only use 3.8.x or 3.10.x or 3.12.x -if((Python_VERSION VERSION_GREATER_EQUAL 3.8) AND (Python_VERSION VERSION_LESS 3.9)) - set(PYTHON_VERSION_MINOR 8) -elseif((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11)) +# NOTE assume only use 3.10.x or 3.12.x +if((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11)) set(PYTHON_VERSION_MINOR 10) elseif((Python_VERSION VERSION_GREATER_EQUAL 3.12) AND (Python_VERSION VERSION_LESS 3.13)) set(PYTHON_VERSION_MINOR 12) @@ -48,8 +46,8 @@ set(REQUIREMENTS_FILE "requirements.txt") set(REQUIREMENTS_OVERLAY_PATH "${VIRTUALENV_OVERLAY}/${REQUIREMENTS_FILE}") set(PYTHON_OVERLAY python3) -if(PYTHON_EXECUTABLE MATCHES python3.8) - set(PYTHON_OVERLAY python3.8) +if(PYTHON_EXECUTABLE MATCHES python3.10) + set(PYTHON_OVERLAY python3.10) endif() # NOTE when using behind proxy with self signed certificate, need to set '--trusted-host' options @@ -61,21 +59,7 @@ endif() # NOTE `tensorflow-cpu` package is not available for aarch64, so we use `tensorflow` package. # NOTE refer https://github.com/Samsung/ONE/issues/15226 for versions set(PY_PKG_LIST ) -if (PYTHON_VERSION_MINOR EQUAL 8) - # python3.8; Ubuntu 18.04, 20.04 - if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64") - list(APPEND PY_PKG_LIST "tensorflow==2.12.1") - else(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64") - list(APPEND PY_PKG_LIST "tensorflow-cpu==2.12.1") - endif(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64") - list(APPEND PY_PKG_LIST "flatbuffers==23.5.26") - list(APPEND PY_PKG_LIST "protobuf==4.23.3") - list(APPEND PY_PKG_LIST "pydot==1.4.2") - list(APPEND PY_PKG_LIST "pytest==7.4.3") - list(APPEND PY_PKG_LIST "h5py==3.11.0") - list(APPEND PY_PKG_LIST "cffi==1.16.0") - list(APPEND PY_PKG_LIST "numpy==1.24.3") -elseif (PYTHON_VERSION_MINOR EQUAL 10) +if (PYTHON_VERSION_MINOR EQUAL 10) # python 3.10; Ubuntu 22.04 # TF2.19.0 release at 2025-03-12 if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64") diff --git a/compiler/dalgona/CMakeLists.txt b/compiler/dalgona/CMakeLists.txt index f040e8010e5..64d164ce3fc 100644 --- a/compiler/dalgona/CMakeLists.txt +++ b/compiler/dalgona/CMakeLists.txt @@ -1,14 +1,14 @@ -# NOTE find_package will try to use at least python3.8 as follows depending on platform version -# Ubuntu18.04; explictly installed python3.8 (default is python3.6) -# Ubuntu20.04; default python3.8 +# NOTE find_package will try to use at least python3.10 as follows depending on platform version +# Ubuntu20.04; explictly installed python3.10 (default is python3.8) # Ubuntu22.04; default python3.10 # Ubuntu24.04; default python3.12 # refer https://github.com/Samsung/ONE/issues/9962 +# refer https://github.com/Samsung/ONE/issues/15226 # NOTE Require same python version of common-artifacts -# find python 3.8 or above -find_package(Python 3.8 EXACT COMPONENTS Interpreter Development QUIET) +# find python 3.10 or above +find_package(Python 3.10 EXACT COMPONENTS Interpreter Development QUIET) if(NOT Python_FOUND) - find_package(Python 3.8 COMPONENTS Interpreter Development QUIET) + find_package(Python 3.10 COMPONENTS Interpreter Development QUIET) endif() if(NOT Python_Development_FOUND) @@ -16,10 +16,8 @@ if(NOT Python_Development_FOUND) return() endif() -# NOTE assume only use 3.8.x or 3.10.x or 3.12.x -if((Python_VERSION VERSION_GREATER_EQUAL 3.8) AND (Python_VERSION VERSION_LESS 3.9)) - set(PYTHON_VERSION_MINOR 8) -elseif((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11)) +# NOTE assume only use 3.10.x or 3.12.x +if((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11)) set(PYTHON_VERSION_MINOR 10) elseif((Python_VERSION VERSION_GREATER_EQUAL 3.12) AND (Python_VERSION VERSION_LESS 3.13)) set(PYTHON_VERSION_MINOR 12) diff --git a/compiler/one-cmds/CMakeLists.txt b/compiler/one-cmds/CMakeLists.txt index e207d2326e7..0d4f98b4db1 100644 --- a/compiler/one-cmds/CMakeLists.txt +++ b/compiler/one-cmds/CMakeLists.txt @@ -1,12 +1,12 @@ -# NOTE find_package try to use at least python3.8 as follows depending on platform version -# Ubuntu18.04; explictly installed python3.8 (default is python3.6) -# Ubuntu20.04; default python3.8 +# NOTE find_package will try to use at least python3.10 as follows depending on platform version +# Ubuntu20.04; explictly installed python3.10 (default is python3.8) # Ubuntu22.04; default python3.10 # Ubuntu24.04; explicitly installed python3.8 (default is python3.12) # refer https://github.com/Samsung/ONE/issues/9962 -find_package(Python 3.8 EXACT COMPONENTS Interpreter QUIET) +# refer https://github.com/Samsung/ONE/issues/15226 +find_package(Python 3.10 EXACT COMPONENTS Interpreter QUIET) if(NOT Python_FOUND) - find_package(Python 3.8 COMPONENTS Interpreter QUIET) + find_package(Python 3.10 COMPONENTS Interpreter QUIET) endif() if(NOT Python_Interpreter_FOUND) @@ -14,11 +14,9 @@ if(NOT Python_Interpreter_FOUND) return() endif() -# NOTE assume only use 3.8.x or 3.10.x or 3.12.x +# NOTE assume only use 3.10.x or 3.12.x # NOTE PYTHON_VERSION_MINOR is not used but added for consistancy with common-artifacts and dalgona -if((Python_VERSION VERSION_GREATER_EQUAL 3.8) AND (Python_VERSION VERSION_LESS 3.9)) - set(PYTHON_VERSION_MINOR 8) -elseif((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11)) +if((Python_VERSION VERSION_GREATER_EQUAL 3.10) AND (Python_VERSION VERSION_LESS 3.11)) set(PYTHON_VERSION_MINOR 10) elseif((Python_VERSION VERSION_GREATER_EQUAL 3.12) AND (Python_VERSION VERSION_LESS 3.13)) set(PYTHON_VERSION_MINOR 12) diff --git a/compiler/one-cmds/one-prepare-venv b/compiler/one-cmds/one-prepare-venv index b8331b89ca8..90ddbef0a83 100644 --- a/compiler/one-cmds/one-prepare-venv +++ b/compiler/one-cmds/one-prepare-venv @@ -57,22 +57,16 @@ source /etc/lsb-release # - https://github.com/onnx/onnx/blob/master/docs/Versioning.md # - https://github.com/onnx/onnx-tensorflow/blob/master/Versioning.md -VER_TENSORFLOW=2.12.1 -VER_TF_PROB=0.20.1 -VER_TF_ADON=0.20.0 -VER_ONNX=1.16.0 -VER_ONNXRUNTIME=1.18.0 -VER_ONNX_TF=1.10.0 +VER_TENSORFLOW=2.19.0 +VER_ONNX=1.18.0 +VER_ONNXRUNTIME=1.21.1 VER_PYDOT=1.4.2 -VER_TORCH="2.1.2+cpu" -VER_PROTOBUF=4.23.3 -VER_NUMPY=1.24.3 +VER_TORCH="2.7.0+cpu" +VER_PROTOBUF="5.29.4" +VER_NUMPY="1.26.4" PYTHON_VER=$(${PYTHON3_EXEC} -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" ) -echo "Setting package version for python $PYTHON_VER" -if [[ "$PYTHON_VER" == "3.8" ]]; then - : # use as is -elif [[ "$PYTHON_VER" == "3.10" ]]; then +if [[ "$PYTHON_VER" == "3.10" ]]; then : # TODO change vesions elif [[ "$PYTHON_VER" == "3.12" ]]; then : # TODO change vesions @@ -110,12 +104,9 @@ ${VENV_PYTHON} -m pip ${PIP_OPTIONS} install --upgrade pip setuptools PYTHON_PACKAGES="tensorflow-cpu==${VER_TENSORFLOW} " PYTHON_PACKAGES+="Pillow " -PYTHON_PACKAGES+="tensorflow_probability==${VER_TF_PROB} " -PYTHON_PACKAGES+="tensorflow_addons==${VER_TF_ADON} " PYTHON_PACKAGES+="torch==${VER_TORCH} " PYTHON_PACKAGES+="onnx==${VER_ONNX} " PYTHON_PACKAGES+="onnxruntime==${VER_ONNXRUNTIME} " -PYTHON_PACKAGES+="onnx-tf==${VER_ONNX_TF} " PYTHON_PACKAGES+="protobuf==${VER_PROTOBUF} " PYTHON_PACKAGES+="fsspec==2024.6.1 " PYTHON_PACKAGES+="pydot==${VER_PYDOT} " diff --git a/compiler/one-cmds/tests/one-quantize_009.qconf.json b/compiler/one-cmds/tests/one-quantize_009.qconf.json index d4bac36b53f..559c3a66180 100644 --- a/compiler/one-cmds/tests/one-quantize_009.qconf.json +++ b/compiler/one-cmds/tests/one-quantize_009.qconf.json @@ -5,7 +5,9 @@ { "name" : "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Mixed_6a/Branch_1/Conv2d_0a_1x1/Conv2D;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D", "alternate": { - "tf2190": "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D" + "tf2190": [ + "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D" + ] }, "dtype" : "int16", "granularity" : "channel" diff --git a/compiler/one-cmds/tests/one-quantize_012.qconf.json b/compiler/one-cmds/tests/one-quantize_012.qconf.json index a9d464a55d5..6cbb7594278 100644 --- a/compiler/one-cmds/tests/one-quantize_012.qconf.json +++ b/compiler/one-cmds/tests/one-quantize_012.qconf.json @@ -10,7 +10,9 @@ "InceptionV3/InceptionV3/Mixed_7c/concat", "InceptionV3/Predictions/Reshape_1"], "alternate": { - "tf2190": "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D" + "tf2190": [ + "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D" + ] }, "dtype" : "int16", "granularity" : "channel" diff --git a/compiler/one-cmds/tests/one-quantize_013.qconf.json b/compiler/one-cmds/tests/one-quantize_013.qconf.json index a9d464a55d5..6cbb7594278 100644 --- a/compiler/one-cmds/tests/one-quantize_013.qconf.json +++ b/compiler/one-cmds/tests/one-quantize_013.qconf.json @@ -10,7 +10,9 @@ "InceptionV3/InceptionV3/Mixed_7c/concat", "InceptionV3/Predictions/Reshape_1"], "alternate": { - "tf2190": "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D" + "tf2190": [ + "InceptionV3/InceptionV3/Conv2d_2b_3x3/Relu;InceptionV3/InceptionV3/Conv2d_2b_3x3/BatchNorm/FusedBatchNorm;InceptionV3/InceptionV3/Conv2d_2b_3x3/Conv2D" + ] }, "dtype" : "int16", "granularity" : "channel" diff --git a/compiler/tf2tfliteV2/tf2tfliteV2.py b/compiler/tf2tfliteV2/tf2tfliteV2.py index e79c6ccb97a..fcc7b1d249b 100755 --- a/compiler/tf2tfliteV2/tf2tfliteV2.py +++ b/compiler/tf2tfliteV2/tf2tfliteV2.py @@ -263,9 +263,8 @@ def _convert(flags): _apply_verbosity(flags.verbose) if (flags.v1): - _v1_convert(flags) - else: - _v2_convert(flags) + print("tf2tfliteV2: v1 not supported anymore. convert with v2.") + _v2_convert(flags) """ diff --git a/docs/howto/how-to-build-compiler.md b/docs/howto/how-to-build-compiler.md index 955d5d1e315..d9841a6b41a 100644 --- a/docs/howto/how-to-build-compiler.md +++ b/docs/howto/how-to-build-compiler.md @@ -116,6 +116,71 @@ $ NNCC_WORKSPACE=build/release ./nncc build ``` will build release version in `build/release` folder. +## Local install and command line tests + +Local install test is to make an local installation in `build/install` path, +check the build artifacts and run compiler command line tests of `onecc` +and others. + +### Prepare onnx2circle + +`one-import-onnx` tool now uses `onnx2circle` tool from `circle-mlir`. + +`onnx2cicle` tool can be prepared with (1) build from source or (2) install +from [launchpad.net](https://launchpad.net/~circletools). + +Please read [README.md](../circle-mlir/README.md) for details how to build. + +As `circle-mlir/externals` uses lots of storage(about 150~200GB) and time, +we recommand to use build with Docker image as described in above README file. + +To use Debian package from [launchpad.net](https://launchpad.net/~circletools), +``` +sudo apt-get update +sudo apt-get install software-properties-common +sudo -E add-apt-repository ppa:circletools/nightly +sudo apt-get install onnx2circle +``` +- this will install `onnx2circle` tool to `/usr/share/circletools` folder + +`one-cmds/CMakeLists.txt` will install `onnx2circle` from (1) or (2) to +`buid/install` as conversion tool of `one-import-onnx` command. + +### How to make local installation +``` +NNAS_BUILD_PREFIX=build/setup \ +BUILD_TYPE=Release \ +./nnas create-package --preset 20230907 --prefix build/install +``` +This will configure and build necessary modules and install files in +`build/install` path. + +### Local command line tests + +To run local command line tests, python virtual envirnment is necessary with +test models. + +Prepare python virtual environment: +``` +pushd build/install/bin +bash ./one-prepare-venv +popd +``` + +Download test models: +``` +pushd build/install/test +bash ./prepare_test_materials.sh +popd +``` + +Run command line tests: +``` +pushd build/install/test +bash ./runtestall.sh +popd +``` + ## Build for Windows To build for Windows, we use MinGW(Minimalist GNU for Windows). [Here](https://github.com/git-for-windows/build-extra/releases) you can download a tool that includes it.